« 查询时覆盖字段值 索引运行时字段 »
Elastic 文档 ›Elasticsearch 指南 [8.16] ›映射 ›运行时字段

检索运行时字段

编辑

检索运行时字段

编辑

使用 fields 参数在 _search API 上检索运行时字段的值。运行时字段不会显示在 _source 中，但 fields API 适用于所有字段，即使那些不是作为原始 _source 部分发送的字段。

定义一个计算星期几的运行时字段

编辑

例如，以下请求添加一个名为 day_of_week 的运行时字段。运行时字段包含一个脚本，该脚本根据 @timestamp 字段的值计算星期几。我们将在请求中包含 "dynamic":"runtime"，以便将新字段作为运行时字段添加到映射中。

resp = client.indices.create(
    index="my-index-000001",
    mappings={
        "dynamic": "runtime",
        "runtime": {
            "day_of_week": {
                "type": "keyword",
                "script": {
                    "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
                }
            }
        },
        "properties": {
            "@timestamp": {
                "type": "date"
            }
        }
    },
)
print(resp)
const response = await client.indices.create({
  index: "my-index-000001",
  mappings: {
    dynamic: "runtime",
    runtime: {
      day_of_week: {
        type: "keyword",
        script: {
          source:
            "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))",
        },
      },
    },
    properties: {
      "@timestamp": {
        type: "date",
      },
    },
  },
});
console.log(response);
PUT my-index-000001/
{
  "mappings": {
    "dynamic": "runtime",
    "runtime": {
      "day_of_week": {
        "type": "keyword",
        "script": {
          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
        }
      }
    },
    "properties": {
      "@timestamp": {"type": "date"}
    }
  }
}

摄入一些数据

编辑

让我们摄入一些示例数据，这将产生两个索引字段：@timestamp 和 message。

resp = client.bulk(
    index="my-index-000001",
    refresh=True,
    operations=[
        {
            "index": {}
        },
        {
            "@timestamp": "2020-06-21T15:00:01-05:00",
            "message": "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-06-21T15:00:01-05:00",
            "message": "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:30:17-05:00",
            "message": "40.135.0.0 - - [2020-04-30T14:30:17-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:30:53-05:00",
            "message": "232.0.0.0 - - [2020-04-30T14:30:53-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:12-05:00",
            "message": "26.1.0.0 - - [2020-04-30T14:31:12-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:19-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:19-05:00] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:27-05:00",
            "message": "252.0.0.0 - - [2020-04-30T14:31:27-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:29-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_brdl.gif HTTP/1.0\" 304 0"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:29-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_arw.gif HTTP/1.0\" 304 0"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:32-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:32-05:00] \"GET /images/nav_bg_top.gif HTTP/1.0\" 200 929"
        },
        {
            "index": {}
        },
        {
            "@timestamp": "2020-04-30T14:31:43-05:00",
            "message": "247.37.0.0 - - [2020-04-30T14:31:43-05:00] \"GET /french/images/nav_venue_off.gif HTTP/1.0\" 304 0"
        }
    ],
)
print(resp)
response = client.bulk(
  index: 'my-index-000001',
  refresh: true,
  body: [
    {
      index: {}
    },
    {
      "@timestamp": '2020-06-21T15:00:01-05:00',
      message: '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-06-21T15:00:01-05:00',
      message: '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:30:17-05:00',
      message: '40.135.0.0 - - [2020-04-30T14:30:17-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:30:53-05:00',
      message: '232.0.0.0 - - [2020-04-30T14:30:53-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:12-05:00',
      message: '26.1.0.0 - - [2020-04-30T14:31:12-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:19-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:19-05:00] "GET /french/splash_inet.html HTTP/1.0" 200 3781'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:27-05:00',
      message: '252.0.0.0 - - [2020-04-30T14:31:27-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:29-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_brdl.gif HTTP/1.0" 304 0'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:29-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_arw.gif HTTP/1.0" 304 0'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:32-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:32-05:00] "GET /images/nav_bg_top.gif HTTP/1.0" 200 929'
    },
    {
      index: {}
    },
    {
      "@timestamp": '2020-04-30T14:31:43-05:00',
      message: '247.37.0.0 - - [2020-04-30T14:31:43-05:00] "GET /french/images/nav_venue_off.gif HTTP/1.0" 304 0'
    }
  ]
)
puts response
const response = await client.bulk({
  index: "my-index-000001",
  refresh: "true",
  operations: [
    {
      index: {},
    },
    {
      "@timestamp": "2020-06-21T15:00:01-05:00",
      message:
        '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-06-21T15:00:01-05:00",
      message:
        '211.11.9.0 - - [2020-06-21T15:00:01-05:00] "GET /english/index.html HTTP/1.0" 304 0',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:30:17-05:00",
      message:
        '40.135.0.0 - - [2020-04-30T14:30:17-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:30:53-05:00",
      message:
        '232.0.0.0 - - [2020-04-30T14:30:53-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:12-05:00",
      message:
        '26.1.0.0 - - [2020-04-30T14:31:12-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:19-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:19-05:00] "GET /french/splash_inet.html HTTP/1.0" 200 3781',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:27-05:00",
      message:
        '252.0.0.0 - - [2020-04-30T14:31:27-05:00] "GET /images/hm_bg.jpg HTTP/1.0" 200 24736',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:29-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_brdl.gif HTTP/1.0" 304 0',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:29-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:29-05:00] "GET /images/hm_arw.gif HTTP/1.0" 304 0',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:32-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:32-05:00] "GET /images/nav_bg_top.gif HTTP/1.0" 200 929',
    },
    {
      index: {},
    },
    {
      "@timestamp": "2020-04-30T14:31:43-05:00",
      message:
        '247.37.0.0 - - [2020-04-30T14:31:43-05:00] "GET /french/images/nav_venue_off.gif HTTP/1.0" 304 0',
    },
  ],
});
console.log(response);
POST /my-index-000001/_bulk?refresh
{ "index": {}}
{ "@timestamp": "2020-06-21T15:00:01-05:00", "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-06-21T15:00:01-05:00", "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [2020-04-30T14:30:17-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [2020-04-30T14:30:53-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [2020-04-30T14:31:12-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:19-05:00] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [2020-04-30T14:31:27-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:29-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_brdl.gif HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:29-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:29-05:00] \"GET /images/hm_arw.gif HTTP/1.0\" 304 0"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:32-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:32-05:00] \"GET /images/nav_bg_top.gif HTTP/1.0\" 200 929"}
{ "index": {}}
{ "@timestamp": "2020-04-30T14:31:43-05:00", "message" : "247.37.0.0 - - [2020-04-30T14:31:43-05:00] \"GET /french/images/nav_venue_off.gif HTTP/1.0\" 304 0"}

搜索计算出的星期几

编辑

以下请求使用搜索 API 来检索原始请求在映射中定义为运行时字段的 day_of_week 字段。此字段的值是在查询时动态计算的，无需重新索引文档或索引 day_of_week 字段。这种灵活性允许您修改映射而不更改任何字段值。

resp = client.search(
    index="my-index-000001",
    fields=[
        "@timestamp",
        "day_of_week"
    ],
    source=False,
)
print(resp)
response = client.search(
  index: 'my-index-000001',
  body: {
    fields: [
      '@timestamp',
      'day_of_week'
    ],
    _source: false
  }
)
puts response
const response = await client.search({
  index: "my-index-000001",
  fields: ["@timestamp", "day_of_week"],
  _source: false,
});
console.log(response);
GET my-index-000001/_search
{
  "fields": [
    "@timestamp",
    "day_of_week"
  ],
  "_source": false
}

之前的请求返回所有匹配文档的 day_of_week 字段。我们可以定义另一个名为 client_ip 的运行时字段，它也作用于 message 字段，并将进一步细化查询。

resp = client.indices.put_mapping(
    index="my-index-000001",
    runtime={
        "client_ip": {
            "type": "ip",
            "script": {
                "source": "String m = doc[\"message\"].value; int end = m.indexOf(\" \"); emit(m.substring(0, end));"
            }
        }
    },
)
print(resp)
response = client.indices.put_mapping(
  index: 'my-index-000001',
  body: {
    runtime: {
      client_ip: {
        type: 'ip',
        script: {
          source: 'String m = doc["message"].value; int end = m.indexOf(" "); emit(m.substring(0, end));'
        }
      }
    }
  }
)
puts response
const response = await client.indices.putMapping({
  index: "my-index-000001",
  runtime: {
    client_ip: {
      type: "ip",
      script: {
        source:
          'String m = doc["message"].value; int end = m.indexOf(" "); emit(m.substring(0, end));',
      },
    },
  },
});
console.log(response);
PUT /my-index-000001/_mapping
{
  "runtime": {
    "client_ip": {
      "type": "ip",
      "script" : {
      "source" : "String m = doc[\"message\"].value; int end = m.indexOf(\" \"); emit(m.substring(0, end));"
      }
    }
  }
}

运行另一个查询，但使用 client_ip 运行时字段搜索特定的 IP 地址。

resp = client.search(
    index="my-index-000001",
    size=1,
    query={
        "match": {
            "client_ip": "211.11.9.0"
        }
    },
    fields=[
        "*"
    ],
)
print(resp)
const response = await client.search({
  index: "my-index-000001",
  size: 1,
  query: {
    match: {
      client_ip: "211.11.9.0",
    },
  },
  fields: ["*"],
});
console.log(response);
GET my-index-000001/_search
{
  "size": 1,
  "query": {
    "match": {
      "client_ip": "211.11.9.0"
    }
  },
  "fields" : ["*"]
}

这次，响应只包含两个命中。 day_of_week (Sunday) 的值是在查询时使用映射中定义的运行时脚本计算的，结果仅包含与 211.11.9.0 IP 地址匹配的文档。

{
  ...
  "hits" : {
    "total" : {
      "value" : 2,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "my-index-000001",
        "_id" : "oWs5KXYB-XyJbifr9mrz",
        "_score" : 1.0,
        "_source" : {
          "@timestamp" : "2020-06-21T15:00:01-05:00",
          "message" : "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
        },
        "fields" : {
          "@timestamp" : [
            "2020-06-21T20:00:01.000Z"
          ],
          "client_ip" : [
            "211.11.9.0"
          ],
          "message" : [
            "211.11.9.0 - - [2020-06-21T15:00:01-05:00] \"GET /english/index.html HTTP/1.0\" 304 0"
          ],
          "day_of_week" : [
            "Sunday"
          ]
        }
      }
    ]
  }
}

从相关索引检索字段

编辑

在 _search API 上的 fields 参数也可用于通过类型为 lookup 的运行时字段从相关索引检索字段。

通过类型为 lookup 的运行时字段检索的字段可用于丰富搜索响应中的命中结果。无法对这些字段进行查询或聚合。

resp = client.index(
    index="ip_location",
    refresh=True,
    document={
        "ip": "192.168.1.1",
        "country": "Canada",
        "city": "Montreal"
    },
)
print(resp)

resp1 = client.index(
    index="logs",
    id="1",
    refresh=True,
    document={
        "host": "192.168.1.1",
        "message": "the first message"
    },
)
print(resp1)

resp2 = client.index(
    index="logs",
    id="2",
    refresh=True,
    document={
        "host": "192.168.1.2",
        "message": "the second message"
    },
)
print(resp2)

resp3 = client.search(
    index="logs",
    runtime_mappings={
        "location": {
            "type": "lookup",
            "target_index": "ip_location",
            "input_field": "host",
            "target_field": "ip",
            "fetch_fields": [
                "country",
                "city"
            ]
        }
    },
    fields=[
        "host",
        "message",
        "location"
    ],
    source=False,
)
print(resp3)
response = client.index(
  index: 'ip_location',
  refresh: true,
  body: {
    ip: '192.168.1.1',
    country: 'Canada',
    city: 'Montreal'
  }
)
puts response

response = client.index(
  index: 'logs',
  id: 1,
  refresh: true,
  body: {
    host: '192.168.1.1',
    message: 'the first message'
  }
)
puts response

response = client.index(
  index: 'logs',
  id: 2,
  refresh: true,
  body: {
    host: '192.168.1.2',
    message: 'the second message'
  }
)
puts response

response = client.search(
  index: 'logs',
  body: {
    runtime_mappings: {
      location: {
        type: 'lookup',
        target_index: 'ip_location',
        input_field: 'host',
        target_field: 'ip',
        fetch_fields: [
          'country',
          'city'
        ]
      }
    },
    fields: [
      'host',
      'message',
      'location'
    ],
    _source: false
  }
)
puts response
const response = await client.index({
  index: "ip_location",
  refresh: "true",
  document: {
    ip: "192.168.1.1",
    country: "Canada",
    city: "Montreal",
  },
});
console.log(response);

const response1 = await client.index({
  index: "logs",
  id: 1,
  refresh: "true",
  document: {
    host: "192.168.1.1",
    message: "the first message",
  },
});
console.log(response1);

const response2 = await client.index({
  index: "logs",
  id: 2,
  refresh: "true",
  document: {
    host: "192.168.1.2",
    message: "the second message",
  },
});
console.log(response2);

const response3 = await client.search({
  index: "logs",
  runtime_mappings: {
    location: {
      type: "lookup",
      target_index: "ip_location",
      input_field: "host",
      target_field: "ip",
      fetch_fields: ["country", "city"],
    },
  },
  fields: ["host", "message", "location"],
  _source: false,
});
console.log(response3);
POST ip_location/_doc?refresh
{
  "ip": "192.168.1.1",
  "country": "Canada",
  "city": "Montreal"
}

PUT logs/_doc/1?refresh
{
  "host": "192.168.1.1",
  "message": "the first message"
}

PUT logs/_doc/2?refresh
{
  "host": "192.168.1.2",
  "message": "the second message"
}

POST logs/_search
{
  "runtime_mappings": {
    "location": {
        "type": "lookup", 
        "target_index": "ip_location", 
        "input_field": "host", 
        "target_field": "ip", 
        "fetch_fields": ["country", "city"] 
    }
  },
  "fields": [
    "host",
    "message",
    "location"
  ],
  "_source": false
}

在主搜索请求中定义一个类型为 lookup 的运行时字段，该字段使用 term 查询从目标索引检索字段。

执行查找查询的目标索引

主索引上的字段，其值用作查找术语查询的输入值

查找索引上的字段，查找查询针对该字段进行搜索

要从查找索引检索的字段列表。请参阅搜索请求的 fields 参数。

上述搜索将从 ip_location 索引返回每个返回搜索命中的 IP 地址的国家和城市。

{
  "took": 3,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": {
      "value": 2,
      "relation": "eq"
    },
    "max_score": 1.0,
    "hits": [
      {
        "_index": "logs",
        "_id": "1",
        "_score": 1.0,
        "fields": {
          "host": [ "192.168.1.1" ],
          "location": [
            {
              "city": [ "Montreal" ],
              "country": [ "Canada" ]
            }
          ],
          "message": [ "the first message" ]
        }
      },
      {
        "_index": "logs",
        "_id": "2",
        "_score": 1.0,
        "fields": {
          "host": [ "192.168.1.2" ],
          "message": [ "the second message" ]
        }
      }
    ]
  }
}

查找字段的响应被分组以保持每个文档与查找索引的独立性。每个输入值的查找查询最多应匹配查找索引上的一个文档。如果查找查询匹配多个文档，则将随机选择一个文档。

« 查询时覆盖字段值 索引运行时字段 »