Created
November 14, 2024 03:40
process Kibana logs with tag `query-result-out-of-time-range`
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env node | |
'use strict' | |
// Processes Kibana logs queried from DevTools in an overview cluster, | |
// finding messages tagged with `query-result-out-of-time-range`. | |
// The output of the query should be saved to a file, which is then | |
// passed as an argument to this script. The query is in a comment | |
// at the bottom of this file | |
const fs = require('fs') | |
const [ fileName ] = process.argv.slice(2) | |
if (fileName == null) { | |
console.error('input file with search response required') | |
process.exit(0) | |
} | |
const contentsJSON = fs.readFileSync(fileName, 'utf-8') | |
const contents = JSON.parse(contentsJSON) | |
const linePattern = /.*?'(.*?)'.*?'(.*?)'.*?'(.*?)'.*Query: \<(.*?)\>. Document: \<(.*)\>/ | |
for (const hit of contents.hits.hits) { | |
const message = hit.fields.message[0] | |
const tstamp = hit.fields['@timestamp'][0] | |
const [_, ruleId, srcTime, field, queryJSON, docJSON] = linePattern.exec(message) || [] | |
if (!ruleId || !srcTime || !field || !queryJSON || !docJSON) { | |
console.log(`unable to parse hit ${JSON.stringify(hit, null, 4)}:`) | |
continue | |
} | |
// note that `field` is supposed to be the time field used in the query, | |
// however for searchsource queries (KQL) it may not be correct; the | |
// time field is in the dataview, and I've seen cases where the `timeField` | |
// in the rule is different than what's used in the query, which is | |
// presumably coming from the dataview. The code in the searchsource | |
// flavor of the query ignores the rule's `timeField`, so the query should | |
// work right, we just don't know 100% what the real timeField is. Probably | |
// we should pull it from the sort? | |
const query = JSON.parse(queryJSON) | |
const doc = JSON.parse(docJSON) | |
const { queryBeg, queryEnd, docuDate, exclDate } = | |
// data can be in different places, so try them all! | |
getDateInfo_1(field, doc, query) || | |
getDateInfo_2(field, doc, query) || | |
{} | |
if (!queryBeg || !queryEnd || !docuDate) { | |
console.log(`unable to get date info for hit ${JSON.stringify(hit, null, 4)}`) | |
console.log(`query: ${JSON.stringify(query, null, 4)}`) | |
console.log(`doc: ${JSON.stringify(doc, null, 4)}`) | |
continue | |
} | |
if (docuDate > queryBeg || docuDate > queryEnd) { | |
console.log(`found one: ${ | |
JSON.stringify({queryBeg, docuDate, queryEnd, tstamp, ruleId, srcTime, field, exclDate}) | |
}`) | |
} | |
} | |
/** @typedef { {queryBeg: string, queryEnd: string, docuDate: string, exclDate?: string} } DateInfo */ | |
/** @type { (field: string, doc: any, query: any) => DateInfo | undefined } */ | |
function getDateInfo_1(field, doc, query) { | |
try { | |
const body = query.body ? query.body : query | |
const bool = body.query.bool | |
const range = bool.filter[0].range || bool.filter[1].range | |
const queryBeg = range[field].gte | |
const queryEnd = range[field].lte | |
const docuDate = doc.fields[field][0] | |
return getDateInfo(queryBeg, queryEnd, docuDate) | |
} catch (e) {} | |
} | |
/** @type { (field: string, doc: any, query: any) => DateInfo | undefined } */ | |
function getDateInfo_2(field, doc, query) { | |
try { | |
const body = query.body ? query.body : query | |
const bool = body.query.bool | |
const range = bool.filter[0].bool.filter[0].range || bool.filter[1].bool.filter[0].range | |
const queryBeg = range[field].gte | |
const queryEnd = range[field].lte | |
const docuDate = doc.fields[field][0] | |
return getDateInfo(queryBeg, queryEnd, docuDate) | |
} catch (e) {} | |
} | |
/** @type { (queryBeg: string, queryEnd: string, docuDate: string, exclDate?: string) => DateInfo } */ | |
function getDateInfo(queryBeg, queryEnd, docuDate, exclDate) { | |
return { | |
queryBeg, | |
queryEnd, | |
docuDate, | |
exclDate | |
} | |
} | |
/* Dev Tools query: | |
POST /logging-*%3Acluster-kibana-%2A/_search | |
{ | |
"size": 100, | |
"sort": [ | |
{ | |
"@timestamp": { | |
"order": "desc", | |
"format": "strict_date_optional_time", | |
"unmapped_type": "boolean" | |
} | |
} | |
], | |
"track_total_hits": false, | |
"fields": [ | |
{ | |
"field": "@timestamp", | |
"format": "strict_date_optional_time" | |
}, | |
{ | |
"field": "message" | |
} | |
], | |
"_source": false, | |
"query": { | |
"bool": { | |
"filter": { | |
"match_phrase": { | |
"tags": "query-result-out-of-time-range" | |
} | |
} | |
} | |
} | |
} | |
*/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment