This transformation function is used to parse unstructured log data into a structured and queryable format.
The SYNTAX represents the name of the pattern that matches a portion of the text. For example, 3.44 can be matched by the NUMBER pattern, and 55.3.244.1 can be matched by the IP pattern. The SYNTAX defines how the text is matched.
The SEMANTIC is the identifier assigned to the matched text. For example, 3.44 could represent the duration of an event and may be labeled as duration. Similarly, 55.3.244.1 could represent the client IP address and may be labeled as client.
Based on the above examples, the corresponding grok filter would look something like this:
You can also use Oniguruma syntax for named capture, which allows you to match a portion of text and store it as a field.
For example, Postfix logs have a queue ID that is a 10- or 11-character hexadecimal value. You can capture it using a named capture group like this:
| PRE-BUILTIN DICTIONARY | EXPRESSION |
|---|
| BASE10NUM | (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) |
| BASE16FLOAT | \b(? |
| BASE16NUM | (? |
| CISCOMAC | (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) |
| COMBINEDAPACHELOG | %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} |
| COMMONAPACHELOG | %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) |
| COMMONMAC | (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) |
| CRONLOG | %{SYSLOGBASE} \(%{USER:user}\) %{CRON_ACTION:action} \(%{DATA:message}\) |
| CRON_ACTION | [A-Z ]+ |
| DATA | .*? |
| DATE | %{DATE_US}|%{DATE_EU} |
| DATESTAMP | %{DATE}[- ]%{TIME} |
| DATESTAMP_EVENTLOG | %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} |
| DATESTAMP_OTHER | %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} |
| DATESTAMP_RFC2822 | %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} |
| DATESTAMP_RFC822 | %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} |
| DATE_EU | %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} |
| DATE_US | %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} |
| DAY | (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) |
| DOMAINTLD | [a-zA-Z]+ |
| EMAIL | %{NOTSPACE}@%{WORD}\.%{DOMAINTLD} |
| GREEDYDATA | .* |
| HAPROXYCAPTUREDREQUESTHEADERS | %{DATA:captured_request_headers} |
| HAPROXYCAPTUREDRESPONSEHEADERS | %{DATA:captured_response_headers} |
| HAPROXYDATE | %{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds} |
| HAPROXYHTTP | (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE} |
| HAPROXYHTTPBASE | %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?" |
| HAPROXYTCP | (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} |
| HAPROXYTIME | (?!<[0-9])%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second})(?![0-9]) |
| HOST | %{HOSTNAME} |
| HOSTNAME | \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b) |
| HOSTPORT | %{IPORHOST}:%{POSINT} |
| HOUR | (?:2[0123]|[01]?[0-9]) |
| HTTPDATE | %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} |
| INT | (?:[+-]?(?:[0-9]+)) |
| IP | (?:%{IPV6}|%{IPV4}) |
| IPORHOST | (?:%{HOSTNAME}|%{IP}) |
| IPV4 | (? |
| IPV6 | ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? |
| ISO8601_SECOND | (?:%{SECOND}|60) |
| ISO8601_TIMEZONE | (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) |
| JAVACLASS | (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$_]+ |
| JAVAFILE | (?:[A-Za-z0-9_. -]+) |
| JAVASTACKTRACEPART | at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\) |
| LOGLEVEL | ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) |
| MAC | (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) |
| MCOLLECTIVE | ., \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\]%{SPACE}%{LOGLEVEL:event_level} |
| MCOLLECTIVEAUDIT | %{TIMESTAMP_ISO8601:timestamp}: |
| MINUTE | (?:[0-5][0-9]) |
| MONGO_LOG | %{SYSLOGTIMESTAMP:timestamp} \[%{WORD:component}\] %{GREEDYDATA:message} |
| MONGO_QUERY | \{ (?<={ ).*(?= } ntoreturn:) \} |
| MONGO_SLOWQUERY | %{WORD} %{MONGO_WORDDASH:database}\.%{MONGO_WORDDASH:collection} %{WORD}: %{MONGO_QUERY:query} %{WORD}:%{NONNEGINT:ntoreturn} %{WORD}:%{NONNEGINT:ntoskip} %{WORD}:%{NONNEGINT:nscanned}.*nreturned:%{NONNEGINT:nreturned}..+ (?[0-9]+)ms |
| MONGO_WORDDASH | \b[\w-]+\b |
| MONTH | \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b |
| MONTHDAY | (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) |
| MONTHNUM | (?:0?[1-9]|1[0-2]) |
| MONTHNUM2 | (?:0[1-9]|1[0-2]) |
| NONNEGINT | \b(?:[0-9]+)\b |
| NOTSPACE | \S+ |
| NUMBER | (?:%{BASE10NUM}) |
| PATH | (?:%{UNIXPATH}|%{WINPATH}) |
| POSINT | \b(?:[1-9][0-9]*)\b |
| POSTGRESQL | %{DATESTAMP:timestamp} %{TZ} %{DATA:user_id} %{GREEDYDATA:connection_id} %{POSINT:pid} |
| PROG | (?:[\w._/%-]+) |
| QS | %{QUOTEDSTRING} |
| QUOTEDSTRING | (?>(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) |
| REDISLOG | \[%{POSINT:pid}\] %{REDISTIMESTAMP:timestamp} \* |
| REDISTIMESTAMP | %{MONTHDAY} %{MONTH} %{TIME} |
| RT_FLOW1 | %{RT_FLOW_EVENT:event}: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} \d+\(%{DATA:sent}\) \d+\(%{DATA:received}\) %{INT:elapsed-time} .* |
| RT_FLOW2 | %{RT_FLOW_EVENT:event}: session created %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .* |
| RT_FLOW3 | %{RT_FLOW_EVENT:event}: session denied %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{INT:protocol-id}\(\d\) %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .* |
| RT_FLOW_EVENT | (RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY) |
| RUBY_LOGGER | [DFEWI], \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\] *%{RUBY_LOGLEVEL:loglevel} -- +%{DATA:progname}: %{GREEDYDATA:message} |
| RUBY_LOGLEVEL | (?:DEBUG|FATAL|ERROR|WARN|INFO) |
| SECOND | (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) |
| SPACE | \s* |
| SYSLOG5424BASE | %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(?:%{WORD:syslog5424_app}|-) +(?:%{WORD:syslog5424_proc}|-) +(?:%{WORD:syslog5424_msgid}|-) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|) |
| SYSLOG5424LINE | %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg} |
| SYSLOG5424PRI | <%{NONNEGINT:syslog5424_pri}> |
| SYSLOG5424SD | \[%{DATA}\]+ |
| SYSLOGBASE | %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: |
| SYSLOGBASE2 | (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: |
| SYSLOGFACILITY | <%{NONNEGINT:facility}.%{NONNEGINT:priority}> |
| SYSLOGHOST | %{IPORHOST} |
| SYSLOGLINE | %{SYSLOGBASE2} %{GREEDYDATA:message} |
| SYSLOGPAMSESSION | %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}\(%{DATA:pam_caller}\): session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})? |
| SYSLOGPROG | %{PROG:program}(?:\[%{POSINT:pid}\])? |
| SYSLOGTIMESTAMP | %{MONTH} +%{MONTHDAY} %{TIME} |
| TIME | (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) |
| TIMESTAMP_ISO8601 | %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? |
| TTY | (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+)) |
| TZ | (?:[PMCE][SD]T|UTC) |
| UNIXPATH | (?>/(?>[\w_%!$@:.,-]+|\\.)*)+ |
| URI | %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? |
| URIHOST | %{IPORHOST}(?::%{POSINT:port})? |
| URIPARAM | \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* |
| URIPATH | (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ |
| URIPATHPARAM | %{URIPATH}(?:%{URIPARAM})? |
| URIPROTO | [A-Za-z]+(\+[A-Za-z+]+)? |
| USER | %{USERNAME} |
| USERNAME | [a-zA-Z0-9._-]+ |
| UUID | [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} |
| WINDOWSMAC | (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) |
| WINPATH | (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ |
| WORD | \b\w+\b |
| YEAR | (?>\d\d){1,2} |
