Rules (XQL)
[RULE: Log_Sample_v1]
alter
tmp_unite_date_time = concat(Date, " ", Time)
| alter
_time = parse_timestamp("%Y-%m-%d %k:%M:%S", tmp_unite_date_time)
| fields -tmp_unite_date_time;
[RULE: Log_Sample_v2]
alter
tmp_date = arrayindex(regextract(_raw_log, "date\=(\d{4}\-\d{2}\-\d{2})"),0),
tmp_time = arrayindex(regextract(_raw_log ,"time\=(\d{2}\:\d{2}\:\d{2}\.\d+)"),0)
| alter
tmp_time_date = concat(tmp_date ,"T",tmp_time)
| filter
tmp_time_date ~= "\d{4}\-\d{2}\-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d+"
| alter
_time = parse_timestamp("%Y-%m-%dT%H:%M:%E3S", tmp_time_date)
| alter
tmp_keys = if(_raw_log ~= "subtype\=",arrayconcat(arraycreate("date","time","device_id","log_id","type","subtype","priority"),regextract(_raw_log, "([A-Za-z_]+?)=\"")),arrayconcat(arraycreate("date","time","device_id","log_id","type","priority"),regextract(_raw_log, "([A-Za-z_]+?)=\""))),
tmp_values = if(_raw_log ~= "subtype\=",arraystring(arrayconcat(regextract(_raw_log, "date\=(\d{4}\-\d{2}\-\d{2})"),regextract(_raw_log, "time\=(\d{2}\:\d{2}\:\d{2}\.\d+)"), regextract(_raw_log, "device\_id\=([^\s]+)"),regextract(_raw_log , "log\_id\=(\d+)"),regextract(_raw_log,"\stype\=([^\s]+)"),regextract(_raw_log,"\ssubtype\=([^\s]+)"),regextract(_raw_log,"pri\=([^\s]+)"),regextract(_raw_log, "[a-z_]+\=\"(.*?\")")), "|"),arraystring(arrayconcat(regextract(_raw_log, "date\=(\d{4}\-\d{2}\-\d{2})"),regextract(_raw_log, "time\=(\d{2}\:\d{2}\:\d{2}\.\d+)"), regextract(_raw_log, "device\_id\=([^\s]+)"),regextract(_raw_log , "log\_id\=(\d+)"),regextract(_raw_log,"\stype\=([^\s]+)"),regextract(_raw_log,"pri\=([^\s]+)"),regextract(_raw_log, "[a-z_]+\=\"(.*?\")")), "|"))
| alter
tmp_values = replex(tmp_values ,"\|\"","|null")
| alter
tmp_values = replex(tmp_values ,"\"","")
| alter
tmp_obj_struct = arraystring(arraymap(tmp_keys, concat("(?P<", "@element", ">[^\|]+)")), "\|")
| alter
tmp_obj_struct = replex(tmp_obj_struct ,"\-","_")
| alter
parsed_fields = regexcapture(tmp_values, tmp_obj_struct)
| fields -tmp*;
[INGEST:vendor="fortinet", product="fortimail", target_dataset="fortinet_fortimail_raw", no_hit=drop]
filter len(Message) > 0
| call Log_Sample_v1;
filter _raw_log ~= "\w+\=\d{4}\-\d{2}\-\d{2}\s\w+\=\d{2}\:\d{2}\:\d{2}\.\d+\s\w+\=[A-Za-z0-9]+"
| call Log_Sample_v2;
filter _raw_log !~= "\w+\=\d{4}\-\d{2}\-\d{2}\s\w+\=\d{2}\:\d{2}\:\d{2}\.\d+\s\w+\=[A-Za-z0-9]+" and (len(Message) = 0 or Message = null)
| alter _time = _insert_time;