[MODEL: dataset = "aws_elb_raw"] alter // initialize util constants field_separator = "##", // an arbitrary string that would be used as a temporary field separator to construct an ordinal list of all fields. This string must be different from any valid consecutive string that may appear on the original raw log. NIL = "-" // the character that represents a blank field // extract log fields which are separated by double quotes e.g., "user agent" | alter quoted_fields = regextract(_raw_log, "\"(\S[^\"]+)") // split the raw log into segments, using quotes as a segment delimiter | alter log_segments = split(_raw_log, "\"") // clean white spaces from the extracted segments | alter log_segments_cleaned = arrayfilter(arraymap(log_segments, trim("@element")), len("@element") > 0) // insert a delimiter between the different fields within each segment | alter field_delimited_log_segments = arraymap(log_segments_cleaned, // if the segment is originally a quoted string, leave it as is: if(quoted_fields contains trim("@element"), "@element", // edge case where the field is a comma-separated list (e.g. cs-auth-groups) of values and the original comparison fails "@element" ~= ",\s", "@element", // else, add the defined separator between each field in the segment (separated originally by a space) arraystring(regextract("@element", "\S+"), field_separator))) // add the field separator between the segments and flatten them to a unified string | alter flattened_field_list = arraystring(field_delimited_log_segments, field_separator) // split the delimited string of fields into an index-based array | alter field_list = split(flattened_field_list, field_separator) | alter client_ipv4 = regextract(arrayindex(field_list,3),"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"), client_ipv6 = regextract(arrayindex(field_list,3),"([a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5})\:"), client_port = regextract(arrayindex(field_list,3),"\:(\d+)"), target_ipv4 = regextract(arrayindex(field_list,4),"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"), target_ipv6 = regextract(arrayindex(field_list,4),"([a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5}[a-fA-F0-9\:]{1,5})\:"), req_port = regextract(arrayindex(field_list,4),"\:(\d+)"), req_method = regextract(arrayindex(field_list,12),"(\w+)\s"), req_url = regextract(arrayindex(field_list,12),"\w+\s(.+)\:"), target_port = regextract(arrayindex(field_list,12),"\:(\d+)"), req_uri = regextract(arrayindex(field_list,12),"\:\d+(.+)\s"), req_method_version = regextract(arrayindex(field_list,12),"\:\d+.+\s(.+)"), req_process_time = if(arrayindex(field_list,5)="-1","0",arrayindex(field_list,5)), target_process_time = if(arrayindex(field_list,6)="-1","0",arrayindex(field_list,6)), response_process_time = if(arrayindex(field_list,7)="-1","0",arrayindex(field_list,7)), resource_zone = arrayindex(regextract(arrayindex(field_list,16),"\w+\:\w+\:\w+\:(.+?)\:"),0) //calculation of the duration time, this is separated due to the limitation of ADD function | alter partial_duration_time = add(to_float(req_process_time), to_float(target_process_time)) | alter xdm.event.description = _raw_log, xdm.intermediate.cloud.provider = XDM_CONST.CLOUD_PROVIDER_AWS, xdm.network.application_protocol = arrayindex(field_list,0), xdm.intermediate.host.device_id = arrayindex(field_list,2), xdm.source.ipv4 = arrayindex(client_ipv4,0), xdm.source.ipv6 = arrayindex(client_ipv6,0), xdm.source.port = to_number(arrayindex(client_port,0)), xdm.target.ipv4 = arrayindex(target_ipv4,0), xdm.target.ipv6 = arrayindex(target_ipv6,0), xdm.intermediate.port = to_number(arrayindex(req_port,0)), xdm.target.port = to_number(arrayindex(target_port,0)), xdm.event.duration = to_integer(add(to_float(partial_duration_time), to_float(response_process_time))), xdm.network.http.response_code = if(arrayindex(field_list,9)=NIL, null, arrayindex(field_list,9)), xdm.source.sent_bytes = to_number(arrayindex(field_list,11)), xdm.network.http.url = concat(arrayindex(req_url,0),arrayindex(req_uri,0)), xdm.network.http.method = arrayindex(req_method,0), xdm.network.application_protocol_category = arrayindex(req_method_version,0), xdm.source.host.device_model = if(arrayindex(field_list,13)=NIL,null,arrayindex(field_list,13)), xdm.network.tls.cipher = if(arrayindex(field_list,14) = NIL, null, arrayindex(field_list,14)), xdm.network.tls.protocol_version = if(arrayindex(field_list,15)=NIL, null, arrayindex(field_list,15)), xdm.target.resource.name = if(arrayindex(field_list,16) = NIL, null,arrayindex(regextract(arrayindex(field_list,16),"\w+\:\w+\:\w+\:(.+)"),0)), xdm.target.resource.type = if(arrayindex(field_list,16) = NIL, null,arrayindex(regextract(arrayindex(field_list,16),"\w+\:\w+\:\w+"),0)), xdm.target.cloud.region = if(arrayindex(field_list,16) = NIL, null,resource_zone), xdm.network.http.http_header.value = arrayindex(field_list,17), xdm.network.http.http_header.header = if(arrayindex(field_list,17) != null,"X-Amzn-Trace_Id"), xdm.network.tls.server_name = if(arrayindex(field_list,18)= NIL, null, arrayindex(field_list,18)), xdm.network.tls.server_certificate.subject = if(arrayindex(field_list,19)=NIL, null, arrayindex(field_list,19)), xdm.network.tls.server_certificate.issuer = if(arrayindex(field_list,19)!= NIL, arrayindex(regextract(arrayindex(field_list,19),"\w+\:\w+\:\w+"),0)), xdm.network.rule = arrayindex(field_list,20), xdm.observer.action = if(arrayindex(field_list,22) = NIL,null, arrayindex(field_list,22)), xdm.target.url = if(arrayindex(field_list,23) = NIL,null,arrayindex(field_list,23)), xdm.event.outcome_reason = if(arrayindex(field_list,24) = NIL,null,arrayindex(field_list,24)), xdm.event.outcome = if(arrayindex(field_list,24) = NIL,XDM_CONST.OUTCOME_SUCCESS, XDM_CONST.OUTCOME_FAILED), xdm.alert.severity = if(arrayindex(field_list,27) = NIL,null, arrayindex(field_list,27)), xdm.alert.name = if(arrayindex(field_list,28)=NIL, null,arrayindex(field_list,28));