Summary: I would like to parse the JSON output of tshark as it is outputted.
As of now I was parsing normal output, line by line, and each line had the complete information. It was therefore a matter of
p = subprocess.Popen["/usr/bin/tshark", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True]
for line in p.stdout:
event = decode_event[line]
tshark can also output pretty-printed JSON via the -T json switch [I just give the first packet, the output is a list]:
[
{
"_index": "packets-2018-03-08",
"_type": "pcap_file",
"_score": null,
"_source": {
"layers": {
"frame": {
"frame.interface_id": "0",
"frame.encap_type": "1",
"frame.time": "Mar 8, 2018 16:17:20.478658037 CET",
"frame.offset_shift": "0.000000000",
"frame.time_epoch": "1520522240.478658037",
"frame.time_delta": "0.000113952",
"frame.time_delta_displayed": "0.000113952",
"frame.time_relative": "3.351515496",
"frame.number": "11133",
"frame.len": "60",
"frame.cap_len": "60",
"frame.marked": "0",
"frame.ignored": "0",
"frame.protocols": "eth:ethertype:ip:tcp"
},
"eth": {
"eth.dst": "00:50:56:bb:40:70",
"eth.dst_tree": {
"eth.dst_resolved": "Vmware_bb:40:70",
"eth.addr": "00:50:56:bb:40:70",
"eth.addr_resolved": "Vmware_bb:40:70",
"eth.lg": "0",
"eth.ig": "0"
},
"eth.src": "64:a0:e7:42:af:41",
"eth.src_tree": {
"eth.src_resolved": "Cisco_42:af:41",
"eth.addr": "64:a0:e7:42:af:41",
"eth.addr_resolved": "Cisco_42:af:41",
"eth.lg": "0",
"eth.ig": "0"
},
"eth.type": "0x00000800",
"eth.padding": "00:00:00:00:00:00"
},
"ip": {
"ip.version": "4",
"ip.hdr_len": "20",
"ip.dsfield": "0x00000000",
"ip.dsfield_tree": {
"ip.dsfield.dscp": "0",
"ip.dsfield.ecn": "0"
},
"ip.len": "40",
"ip.id": "0x00005a57",
"ip.flags": "0x00000002",
"ip.flags_tree": {
"ip.flags.rb": "0",
"ip.flags.df": "1",
"ip.flags.mf": "0"
},
"ip.frag_offset": "0",
"ip.ttl": "125",
"ip.proto": "6",
"ip.checksum": "0x0000dd25",
"ip.checksum.status": "2",
"ip.src": "10.237.78.2",
"ip.addr": "10.237.78.2",
"ip.src_host": "10.237.78.2",
"ip.host": "10.237.78.2",
"ip.dst": "10.81.99.19",
"ip.addr": "10.81.99.19",
"ip.dst_host": "10.81.99.19",
"ip.host": "10.81.99.19",
"Source GeoIP: Unknown": "",
"Destination GeoIP: Unknown": ""
},
"tcp": {
"tcp.srcport": "31316",
"tcp.dstport": "22",
"tcp.port": "31316",
"tcp.port": "22",
"tcp.stream": "0",
"tcp.len": "0",
"tcp.seq": "3025",
"tcp.ack": "774293",
"tcp.hdr_len": "20",
"tcp.flags": "0x00000010",
"tcp.flags_tree": {
"tcp.flags.res": "0",
"tcp.flags.ns": "0",
"tcp.flags.cwr": "0",
"tcp.flags.ecn": "0",
"tcp.flags.urg": "0",
"tcp.flags.ack": "1",
"tcp.flags.push": "0",
"tcp.flags.reset": "0",
"tcp.flags.syn": "0",
"tcp.flags.fin": "0",
"tcp.flags.str": "\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7A\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7\u00c2\u00b7"
},
"tcp.window_size_value": "2047",
"tcp.window_size": "2047",
"tcp.window_size_scalefactor": "-1",
"tcp.checksum": "0x000073f4",
"tcp.checksum.status": "2",
"tcp.urgent_pointer": "0",
"tcp.analysis": {
"tcp.analysis.acks_frame": "11126",
"tcp.analysis.ack_rtt": "0.000426928"
}
}
}
}
},
What would be the correct approach to parse such a stream?
When searching for stream
parsing, I found a few libraries [notably NAYA], but they require a file like object.
It would seem that StringIO[] would be appropriate but I do not know how to connect it with stdout?
Per @omu_negru request, specifically in case of NAYA,
directly attaching stdout as in
import naya
import subprocess
def handle_message[event]:
print[event]
cmd = "/usr/bin/tshark -i eth0 -T json"
proc = subprocess.Popen[cmd, bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True]
messages = naya.stream_array[proc.stdout]
for message in messages:
handle_message[message]
raises an exception
Traceback [most recent call last]:
File "/root/dev/readtshark.py", line 12, in
for message in messages:
File "/usr/local/lib/python3.5/dist-packages/naya/json.py", line 544, in stream_array
token_type, token = next[token_stream]
ValueError: too many values to unpack [expected 2]