Class 21: Python - parsing binary data - SBET IMU data

Thanks to Glen Rice for this topic and sample data.

http://vislab-ccom.unh.edu/~schwehr/Classes/2011/esci895-researchtools/python-binary-files.org

mkdir -p ~/class/21
cd ~/class/21

pwd
# Make sure you are in the right location

curl -O http://vislab-ccom.unh.edu/~schwehr/Classes/2011/esci895-researchtools/examples/21/sample.sbet.bz2
bunzip2 sample.sbet.bz2

md5sum sample.sbet 
196c21f16f07ceae180888b12e9edc56  sample.sbet

Applanix provides documentation for the POSPac SBET files. It is essential to look at the documentation (if it is available) before starting to parse the data. The documentation might not be perfect, but it can save you tons of time (and likely frustration).

Based on Table 4 of PosPac Quick Start Guide.pdf in 3.0 POSPac Land Output Data Files Post-Processed Solution File:

Data	Units	Type
time	seconds	double
latitude	radians	double
longitude	radians	double
altitude	meters	double
x velocity	meters/second	double
y velocity	meters/second	double
z velocity	meters/second	double
roll	radians	double
pitch	radians	double
platform heading	radians	double
wander angle	radians	double
x body acceleration	meters/second2	double
y body acceleration	meters/second2	double
z body acceleration	meters/second2	double
x body angular rate	radians/second	double
y body angular rate	radians/second	double
z body angular rate	radians/second	double

ls -l *.sbet
-rw-r--r--  1 schwehr  staff      22712 Dec 14 12:44 sample.sbet

-rw-r--r--  1 schwehr  staff  225820248 Dec 12 09:02 original.sbet

file *.sbet
original.sbet: data
sample.sbet:   data

less sample.sbet

od -a sample.sbet  | head

0000000    ##  g   ####t   #   q dc4   A   ### H soh   # nak   ### ####?
0000020    G   ####f   Q   ##  z eot   ##  n   ### ##  ##dle   #   )   @
0000040    K   ##  #  9a   *   ### $   @   R   #  90  83   |   ####### ?
0000060    ###yn   ##  ##  ### f   ##  ? esc   u   # etx bel  88   e   #
0000100    #  vt   #   '   ######  #   ?   #   ### #   2   8  90   #   #
0000120    z   ##  = can   #####   ##  #   ### R   ### $   #   e   ### #
0000140    #####   ### #   #   ##  ##  # stx   <   w   u  83   #####   ?
0000160    ### B   #####   ###### 95   ?   m   #  83   4   /   #  9d   ?
0000200   si   { etb   D   ##  ### {   #   ###ff  99 etx  84   r dc4   A
0000220  dc3  95   #   ### e   ### ####?   #######c3 stx   #   z eot   ##

Better yet, Octal Dump has a mode that will try to treat the file as uniform binary data (for example, a series of 4 byte integers). Since we know that our SBET file will contain a series of 17 doubles (8 bytes each) in a row, let's try out a sample file that contains the numbers 0 through 16,

wget http://vislab-ccom.unh.edu/~schwehr/Classes/2011/esci895-researchtools/examples/21/s1.bin

od -t fD s1.bin 

0000000     0.000000000000000e+00    1.000000000000000e+00
0000020     2.000000000000000e+00    3.000000000000000e+00
0000040     4.000000000000000e+00    5.000000000000000e+00
0000060     6.000000000000000e+00    7.000000000000000e+00
0000100     8.000000000000000e+00    9.000000000000000e+00
0000120     1.000000000000000e+01    1.100000000000000e+01
0000140     1.200000000000000e+01    1.300000000000000e+01
0000160     1.400000000000000e+01    1.500000000000000e+01
0000200     1.600000000000000e+01                         
0000210

ipython

Setup:

logstart -o -r log-class-21.py

import struct
import numpy

http://docs.python.org/library/struct.html#format-characters

Run this section to create some sample files:

import struct
open('c-65.bin','w').write(struct.pack('c','z'))
open('b-120.bin','w').write(struct.pack('b',120 ))
open('B-121.bin','w').write(struct.pack('B',121 ))
open('B-series.bin','w').write(struct.pack('10B',*range(115,125) ))
open('i-nine.bin','w').write(struct.pack('i',9))
open('d-pi.bin','w').write(struct.pack('d',math.pi))
open('d-1.bin','w').write(struct.pack('d',1.0))
open('d-series.bin','w').write(struct.pack('10d',*range(10)))

!file *.bin

Open B-series.bin in emacs. Try this emacs command:

M-x hexl-mode

Open another terminal and run this:

man ascii

Or in emacs, M-x man <RET> ascii

sbet_file = open('sample.sbet')

There are many things you can do with a file, but in our case, we want to pull the entire file into a variable.

sbet_data = sbet_file.read()

type(sbet_data)
# Out: <type 'str'>

len(sbet_data)
# 22712

struct.unpack('d',sbet_data[0:8])
# (334959.0048233234,)

struct.unpack('d',sbet_data[0:8])[0]
# 334959.0048233234

struct.unpack('dd',sbet_data[8:24])
(1.0549522638507869, -2.559965741819528)

struct.unpack('2d',sbet_data[8:24])
(1.0549522638507869, -2.559965741819528)

struct.unpack('17d',sbet_data[0:8*17])
Out[38]: 
(334959.0048233234,
 1.0549522638507869,
 -2.559965741819528,
 12.826300557342815,
 10.437825046453915,
 0.998228318178983,
 0.18282804536664027,
 -0.0026283394812042344,
 0.11416603057936824,
 -0.09985686530029529,
 -0.40154673926674145,
 -0.8249097558096672,
 -0.3413483211034812,
 0.07018300645653144,
 0.021320176833628756,
 0.029000032024608147,
 -0.006807197876212325)

time, latitude, longitude, altitude, \
    x_vel, y_vel, z_vel, \
    roll, pitch, platform_heading, wander_angle, \
    x_acceleration, y_acceleration, z_acceleration, \
    x_angular_rate, y_angular_rate, z_angular = struct.unpack('ddddddddddddddddd',data[0:17*8])

field_names = ('time', 'latitude', 'longitude', 'altitude', \
          'x_vel', 'y_vel', 'z_vel', \
          'roll', 'pitch', 'platform_heading', 'wander_angle', \
          'x_acceleration', 'y_acceleration', 'z_acceleration', \
          'x_angular_rate', 'y_angular_rate', 'z_angular')

values = struct.unpack('17d',sbet_data[0:8*17])

# Crazy stuff happens here!
dict(  zip(field_names, values) )

zip(field_names[:4], values[:4])

# Results in:
[('time', 334959.0048233234),
 ('latitude', 1.0549522638507869),
 ('longitude', -2.559965741819528),
 ('altitude', 12.826300557342815)]

sbet_values = dict(zip (field_names, values))

# convert radians to degrees and put it in a new dictionary key
sbet_values['lat_deg'] = math.degrees(sbet_values['latitude'])

sbet_values['lat_deg']
# 60.444312306421736

Open sbet.py and add this:

# Decode Applanix POSPac SBET IMU binary files

def decode():
    print "hello from decode"

in ipython:

import sbet
sbet.decode()
# hello from decode

edit sbet.py:

# Decode Applanix POSPac SBET IMU binary files

# You will see this when you load or reload this file, this line will print
print "load or reload happening"

def decode():
    print "hello from decode"
    print 7*6

in ipython:

reload(sbet)
# load or reload happening

sbet.decode()
# hello from decode
# 42

# Decode Applanix POSPac SBET IMU binary files

# You will see this when you load or reload this file, this line will print
print "load or reload happening"

def decode():
    print "hello from decode"
    print 7*6

def main():
    print 'Starting main'
    sbet_file = open('sample.sbet')
    sbet_data = sbet_file.read()
    print 'Finishing main'

in python:

reload(sbet)
# load or reload happening

In [12]: sbet.main()
# Starting main
# Finishing main

update sbet.py to have a decode with an argument:

# Add data argument to decode
def decode(data):
    'Decipher a SBET datagram from binary'
    print "hello from decode"
    print 'Data length:', len(data)

def main():
    print 'Starting main'
    sbet_file = open('sample.sbet')
    sbet_data = sbet_file.read()
    print 'Read this many bytes:',len(sbet_data)

    decode(sbet_data)  # Pass in the sbet_data variable to decode

    print 'Finishing main'

help(sbet.decode)
# Help on function decode in module sbet:
# 
# decode(data)
#     Decipher a SBET datagram from binary

sbet.decode?
# Type:           function
# Base Class:     <type 'function'>
# String Form:    <function decode at 0x11f09f0>
# Namespace:      Interactive
# File:           /Users/schwehr/Desktop/sbet/sbet.py
# Definition:     sbet.decode(data)
# Docstring:
#     Decipher a SBET datagram from binary

edit decode function:

import math
import struct

def decode(data):
    "Decipher a SBET datagram from binary"
    print "Start decoding datagram"
    values = struct.unpack('17d',data[0:8*17])

    time = values[0]

    latitude = values[1]
    lat_deg = math.degrees(latitude)

    longitude = values[2]
    lon_deg = math.degrees(longitude)

    print 'results:', time, lat_deg, lon_deg

if __name__=='__main__':
    print 'starting to run script...'
    main()
    print 'script done!'

Add this to the top of sbet.py

#!/usr/bin/env python

#!/usr/bin/env python

# Decode Applanix POSPac SBET IMU binary files

import math, struct

def decode(data):
    "Decipher a SBET datagram from binary"
    values = struct.unpack('17d',data[0:8*17])

    time = values[0]

    latitude = values[1]
    lat_deg = math.degrees(latitude)

    longitude = values[2]
    lon_deg = math.degrees(longitude)

    print 'results:', time, lat_deg, lon_deg

def main():
    sbet_file = open('sample.sbet')
    sbet_data = sbet_file.read()

    decode(sbet_data)

if __name__=='__main__':
    main()

Decode needs to return something!

field_names = ('time', 'latitude', 'longitude', 'altitude', \
          'x_vel', 'y_vel', 'z_vel', \
          'roll', 'pitch', 'platform_heading', 'wander_angle', \
          'x_acceleration', 'y_acceleration', 'z_acceleration', \
          'x_angular_rate', 'y_angular_rate', 'z_angular')

def decode(data):
    "Decipher a SBET datagram from binary"
    values = struct.unpack('17d',data[0:8*17])

    # Create a dictionary for all the values
    sbet_values = dict(zip (field_names, values))

    sbet_values['lat_deg'] = math.degrees(sbet_values['latitude'])
    sbet_values['lon_deg'] = math.degrees(sbet_values['longitude'])

    print 'results:'
    for key in sbet_values:
        print '    ', key, sbet_values[key]

def main():
    sbet_file = open('sample.sbet')
    sbet_data = sbet_file.read()

    datagram = decode(sbet_data)
    print datagram

def decode(data):
    "Decipher a SBET datagram from binary"
    values = struct.unpack('17d',data[0:8*17])

    # Create a dictionary for all the values
    sbet_values = dict(zip (field_names, values))

    sbet_values['lat_deg'] = math.degrees(sbet_values['latitude'])
    sbet_values['lon_deg'] = math.degrees(sbet_values['longitude'])

    return sbet_values  # Send the sbet_values dictionary back to the caller

def sbet_print(sbet_values):
    'Print out all the values of a SBET dictionary'
    print 'results:'
    for key in sbet_values:
        print '    ', key, sbet_values[key]

def decode(data, offset=0):
    '''Decipher a SBET datagram from binary'''

    # Offset now tells it how far to start
    values = struct.unpack('17d',data[ offset + 0 : offset + 8*17 ])

    # Create a dictionary for all the values
    sbet_values = dict(zip (field_names, values))

    sbet_values['lat_deg'] = math.degrees(sbet_values['latitude'])
    sbet_values['lon_deg'] = math.degrees(sbet_values['longitude'])

    return sbet_values

datagram_size = 136 # 8*17 bytes per datagram

def num_datagrams(data):
    'How many packets are in data'

    # Make sure we have an even number of datagrams
    assert (len(data) % datagram_size == 0)

    return len(data) / datagram_size

def main():
    sbet_file = open('sample.sbet')
    sbet_data = sbet_file.read()

    print 'Number of datagrams:', num_datagrams(sbet_data)

def get_offset(datagram_number):
    'Calculate the starting offset of a datagram'
    return datagram_number * datagram_size