Package aisutils :: Module database
[hide private]
[frames] | no frames]

Source Code for Module aisutils.database

  1  #!/usr/bin/env python 
  2  __author__ = 'Kurt Schwehr' 
  3  __version__ = '$Revision: 8545 $'.split()[1] 
  4  __revision__  = __version__ # For pylint 
  5  __date__ = '$Date: 2008-02-06 17:37:24 -0500 (Wed, 06 Feb 2008) $'.split()[1] 
  6  __copyright__ = '2008' 
  7  __license__   = 'GPL v2' 
  8  __contact__   = 'kurt at ccom.unh.edu' 
  9   
 10  __doc__=''' 
 11  AIS database utilities. 
 12   
 13  @status: under development 
 14  @since: 2008 Jan 09 
 15  @undocumented: __doc__ parser 
 16   
 17  @todo: probably lots that can be added here 
 18  @todo: createtables for the binary messages 
 19  @todo: use logging or sys.stderr rather than print 
 20  @see: U{WKT<http://dev.mysql.com/doc/refman/5.0/en/gis-wkt-format.html>} 
 21  ''' 
 22   
 23  import os 
 24  import sys 
 25  import ais 
 26  import psycopg2 as psycopg 
 27   
 28  dbTypes=( 
 29      'postgres' # AKA postgis 
 30      ,'sqlite' # Only sqlite3.  Time to ditch sqlite2 
 31  ) 
 32  ''' 
 33  The choices of databases that are supported. 
 34  ''' 
 35   
36 -def stdCmdlineOptions(parser,dbType='postgres',verbose=False):
37 ''' 38 Standard command line options 39 @param parser: OptionParser parser that will get the additional options 40 @param dbType: 'postgres' or 'sqlite' 41 ''' 42 if dbType != 'all' and dbType not in dbTypes: 43 # FIX: throw a database type exception 44 sys.exit('unknown database type: '+dbType) 45 46 if dbType in ('all','postgres'): 47 if verbose: sys.stderr.write('Adding postgres options\n') 48 parser.add_option('-d','--database-name',dest='databaseName',default='ais' 49 ,help='Name of database within the postgres server [default: %default]') 50 parser.add_option('-D','--database-host',dest='databaseHost',default='localhost' 51 ,help='Host name of the computer serving the dbx [default: %default]') 52 #defaultUser = os.genenv('USER') 53 defaultUser = os.getlogin() 54 parser.add_option('-u','--database-user',dest='databaseUser',default=defaultUser 55 ,help='Host name of the to access the database with [default: %default]') 56 57 parser.add_option('-p','--database-passwd',dest='databasePasswd',default=None 58 ,help='Password to access the database with [default: None]') 59 60 if dbType in ('all','sqlite'): 61 if verbose: sys.stderr.write('Adding sqlite options\n') 62 parser.add_option('-f','--database-file',dest='databaseFilename',default='ais.db3' 63 ,help='Name of the sqlite3 database file to write [default: %default]')
64 65
66 -def createTables(cx,dbType='sqlite',includeList=None, excludeList=None,verbose=False):
67 ''' 68 @param cx: database connection 69 @type cx: db API 2.0 object 70 @param dbType: postgres or sqlite 71 @type dbType: str 72 @param includeList: If a list of message numbers is passed, only these are created 73 @type includeList: list of integers 74 @param excludeList: If a list of message numbers is passed, all but these are created 75 @type excludeList: list of integers 76 ''' 77 cu = cx.cursor() 78 79 tables=[] 80 for msgNum in ais.msgModByNumber: 81 if excludeList is not None and msgNum in excludeList: continue 82 if includeList is not None and msgNum not in includeList: continue 83 84 aisMod = ais.msgModByNumber[msgNum] 85 86 if aisMod.dbTableName in tables: 87 if verbose: sys.stderr.write(str(msgNum)+' ... skipping - already in the db -'+str(aisMod.dbTableName)+'\n') 88 else: 89 if verbose: print msgNum,' ... adding '+aisMod.dbTableName+' table to db' 90 cu.execute(str(aisMod.sqlCreate(dbType=dbType))) 91 tables.append(aisMod.dbTableName) 92 93 cx.commit()
94
95 -def dropTables(cx,includeList=None, excludeList=None,verbose=False):
96 ''' 97 Kiss your data goodbye 98 99 @param cx: database connection 100 @type cx: db API 2.0 object 101 @param dbType: postgres or sqlite 102 @type dbType: str 103 @param includeList: If a list of message numbers is passed, only these are created 104 @type includeList: list of integers 105 @param excludeList: If a list of message numbers is passed, all but these are created 106 @type excludeList: list of integers 107 ''' 108 cu = cx.cursor() 109 110 tables=[] 111 for msgNum in ais.msgModByNumber: 112 if excludeList is not None and msgNum in excludeList: continue 113 if includeList is not None and msgNum not in includeList: continue 114 115 aisMod = ais.msgModByNumber[msgNum] 116 117 if aisMod.dbTableName in tables: 118 if verbose: print msgNum,' ... skipping - already dropped from the db -',aisMod.dbTableName 119 else: 120 if verbose: print msgNum,' ... dropping '+aisMod.dbTableName+' table to db' 121 cu.execute('DROP TABLE '+aisMod.dbTableName+';') 122 tables.append(aisMod.dbTableName) 123 124 cx.commit()
125 126 127 128
129 -def connect(options,dbType=None):
130 ''' 131 options must include the above standard options 132 ''' 133 verbose = options.verbose 134 if dbType is None: 135 dbType = options.dbType 136 137 cx = None 138 139 if dbType=='sqlite': 140 import sqlite3 141 cx = sqlite3.connect(options.databaseFilename) 142 143 elif dbType=='postgres': 144 #import psycopg2 as psycopg 145 # FIX: add password support 146 connectStr = "dbname='"+options.databaseName+"' user='"+options.databaseUser+"' host='"+options.databaseHost+"'" 147 148 if options.verbose: 149 print 'Connect string:',connectStr 150 cx = psycopg.connect(connectStr) 151 152 else: 153 sys.exit('Must specify a database type') 154 155 if verbose: 156 sys.stderr.write('connected to db\n') 157 return cx
158 159
160 -def rebuild_track_lines(cx,vessels=None 161 ,limitPoints=10 162 ,trackTable='track_lines' 163 ,trackKey='ogc_fid' 164 ,startTime=None 165 ,verbose=False):
166 ''' 167 @param vessels: if None, do all vessels in the tables, otherwise a set of MMSI values 168 @param trackTable: the database table where to put the lines 169 @param limitPoints: max number of points in a track line 170 @param startTime: oldest timestamp to allow in the track lines 171 @type startTime: datetime 172 ''' 173 v = verbose 174 cu = cx.cursor() 175 176 # 177 # position table - Class A 178 # 179 vesselsUpdated=0 180 if vessels is None: 181 cu.execute('SELECT distinct(userid) FROM position;') 182 vessels = set() 183 for v in cu.fetchall(): 184 vessels.add(v[0]) 185 186 if v: 187 sys.stderr.write('vessels %s\n' % str(vessels)) 188 189 for vessel in vessels: 190 191 query='SELECT AsText(position) FROM position WHERE userid=%s' 192 if startTime is not None: 193 query += ' AND cg_timestamp > %s' 194 query+=' ORDER BY cg_sec DESC' 195 if limitPoints is not None: query+=' LIMIT '+str(limitPoints) 196 query+=';' 197 198 print 'startTime should be set',startTime 199 if startTime is not None: 200 #print 'FIX: q', (query % (vessel,startTime)) 201 cu.execute(query,(vessel,startTime)) 202 else: 203 sys.exit ('NO!!!') 204 cu.execute(query,(vessel,)) 205 linePoints=[] 206 lineLen=0 207 for row in cu.fetchall(): 208 lineLen+=1 209 linePoints.append(row[0].split('(')[1].split(')')[0]) 210 if lineLen<2: 211 if v: 212 sys.stderr.write('Line needs at least 2 points for vessel %s\n' % vessel) 213 cu.execute ('SELECT '+trackKey+' FROM '+trackTable+' WHERE userid = %s;',( vessel,)) 214 row = cu.fetchall() 215 if len(row)>0: 216 if v: 217 sys.stderr.write('dropping vessel %s from track\n' % vessel) 218 cu.execute('DELETE FROM '+trackTable+' WHERE userid = %s;',(vessel,)) 219 continue 220 lineWKT='LINESTRING('+','.join(linePoints)+')' 221 if v: 222 sys.stderr.write(str(len(linePoints))+' points used for vessel '+str(vessel)+'\n') 223 # Get the most recent vessel name 224 #if v: print 'Getting name...' 225 cu.execute('SELECT name FROM shipdata WHERE userid='+str(vessel)+' LIMIT 1') 226 name = cu.fetchall() 227 if len(name)==1: 228 name=name[0][0].strip() 229 while name[-1]=='@': name=name[:-1] 230 name=name.strip() 231 else: name=str(vessel) 232 sys.stderr.write('NAME for '+str(vessel)+'is "'+name+'"\n') 233 234 cu.execute('SELECT '+trackKey+' FROM '+trackTable+' WHERE userid='+str(vessel)+'\n') 235 track_keys = cu.fetchall() 236 #print 'track_keys',track_keys 237 if len(track_keys)==0: 238 # Does not exist in the database, so insert a new line 239 query = 'INSERT INTO track_lines (userid,name,track) VALUES (%s,%s,GeomFromText(%s,4326));' 240 try: 241 cu.execute(query,(vessel,name,lineWKT)) 242 except psycopg.ProgrammingError,inst: 243 sys.stderr.write('psycopg2 execute flailed: '+str(inst)+'\n') 244 else: 245 vesselsUpdated += 1 246 elif len(track_keys)==1: 247 # Need to replace an existing row 248 query = 'UPDATE track_lines SET name = %s, track = GeomFromText(%s,4326) WHERE '+trackKey+' = %s' 249 key = track_keys[0][0] 250 try: 251 cu.execute(query,(name,lineWKT,key)) 252 except psycopg.ProgrammingError,inst: 253 sys.stderr.write('psycopg2 execute flailed: '+str(inst)+'\n') 254 else: 255 vesselsUpdated += 1 256 else: 257 # How did this happen??? 258 sys.stderr.write('ERROR: database corrupted ... too many track lines for '+str(vessel)+'\n') 259 260 if vesselsUpdated>0: 261 cx.commit() 262 if v: sys.stderr.write('Updated tracks ... '+str(vesselsUpdated)+' tracks updated\n')
263 264 # 265 # Handle class B reports here 266 # 267
268 -def rebuild_last_position(cx 269 ,vesselsClassA=None 270 ,vesselsClassB=None 271 ,lastPosTable='last_position' 272 ,posKey='key' 273 ,startTime=None 274 ,verbose=False):
275 ''' 276 This is to speed up the redrawing of the most recent position drawing in mapserver 277 278 @param vessels: if None, do all vessels in the tables, otherwise a set of MMSI values 279 @param trackTable: the database table where to put the lines 280 @param startTime: oldest timestamp to allow in the last_position table 281 @type startTime: datetime 282 ''' 283 v = verbose 284 cu = cx.cursor() 285 286 # 287 # position table - Class A 288 # 289 290 vesselsUpdated=0 291 if vesselsClassA is None: 292 cu.execute('SELECT distinct(userid) FROM position;') 293 vesselsClassA = set() 294 for v in cu.fetchall(): 295 vesselsClassA.add(v[0]) 296 297 if v: print 'class A vessels',str(vesselsClassA) 298 for vessel in vesselsClassA: 299 300 query='SELECT position,cog,sog,cg_timestamp FROM position WHERE userid=%s ORDER BY cg_sec DESC LIMIT 1;' 301 #if startTime is not None: 302 # query += ' AND cg_timestamp > %s' 303 #query+=' ORDER BY cg_sec DESC LIMIT 1;' 304 #if startTime is not None: 305 # #print 'FIX: q', (query % (vessel,startTime)) 306 # cu.execute(query,(vessel,startTime)) 307 #else: 308 cu.execute(query,(vessel,)); 309 310 rows = cu.fetchall() 311 if len(rows)==0: continue 312 row = rows[0] 313 position = row[0] # Already in WKB 314 cog = int(row[1]) # convert from decimal 315 sog = float(row[2]) # convert from decimal 316 cg_timestamp = row[3] 317 318 if startTime is not None and cg_timestamp < startTime: 319 cu.execute ('SELECT '+posKey+' FROM '+lastPosTable+' WHERE userid = %s;',( vessel,)) 320 row = cu.fetchall() 321 if len(row)>0: 322 if v: 323 sys.stderr.write('dropping vessel %s from last_position\n' % vessel) 324 cu.execute('DELETE FROM '+lastPosTable+' WHERE userid = %s;',(vessel,)) 325 continue 326 327 cu.execute('SELECT name,shipandcargo FROM shipdata WHERE userid='+str(vessel)+' LIMIT 1') 328 name = cu.fetchall() 329 if len(name)==1: 330 name=name[0][0].strip() 331 while name[-1]=='@': name=name[:-1] 332 name=name.strip() 333 else: name=str(vessel) 334 sys.stderr.write('NAME for '+str(vessel)+'is "'+name+'"\n') 335 336 # FIX: set color based on shipandcargo 337 338 cu.execute('SELECT '+posKey+' FROM '+lastPosTable+' WHERE userid=%s\n', (vessel,)) 339 lastpos_keys = cu.fetchall() 340 341 if len(lastpos_keys)==0: 342 # Does not exist in the database, so insert a new line 343 print 'inserting...',vessel,name,cog,cg_timestamp,position 344 query = 'INSERT INTO '+lastPosTable+' (userid,name,cog,sog,cg_timestamp,position) VALUES (%s,%s,%s,%s,%s,%s);' 345 try: 346 print query 347 cu.execute(query,(vessel,name,cog,sog,cg_timestamp,position)) 348 except psycopg.ProgrammingError,inst: 349 sys.stderr.write('psycopg2 execute flailed: '+str(inst)+' for\n ') 350 sys.stderr.write(query+'\n') 351 else: 352 vesselsUpdated += 1 353 elif len(lastpos_keys)==1: 354 # Need to replace an existing row 355 356 query = 'UPDATE '+lastPosTable+' SET name = %s, cog = %s, sog = %s, cg_timestamp=%s, position = %s WHERE '+posKey+' = %s' 357 key = lastpos_keys[0][0] 358 try: 359 cu.execute(query,(name,cog,sog,cg_timestamp,position,key)) 360 except psycopg.ProgrammingError,inst: 361 sys.stderr.write('psycopg2 execute flailed: '+str(inst)+' for\n ') 362 sys.stderr.write(query+'\n') 363 else: 364 vesselsUpdated += 1 365 else: 366 # How did this happen??? 367 sys.stderr.write('ERROR: database corrupted ... too many positions for '+str(vessel)+'\n') 368 369 if vesselsUpdated>0: 370 cx.commit() 371 if v: sys.stderr.write('Updated tracks ... '+str(vesselsUpdated)+' tracks updated\n') 372 373 if vesselsClassB is not None: 374 print 'FIX: class B not yet implemented'
375 376 377 # FIX: unittests here? 378