1
2 __author__ = 'Kurt Schwehr'
3 __version__ = '$Revision: 8545 $'.split()[1]
4 __revision__ = __version__
5 __date__ = '$Date: 2008-02-06 17:37:24 -0500 (Wed, 06 Feb 2008) $'.split()[1]
6 __copyright__ = '2008'
7 __license__ = 'GPL v2'
8 __contact__ = 'kurt at ccom.unh.edu'
9
10 __doc__='''
11 AIS database utilities.
12
13 @status: under development
14 @since: 2008 Jan 09
15 @undocumented: __doc__ parser
16
17 @todo: probably lots that can be added here
18 @todo: createtables for the binary messages
19 @todo: use logging or sys.stderr rather than print
20 @see: U{WKT<http://dev.mysql.com/doc/refman/5.0/en/gis-wkt-format.html>}
21 '''
22
23 import os
24 import sys
25 import ais
26 import psycopg2 as psycopg
27
28 dbTypes=(
29 'postgres'
30 ,'sqlite'
31 )
32 '''
33 The choices of databases that are supported.
34 '''
35
37 '''
38 Standard command line options
39 @param parser: OptionParser parser that will get the additional options
40 @param dbType: 'postgres' or 'sqlite'
41 '''
42 if dbType != 'all' and dbType not in dbTypes:
43
44 sys.exit('unknown database type: '+dbType)
45
46 if dbType in ('all','postgres'):
47 if verbose: sys.stderr.write('Adding postgres options\n')
48 parser.add_option('-d','--database-name',dest='databaseName',default='ais'
49 ,help='Name of database within the postgres server [default: %default]')
50 parser.add_option('-D','--database-host',dest='databaseHost',default='localhost'
51 ,help='Host name of the computer serving the dbx [default: %default]')
52
53 defaultUser = os.getlogin()
54 parser.add_option('-u','--database-user',dest='databaseUser',default=defaultUser
55 ,help='Host name of the to access the database with [default: %default]')
56
57 parser.add_option('-p','--database-passwd',dest='databasePasswd',default=None
58 ,help='Password to access the database with [default: None]')
59
60 if dbType in ('all','sqlite'):
61 if verbose: sys.stderr.write('Adding sqlite options\n')
62 parser.add_option('-f','--database-file',dest='databaseFilename',default='ais.db3'
63 ,help='Name of the sqlite3 database file to write [default: %default]')
64
65
66 -def createTables(cx,dbType='sqlite',includeList=None, excludeList=None,verbose=False):
67 '''
68 @param cx: database connection
69 @type cx: db API 2.0 object
70 @param dbType: postgres or sqlite
71 @type dbType: str
72 @param includeList: If a list of message numbers is passed, only these are created
73 @type includeList: list of integers
74 @param excludeList: If a list of message numbers is passed, all but these are created
75 @type excludeList: list of integers
76 '''
77 cu = cx.cursor()
78
79 tables=[]
80 for msgNum in ais.msgModByNumber:
81 if excludeList is not None and msgNum in excludeList: continue
82 if includeList is not None and msgNum not in includeList: continue
83
84 aisMod = ais.msgModByNumber[msgNum]
85
86 if aisMod.dbTableName in tables:
87 if verbose: sys.stderr.write(str(msgNum)+' ... skipping - already in the db -'+str(aisMod.dbTableName)+'\n')
88 else:
89 if verbose: print msgNum,' ... adding '+aisMod.dbTableName+' table to db'
90 cu.execute(str(aisMod.sqlCreate(dbType=dbType)))
91 tables.append(aisMod.dbTableName)
92
93 cx.commit()
94
95 -def dropTables(cx,includeList=None, excludeList=None,verbose=False):
96 '''
97 Kiss your data goodbye
98
99 @param cx: database connection
100 @type cx: db API 2.0 object
101 @param dbType: postgres or sqlite
102 @type dbType: str
103 @param includeList: If a list of message numbers is passed, only these are created
104 @type includeList: list of integers
105 @param excludeList: If a list of message numbers is passed, all but these are created
106 @type excludeList: list of integers
107 '''
108 cu = cx.cursor()
109
110 tables=[]
111 for msgNum in ais.msgModByNumber:
112 if excludeList is not None and msgNum in excludeList: continue
113 if includeList is not None and msgNum not in includeList: continue
114
115 aisMod = ais.msgModByNumber[msgNum]
116
117 if aisMod.dbTableName in tables:
118 if verbose: print msgNum,' ... skipping - already dropped from the db -',aisMod.dbTableName
119 else:
120 if verbose: print msgNum,' ... dropping '+aisMod.dbTableName+' table to db'
121 cu.execute('DROP TABLE '+aisMod.dbTableName+';')
122 tables.append(aisMod.dbTableName)
123
124 cx.commit()
125
126
127
128
130 '''
131 options must include the above standard options
132 '''
133 verbose = options.verbose
134 if dbType is None:
135 dbType = options.dbType
136
137 cx = None
138
139 if dbType=='sqlite':
140 import sqlite3
141 cx = sqlite3.connect(options.databaseFilename)
142
143 elif dbType=='postgres':
144
145
146 connectStr = "dbname='"+options.databaseName+"' user='"+options.databaseUser+"' host='"+options.databaseHost+"'"
147
148 if options.verbose:
149 print 'Connect string:',connectStr
150 cx = psycopg.connect(connectStr)
151
152 else:
153 sys.exit('Must specify a database type')
154
155 if verbose:
156 sys.stderr.write('connected to db\n')
157 return cx
158
159
160 -def rebuild_track_lines(cx,vessels=None
161 ,limitPoints=10
162 ,trackTable='track_lines'
163 ,trackKey='ogc_fid'
164 ,startTime=None
165 ,verbose=False):
166 '''
167 @param vessels: if None, do all vessels in the tables, otherwise a set of MMSI values
168 @param trackTable: the database table where to put the lines
169 @param limitPoints: max number of points in a track line
170 @param startTime: oldest timestamp to allow in the track lines
171 @type startTime: datetime
172 '''
173 v = verbose
174 cu = cx.cursor()
175
176
177
178
179 vesselsUpdated=0
180 if vessels is None:
181 cu.execute('SELECT distinct(userid) FROM position;')
182 vessels = set()
183 for v in cu.fetchall():
184 vessels.add(v[0])
185
186 if v:
187 sys.stderr.write('vessels %s\n' % str(vessels))
188
189 for vessel in vessels:
190
191 query='SELECT AsText(position) FROM position WHERE userid=%s'
192 if startTime is not None:
193 query += ' AND cg_timestamp > %s'
194 query+=' ORDER BY cg_sec DESC'
195 if limitPoints is not None: query+=' LIMIT '+str(limitPoints)
196 query+=';'
197
198 print 'startTime should be set',startTime
199 if startTime is not None:
200
201 cu.execute(query,(vessel,startTime))
202 else:
203 sys.exit ('NO!!!')
204 cu.execute(query,(vessel,))
205 linePoints=[]
206 lineLen=0
207 for row in cu.fetchall():
208 lineLen+=1
209 linePoints.append(row[0].split('(')[1].split(')')[0])
210 if lineLen<2:
211 if v:
212 sys.stderr.write('Line needs at least 2 points for vessel %s\n' % vessel)
213 cu.execute ('SELECT '+trackKey+' FROM '+trackTable+' WHERE userid = %s;',( vessel,))
214 row = cu.fetchall()
215 if len(row)>0:
216 if v:
217 sys.stderr.write('dropping vessel %s from track\n' % vessel)
218 cu.execute('DELETE FROM '+trackTable+' WHERE userid = %s;',(vessel,))
219 continue
220 lineWKT='LINESTRING('+','.join(linePoints)+')'
221 if v:
222 sys.stderr.write(str(len(linePoints))+' points used for vessel '+str(vessel)+'\n')
223
224
225 cu.execute('SELECT name FROM shipdata WHERE userid='+str(vessel)+' LIMIT 1')
226 name = cu.fetchall()
227 if len(name)==1:
228 name=name[0][0].strip()
229 while name[-1]=='@': name=name[:-1]
230 name=name.strip()
231 else: name=str(vessel)
232 sys.stderr.write('NAME for '+str(vessel)+'is "'+name+'"\n')
233
234 cu.execute('SELECT '+trackKey+' FROM '+trackTable+' WHERE userid='+str(vessel)+'\n')
235 track_keys = cu.fetchall()
236
237 if len(track_keys)==0:
238
239 query = 'INSERT INTO track_lines (userid,name,track) VALUES (%s,%s,GeomFromText(%s,4326));'
240 try:
241 cu.execute(query,(vessel,name,lineWKT))
242 except psycopg.ProgrammingError,inst:
243 sys.stderr.write('psycopg2 execute flailed: '+str(inst)+'\n')
244 else:
245 vesselsUpdated += 1
246 elif len(track_keys)==1:
247
248 query = 'UPDATE track_lines SET name = %s, track = GeomFromText(%s,4326) WHERE '+trackKey+' = %s'
249 key = track_keys[0][0]
250 try:
251 cu.execute(query,(name,lineWKT,key))
252 except psycopg.ProgrammingError,inst:
253 sys.stderr.write('psycopg2 execute flailed: '+str(inst)+'\n')
254 else:
255 vesselsUpdated += 1
256 else:
257
258 sys.stderr.write('ERROR: database corrupted ... too many track lines for '+str(vessel)+'\n')
259
260 if vesselsUpdated>0:
261 cx.commit()
262 if v: sys.stderr.write('Updated tracks ... '+str(vesselsUpdated)+' tracks updated\n')
263
264
265
266
267
268 -def rebuild_last_position(cx
269 ,vesselsClassA=None
270 ,vesselsClassB=None
271 ,lastPosTable='last_position'
272 ,posKey='key'
273 ,startTime=None
274 ,verbose=False):
275 '''
276 This is to speed up the redrawing of the most recent position drawing in mapserver
277
278 @param vessels: if None, do all vessels in the tables, otherwise a set of MMSI values
279 @param trackTable: the database table where to put the lines
280 @param startTime: oldest timestamp to allow in the last_position table
281 @type startTime: datetime
282 '''
283 v = verbose
284 cu = cx.cursor()
285
286
287
288
289
290 vesselsUpdated=0
291 if vesselsClassA is None:
292 cu.execute('SELECT distinct(userid) FROM position;')
293 vesselsClassA = set()
294 for v in cu.fetchall():
295 vesselsClassA.add(v[0])
296
297 if v: print 'class A vessels',str(vesselsClassA)
298 for vessel in vesselsClassA:
299
300 query='SELECT position,cog,sog,cg_timestamp FROM position WHERE userid=%s ORDER BY cg_sec DESC LIMIT 1;'
301
302
303
304
305
306
307
308 cu.execute(query,(vessel,));
309
310 rows = cu.fetchall()
311 if len(rows)==0: continue
312 row = rows[0]
313 position = row[0]
314 cog = int(row[1])
315 sog = float(row[2])
316 cg_timestamp = row[3]
317
318 if startTime is not None and cg_timestamp < startTime:
319 cu.execute ('SELECT '+posKey+' FROM '+lastPosTable+' WHERE userid = %s;',( vessel,))
320 row = cu.fetchall()
321 if len(row)>0:
322 if v:
323 sys.stderr.write('dropping vessel %s from last_position\n' % vessel)
324 cu.execute('DELETE FROM '+lastPosTable+' WHERE userid = %s;',(vessel,))
325 continue
326
327 cu.execute('SELECT name,shipandcargo FROM shipdata WHERE userid='+str(vessel)+' LIMIT 1')
328 name = cu.fetchall()
329 if len(name)==1:
330 name=name[0][0].strip()
331 while name[-1]=='@': name=name[:-1]
332 name=name.strip()
333 else: name=str(vessel)
334 sys.stderr.write('NAME for '+str(vessel)+'is "'+name+'"\n')
335
336
337
338 cu.execute('SELECT '+posKey+' FROM '+lastPosTable+' WHERE userid=%s\n', (vessel,))
339 lastpos_keys = cu.fetchall()
340
341 if len(lastpos_keys)==0:
342
343 print 'inserting...',vessel,name,cog,cg_timestamp,position
344 query = 'INSERT INTO '+lastPosTable+' (userid,name,cog,sog,cg_timestamp,position) VALUES (%s,%s,%s,%s,%s,%s);'
345 try:
346 print query
347 cu.execute(query,(vessel,name,cog,sog,cg_timestamp,position))
348 except psycopg.ProgrammingError,inst:
349 sys.stderr.write('psycopg2 execute flailed: '+str(inst)+' for\n ')
350 sys.stderr.write(query+'\n')
351 else:
352 vesselsUpdated += 1
353 elif len(lastpos_keys)==1:
354
355
356 query = 'UPDATE '+lastPosTable+' SET name = %s, cog = %s, sog = %s, cg_timestamp=%s, position = %s WHERE '+posKey+' = %s'
357 key = lastpos_keys[0][0]
358 try:
359 cu.execute(query,(name,cog,sog,cg_timestamp,position,key))
360 except psycopg.ProgrammingError,inst:
361 sys.stderr.write('psycopg2 execute flailed: '+str(inst)+' for\n ')
362 sys.stderr.write(query+'\n')
363 else:
364 vesselsUpdated += 1
365 else:
366
367 sys.stderr.write('ERROR: database corrupted ... too many positions for '+str(vessel)+'\n')
368
369 if vesselsUpdated>0:
370 cx.commit()
371 if v: sys.stderr.write('Updated tracks ... '+str(vesselsUpdated)+' tracks updated\n')
372
373 if vesselsClassB is not None:
374 print 'FIX: class B not yet implemented'
375
376
377
378