server.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570
  1. #!/usr/bin/python
  2. # -*- coding: utf-8 -*-
  3. from __future__ import print_function
  4. from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
  5. import cgi
  6. from storage import Storage
  7. import json
  8. import logging
  9. import pygeoip
  10. import re
  11. import socket
  12. from SocketServer import ThreadingMixIn
  13. import time
  14. import ffstatus
  15. class BatcaveHttpRequestHandler(BaseHTTPRequestHandler):
  16. DATAKEY_VPN = '__VPN__'
  17. FIELDKEY_UPDATED = '__UPDATED__'
  18. def __init__(self, request, client_address, server):
  19. self.logger = logging.getLogger('API')
  20. BaseHTTPRequestHandler.__init__(self, request, client_address, server)
  21. def parse_url_pathquery(self):
  22. """Extracts the query parameters from the request path."""
  23. url = re.match(r'^/(?P<path>.*?)(\?(?P<query>.+))?$', self.path.strip())
  24. if url is None:
  25. logging.warn('Failed to parse URL \'' + str(self.path) + '\'.')
  26. return ( None, None )
  27. path = url.group('path')
  28. query = {}
  29. if not url.group('query') is None:
  30. for m in re.finditer(r'(?P<key>.+?)=(?P<value>.+?)(&|$)', url.group('query')):
  31. query[m.group('key')] = m.group('value')
  32. return ( path, query )
  33. def do_GET(self):
  34. """Handles all HTTP GET requests."""
  35. path, query = self.parse_url_pathquery()
  36. if path is None:
  37. self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
  38. return
  39. # / - index page, shows generic help
  40. if path == '':
  41. self.respond_index(query)
  42. return
  43. # /list - list stored nodes
  44. if path == 'list':
  45. self.respond_list(query)
  46. return
  47. # /vpn - notification endpoint for gateway's VPN connections
  48. if path == 'vpn':
  49. self.respond_vpn(query)
  50. return
  51. # /providers
  52. if path == 'providers':
  53. self.respond_providers(query)
  54. return
  55. # /node/<id>.json - node's data
  56. # /node/<id>/field - return specific field from node's data
  57. m = re.match(r'node/(?P<id>[a-fA-F0-9]{12})(?P<cmd>\.json|/[a-zA-Z0-9_\-\.]+)$', path)
  58. if m != None:
  59. cmd = m.group('cmd')
  60. nodeid = m.group('id').lower()
  61. if cmd == '.json':
  62. self.respond_node(nodeid)
  63. else:
  64. self.respond_nodedetail(nodeid, cmd[1:])
  65. return
  66. # /status/<id> - node's status
  67. m = re.match(r'status/([a-f0-9]{12})$', path)
  68. if m != None:
  69. self.respond_nodestatus(m.group(1))
  70. return
  71. # no match -> 404
  72. self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))
  73. def do_POST(self):
  74. """Handles all HTTP POST requests."""
  75. path, query = self.parse_url_pathquery()
  76. if path is None:
  77. self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
  78. return
  79. params = self.parse_post_params()
  80. # node id/mac to name mapping
  81. if path == 'idmac2name':
  82. self.respond_nodeidmac2name(params)
  83. return
  84. # no match -> 404
  85. self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))
  86. def send_nocache_headers(self):
  87. """Sets HTTP headers indicating that this response shall not be cached."""
  88. self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
  89. self.send_header('Pragma', 'no-cache')
  90. self.send_header('Expires', '0')
  91. def send_headers(self, content_type='text/html; charset=utf-8', nocache=True):
  92. """Send HTTP 200 Response header with the given Content-Type.
  93. Optionally send no-caching headers, too."""
  94. self.send_response(200)
  95. self.send_header('Content-Type', content_type)
  96. if nocache: self.send_nocache_headers()
  97. self.end_headers()
  98. def parse_post_params(self):
  99. ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
  100. if ctype == 'multipart/form-data':
  101. postvars = cgi.parse_multipart(self.rfile, pdict)
  102. elif ctype == 'application/x-www-form-urlencoded':
  103. length = int(self.headers.getheader('content-length'))
  104. postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1)
  105. else:
  106. postvars = {}
  107. return postvars
  108. def respond_index(self, query):
  109. """Display the index page."""
  110. storage = self.server.storage
  111. self.send_headers()
  112. self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE</title></head>\n')
  113. self.wfile.write('<body>\n')
  114. self.wfile.write('<H1 title="Batman/Alfred Transmission Collection, Aggregation & Value Engine">BATCAVE</H1>\n')
  115. self.wfile.write('<p>Dies ist ein interner Hintergrund-Dienst. Er wird nur von anderen Diensten\n')
  116. self.wfile.write('angesprochen und sollte aus einer Mehrzahl von Gr&uuml;nden nicht &ouml;ffentlich\n')
  117. self.wfile.write('zug&auml;nglich sein.</p>\n')
  118. self.wfile.write('<H2>Status</H2>\n')
  119. self.wfile.write('Daten: <span id="datacount" class="value">')
  120. self.wfile.write(len(storage.data))
  121. self.wfile.write('</span>\n')
  122. self.wfile.write('<H2>API</H2>\n')
  123. self.wfile.write('<p>Grundsätzlich ist das Antwort-Format JSON und alle Daten sind Live-Daten (kein Cache) die ggf. etwas Bearbeitungs-Zeit erfordern.</p>')
  124. self.wfile.write('<dl>\n')
  125. self.wfile.write('<dt><a href="/nodes.json">nodes.json</a></dt><dd>zur Verwendung mit ffmap (MACs anonymisiert)</dd>\n')
  126. self.wfile.write('<dt><a href="/node/ff00ff00ff00.json">/node/&lt;id&gt;.json</a></dt><dd><u>alle</u> vorhandenen Information zu der gewünschten Node</dd>\n')
  127. self.wfile.write('</dl>\n')
  128. self.wfile.write('</body></html>')
  129. def respond_list(self, query):
  130. """List stored data."""
  131. storage = self.server.storage
  132. self.send_headers()
  133. self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE</title></head>\n')
  134. self.wfile.write('<body>\n')
  135. self.wfile.write('<H1>BATCAVE - LIST</H1>\n')
  136. self.wfile.write('<table>\n')
  137. self.wfile.write('<thead><tr><th>ID</th><th>Name</th></tr></thead>\n')
  138. self.wfile.write('<tbody>\n')
  139. data = storage.data
  140. if 'sort' in query:
  141. if query['sort'] == 'name':
  142. sorteddata = sorted(data, key=lambda x: data[x]['hostname'].lower())
  143. data = sorteddata
  144. elif query['sort'] == 'id':
  145. sorteddata = sorted(data)
  146. data = sorteddata
  147. for nodeid in data:
  148. if nodeid.startswith('__'): continue
  149. nodename = storage.data[nodeid]['hostname'] if 'hostname' in storage.data[nodeid] else '&lt;?&gt;'
  150. self.wfile.write('<tr><td><a href="/node/' + nodeid + '.json">' + nodeid + '</a></td><td>' + nodename + '</td></tr>')
  151. self.wfile.write('</tbody>\n')
  152. self.wfile.write('</table>\n')
  153. def find_node(self, rawid):
  154. """Fetch node data from storage by given id, if necessary looking thorugh node aliases."""
  155. storage = self.server.storage
  156. # if we have a direct hit, return it immediately
  157. if rawid in storage.data:
  158. return storage.data[rawid]
  159. # no direct hit -> search via aliases
  160. nodeid = rawid
  161. for n in storage.data:
  162. if 'aliases' in storage.data[n] and rawid in storage.data[n]['aliases']:
  163. nodeid = n
  164. # return found node
  165. return storage.data[nodeid] if nodeid in storage.data else None
  166. def find_node_by_mac(self, mac):
  167. """Fetch node data from storage by given MAC address."""
  168. storage = self.server.storage
  169. needle = mac.lower()
  170. # iterate over all nodes
  171. for nodeid in storage.data:
  172. if nodeid.startswith('__'): continue
  173. node = storage.data[nodeid]
  174. # check node's primary MAC
  175. if 'mac' in node and needle == node['mac'].lower():
  176. return node
  177. # check alias MACs
  178. if 'macs' in node:
  179. haystack = [ x.lower() for x in node['macs'] ]
  180. if mac in haystack:
  181. return node
  182. # MAC address not found
  183. return None
  184. def respond_node(self, rawid):
  185. """Display node data."""
  186. # handle API example linked on index page
  187. if rawid == 'ff00ff00ff00':
  188. self.send_headers('text/json')
  189. self.wfile.write(json.dumps({
  190. 'name': 'API-Example',
  191. 'nodeid': rawid,
  192. 'META': 'Dies ist ein minimaler Beispiel-Datensatz. Herzlichen Glückwunsch, du hast das Prinzip der API kapiert.',
  193. }))
  194. return
  195. # search node by the given id
  196. node = self.find_node(rawid)
  197. # handle unknown nodes
  198. if node is None:
  199. self.send_error(404, 'No node with id \'' + rawid + '\' present.')
  200. return
  201. # remove fields from output: __RAW__
  202. export = ffstatus.dict_merge({}, node)
  203. if '__RAW__' in export:
  204. del(export['__RAW__'])
  205. # dump node data as JSON
  206. self.send_headers('text/json')
  207. self.wfile.write(json.dumps(export))
  208. def get_nodestatus(self, rawid):
  209. """Determine node's status."""
  210. # search node by the given id
  211. node = self.find_node(rawid)
  212. # handle unknown nodes
  213. if node is None:
  214. return None
  215. # check that the last batadv update is noted in the data
  216. updated = node[self.FIELDKEY_UPDATED] if self.FIELDKEY_UPDATED in node else None
  217. if updated is None or not 'batadv' in updated:
  218. return 'unknown'
  219. # make decision based on time of last batadv update
  220. diff = time.time() - updated['batadv']
  221. if diff < 150:
  222. return 'active'
  223. elif diff < 300:
  224. return 'stale'
  225. else:
  226. return 'offline'
  227. def respond_nodestatus(self, rawid):
  228. """Display node status."""
  229. status = self.get_nodestatus(rawid)
  230. if status is None:
  231. self.send_error(404, 'No node with id \'' + rawid + '\' present.')
  232. self.send_headers('text/plain')
  233. self.wfile.write(status)
  234. def respond_nodeidmac2name(self, ids):
  235. """Return a mapping of the given IDs (or MACs) into their hostname."""
  236. self.send_headers('text/plain')
  237. for nodeid in ids:
  238. node = self.find_node(nodeid) if not ':' in nodeid else self.find_node_by_mac(nodeid)
  239. nodename = node['hostname'] if (not node is None) and 'hostname' in node else nodeid
  240. self.wfile.write('{0}={1}\n'.format(nodeid, nodename))
  241. def respond_nodedetail(self, nodeid, field):
  242. """Return a field from the given node - a string is returned as text, all other as JSON."""
  243. node = self.find_node(nodeid)
  244. if node is None:
  245. self.send_error(404, 'No node with id \'' + nodeid + '\' present.')
  246. return
  247. return_count = False
  248. if field.endswith('.count'):
  249. return_count = True
  250. field = field[0:-6]
  251. if not field in node:
  252. self.send_error(404, 'The node \'' + nodeid + '\' does not have a field named \'' + str(field) + '\'.')
  253. return
  254. value = node[field]
  255. if return_count: value = len(value)
  256. self.send_headers('text/plain' if isinstance(value, basestring) or isinstance(value, int) else 'text/json')
  257. self.wfile.write(value if isinstance(value, basestring) else json.dumps(value))
  258. def respond_vpn(self, query):
  259. storage = self.server.storage
  260. peername = query['peer'] if 'peer' in query else None
  261. key = query['key'] if 'key' in query else None
  262. action = query['action'] if 'action' in query else None
  263. remote = query['remote'] if 'remote' in query else None
  264. gw = query['gw'] if 'gw' in query else None
  265. if action == 'list':
  266. self.respond_vpnlist()
  267. return
  268. if action != 'establish' and action != 'disestablish':
  269. self.logger.error('VPN: unknown action \'{0}\''.format(action))
  270. self.send_error(400, 'Invalid action.')
  271. return
  272. for k,v in { 'peername': peername, 'key': key, 'remote': remote, 'gw': gw }.items():
  273. if v is None or len(v.strip()) == 0:
  274. self.logger.error('VPN {0}: no or empty {1}'.format(action, k))
  275. self.send_error(400, 'Missing value for ' + str(k))
  276. return
  277. if key is None or re.match(r'^[a-fA-F0-9]+$', key) is None:
  278. self.logger.error('VPN peer \'{0}\' {1}: bad key \'{2}\''.format(peername, action, key))
  279. self.send_error(400, 'Bad key.')
  280. return
  281. if not self.DATAKEY_VPN in storage.data: storage.data[self.DATAKEY_VPN] = {}
  282. if not key in storage.data[self.DATAKEY_VPN]: storage.data[self.DATAKEY_VPN][key] = { 'active': {}, 'last': {} }
  283. item = storage.data[self.DATAKEY_VPN][key]
  284. # resolve remote addr to its netblock
  285. remote_raw = remote
  286. remote_resolved = None
  287. if not remote is None:
  288. remote_resolved = ffstatus.resolve_ipblock(remote)
  289. if not remote_resolved is None:
  290. self.logger.debug('Resolved IP \'{0}\' to block \'{1}\'.'.format(remote, remote_resolved['name']))
  291. remote = remote_resolved
  292. if action == 'establish':
  293. item['active'][gw] = {
  294. 'establish': time.time(),
  295. 'peer': peername,
  296. 'remote': remote,
  297. 'remote_raw': remote_raw,
  298. }
  299. elif action == 'disestablish':
  300. active = {}
  301. if gw in item['active']:
  302. active = item['active'][gw]
  303. del(item['active'][gw])
  304. active['disestablish'] = time.time()
  305. item['last'][gw] = active
  306. else:
  307. self.send_error(500, 'Unknown action not filtered (' + str(action) + ')')
  308. return
  309. self.send_headers('text/plain')
  310. self.wfile.write('OK')
  311. storage.save()
  312. def respond_vpnlist(self):
  313. storage = self.server.storage
  314. gateways = ['gw01','gw02','gw03','gw04','gw05','gw06']
  315. self.send_headers()
  316. self.wfile.write('<!DOCTYPE html>\n')
  317. self.wfile.write('<html><head><title>BATCAVE - VPN LIST</title></head>\n')
  318. self.wfile.write('<body>\n')
  319. self.wfile.write('<style type="text/css">\n')
  320. self.wfile.write('table { border: 2px solid #999; border-collapse: collapse; }\n')
  321. self.wfile.write('th, td { border: 1px solid #CCC; }\n')
  322. self.wfile.write('table tbody tr.online { background-color: #CFC; }\n')
  323. self.wfile.write('table tbody tr.offline { background-color: #FCC; }\n')
  324. self.wfile.write('</style>\n')
  325. self.wfile.write('<table>\n<thead>\n')
  326. self.wfile.write('<tr><th rowspan="2">names (key)</th><th colspan="' + str(len(gateways)) + '">active</th><th colspan="' + str(len(gateways)) + '">last</th></tr>\n')
  327. self.wfile.write('<tr><th>' + '</th><th>'.join(gateways) + '</th><th>' + '</th><th>'.join(gateways) + '</th></tr>\n')
  328. self.wfile.write('</thead>\n')
  329. if self.DATAKEY_VPN in storage.data:
  330. for key in storage.data[self.DATAKEY_VPN]:
  331. item = storage.data[self.DATAKEY_VPN][key]
  332. if not isinstance(item, dict):
  333. continue
  334. names = set()
  335. count = {}
  336. for t in [ 'active', 'last' ]:
  337. count[t] = 0
  338. if t in item:
  339. for gw in item[t]:
  340. if 'remote' in item[t][gw] and len(item[t][gw]['remote']) > 0:
  341. count[t] += 1
  342. if 'peer' in item[t][gw]:
  343. names.add(item[t][gw]['peer'])
  344. self.wfile.write('<tr class="online">' if count['active'] > 0 else '<tr class="offline">')
  345. self.wfile.write('<td title="' + str(key) + '">' + (' / '.join(names) if len(names) > 0 else '?') + '</td>')
  346. for t in [ 'active', 'last' ]:
  347. for gw in gateways:
  348. ip = ''
  349. if t in item and gw in item[t]:
  350. ip = item[t][gw]['remote'] if 'remote' in item[t][gw] else ''
  351. if isinstance(ip, dict):
  352. ip = ip['name']
  353. self.wfile.write('<td title="' + ip + '">' + ('&check;' if len(ip) > 0 else '&times;') + '</td>')
  354. self.wfile.write('</tr>\n')
  355. self.wfile.write('</table>\n')
  356. self.wfile.write('</body>')
  357. self.wfile.write('</html>')
  358. def respond_providers(self, query):
  359. """Return a summary of providers."""
  360. vpn = self.server.storage.data[self.DATAKEY_VPN]
  361. outputformat = query['format'].lower() if 'format' in query else 'html'
  362. geo = None
  363. try:
  364. geo = pygeoip.GeoIP('GeoIPISP.dat')
  365. except:
  366. self.return_error(500, 'The GeoIP-ISP database file could not be opened.')
  367. return
  368. isps = {}
  369. for key in vpn:
  370. if key is None: continue
  371. item = vpn[key]
  372. if not isinstance(item, dict): continue
  373. if not 'active' in item: continue
  374. ips = set()
  375. for gw in item['active']:
  376. if 'remote' in item['active'][gw]:
  377. ips.add(item['active'][gw]['remote'])
  378. if len(ips) == 0: continue
  379. if len(ips) > 1:
  380. self.logger.debug('VPN key \'{0}\' has {1} active ips. Possible cause is an in-progress re-dialin.'.format(key, len(ips)))
  381. item_isps = set()
  382. for ip in ips:
  383. try:
  384. isp = geo.org_by_addr(ip)
  385. if not isp is None: item_isps.add(isp)
  386. except Exception as err:
  387. self.logger.debug('Failed to resolve ISP for \'{0}\': {1}'.format(ip, str(err)))
  388. if len(item_isps) == 0:
  389. item_isps.add('unknown')
  390. elif len(item_isps) > 1:
  391. self.logger.warn('VPN key \'{0}\' has {1} active IPs which resolved to {2} ISPs: \'{3}\''.format(key, len(ips), len(item_isps), '\', \''.join(item_isps)))
  392. for isp in item_isps:
  393. if not isp in isps: isps[isp] = 0
  394. isps[isp] += 1.0 / len(item_isps)
  395. isps_sum = sum([isps[x] for x in isps])
  396. if outputformat == 'csv':
  397. self.send_headers('text/csv')
  398. self.wfile.write('Count;Name\n')
  399. for isp in isps:
  400. self.wfile.write('{0};"{1}"\n'.format(isps[isp], isp))
  401. elif outputformat == 'json':
  402. self.send_headers('text/json')
  403. data = [ { 'name': x, 'count': isps[x], 'percentage': isps[x]*100.0/isps_sum } for x in isps ]
  404. self.wfile.write(json.dumps(data))
  405. elif outputformat == 'html':
  406. self.send_headers()
  407. self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE - PROVIDERS</title></head><body>\n')
  408. self.wfile.write('<table border="2"><thead><tr><th>Count</th><th>Percentage</th><th>Name</th></tr></thead><tbody>\n')
  409. for isp in isps:
  410. self.wfile.write('<tr><td>{0}</td><td>{1:.1f}%</td><td>{2}</td></tr>\n'.format(isps[isp], isps[isp]*100.0/isps_sum, isp))
  411. self.wfile.write('</tbody></table>\n')
  412. self.wfile.write('<p>Totals: {0} ISPs, {1} connections</p>\n'.format(len(isps), isps_sum))
  413. self.wfile.write('</body></html>')
  414. else:
  415. self.send_error(400, 'Unknown output format.')
  416. class ApiServer(ThreadingMixIn, HTTPServer):
  417. def __init__(self, endpoint, storage):
  418. if ':' in endpoint[0]: self.address_family = socket.AF_INET6
  419. HTTPServer.__init__(self, endpoint, BatcaveHttpRequestHandler)
  420. self.storage = storage
  421. # check all entries for a proper 'remote' entry
  422. vpn = storage.data[BatcaveHttpRequestHandler.DATAKEY_VPN] if BatcaveHttpRequestHandler.DATAKEY_VPN in storage.data else {}
  423. init_vpn_cache = {}
  424. for key in vpn:
  425. if not isinstance(vpn[key], dict):
  426. continue
  427. for mode in vpn[key]:
  428. if not isinstance(vpn[key][mode], dict):
  429. continue
  430. for gw in vpn[key][mode]:
  431. if not isinstance(vpn[key][mode][gw], dict):
  432. continue
  433. item = vpn[key][mode][gw]
  434. if 'remote' in item and not 'remote_raw' in item:
  435. item['remote_raw'] = item['remote']
  436. resolved = None
  437. if item['remote'] in init_vpn_cache:
  438. resolved = init_vpn_cache[item['remote']]
  439. else:
  440. resolved = ffstatus.resolve_ipblock(item['remote'])
  441. init_vpn_cache[item['remote']] = resolved
  442. if not resolved is None:
  443. self.logger.info('Startup: resolved VPN entry \'{0}\' to net \'{1}\'.'.format(item['remote'], resolved['name']))
  444. if not resolved is None:
  445. item['remote'] = resolved
  446. storage.save()
  447. def __str__(self):
  448. return 'ApiServer on {0}'.format(self.server_address)
  449. if __name__ == '__main__':
  450. dummystorage = Storage()
  451. server = ApiServer(('0.0.0.0', 8888), dummystorage)
  452. print("Server:", str(server))
  453. server.serve_forever()