server.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466
  1. #!/usr/bin/python
  2. # -*- coding: utf-8 -*-
  3. from __future__ import print_function
  4. from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
  5. import cgi
  6. from storage import Storage
  7. import json
  8. import logging
  9. import pygeoip
  10. import re
  11. import socket
  12. from SocketServer import ThreadingMixIn
  13. import time
  14. class BatcaveHttpRequestHandler(BaseHTTPRequestHandler):
  15. DATAKEY_VPN = '__VPN__'
  16. def __init__(self, request, client_address, server):
  17. self.logger = logging.getLogger('API')
  18. BaseHTTPRequestHandler.__init__(self, request, client_address, server)
  19. def parse_url_pathquery(self):
  20. """Extracts the query parameters from the request path."""
  21. url = re.match(r'^/(?P<path>.*?)(\?(?P<query>.+))?$', self.path.strip())
  22. if url is None:
  23. logging.warn('Failed to parse URL \'' + str(self.path) + '\'.')
  24. return ( None, None )
  25. path = url.group('path')
  26. query = {}
  27. if not url.group('query') is None:
  28. for m in re.finditer(r'(?P<key>.+?)=(?P<value>.+?)(&|$)', url.group('query')):
  29. query[m.group('key')] = m.group('value')
  30. return ( path, query )
  31. def do_GET(self):
  32. """Handles all HTTP GET requests."""
  33. path, query = self.parse_url_pathquery()
  34. if path is None:
  35. self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
  36. return
  37. # / - index page, shows generic help
  38. if path == '':
  39. self.respond_index(query)
  40. return
  41. # /list - list stored nodes
  42. if path == 'list':
  43. self.respond_list(query)
  44. return
  45. # /vpn - notification endpoint for gateway's VPN connections
  46. if path == 'vpn':
  47. self.respond_vpn(query)
  48. return
  49. # /providers
  50. if path == 'providers':
  51. self.respond_providers(query)
  52. return
  53. # /node/<id>.json - node's data
  54. # /node/<id>/field - return specific field from node's data
  55. m = re.match(r'node/([a-f0-9]{12})(?P<cmd>\.json|/[a-zA-Z0-9_\-]+)$', path)
  56. if m != None:
  57. cmd = m.group('cmd')
  58. if cmd == '.json':
  59. self.respond_node(m.group(1))
  60. else:
  61. self.respond_nodedetail(m.group(1), cmd[1:])
  62. return
  63. # no match -> 404
  64. self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))
  65. def do_POST(self):
  66. """Handles all HTTP POST requests."""
  67. path, query = self.parse_url_pathquery()
  68. if path is None:
  69. self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
  70. return
  71. params = self.parse_post_params()
  72. # node id/mac to name mapping
  73. if path == 'idmac2name':
  74. self.respond_nodeidmac2name(params)
  75. return
  76. # no match -> 404
  77. self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))
  78. def send_nocache_headers(self):
  79. """Sets HTTP headers indicating that this response shall not be cached."""
  80. self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
  81. self.send_header('Pragma', 'no-cache')
  82. self.send_header('Expires', '0')
  83. def send_headers(self, content_type='text/html; charset=utf-8', nocache=True):
  84. """Send HTTP 200 Response header with the given Content-Type.
  85. Optionally send no-caching headers, too."""
  86. self.send_response(200)
  87. self.send_header('Content-Type', content_type)
  88. if nocache: self.send_nocache_headers()
  89. self.end_headers()
  90. def parse_post_params(self):
  91. ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
  92. if ctype == 'multipart/form-data':
  93. postvars = cgi.parse_multipart(self.rfile, pdict)
  94. elif ctype == 'application/x-www-form-urlencoded':
  95. length = int(self.headers.getheader('content-length'))
  96. postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1)
  97. else:
  98. postvars = {}
  99. return postvars
  100. def respond_index(self, query):
  101. """Display the index page."""
  102. storage = self.server.storage
  103. self.send_headers()
  104. self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE</title></head>\n')
  105. self.wfile.write('<body>\n')
  106. self.wfile.write('<H1 title="Batman/Alfred Transmission Collection, Aggregation & Value Engine">BATCAVE</H1>\n')
  107. self.wfile.write('<p>Dies ist ein interner Hintergrund-Dienst. Er wird nur von anderen Diensten\n')
  108. self.wfile.write('angesprochen und sollte aus einer Mehrzahl von Gr&uuml;nden nicht &ouml;ffentlich\n')
  109. self.wfile.write('zug&auml;nglich sein.</p>\n')
  110. self.wfile.write('<H2>Status</H2>\n')
  111. self.wfile.write('Daten: <span id="datacount" class="value">')
  112. self.wfile.write(len(storage.data))
  113. self.wfile.write('</span>\n')
  114. self.wfile.write('<H2>API</H2>\n')
  115. self.wfile.write('<p>Grundsätzlich ist das Antwort-Format JSON und alle Daten sind Live-Daten (kein Cache) die ggf. etwas Bearbeitungs-Zeit erfordern.</p>')
  116. self.wfile.write('<dl>\n')
  117. self.wfile.write('<dt><a href="/nodes.json">nodes.json</a></dt><dd>zur Verwendung mit ffmap (MACs anonymisiert)</dd>\n')
  118. self.wfile.write('<dt><a href="/node/ff00ff00ff00.json">/node/&lt;id&gt;.json</a></dt><dd><u>alle</u> vorhandenen Information zu der gewünschten Node</dd>\n')
  119. self.wfile.write('</dl>\n')
  120. self.wfile.write('</body></html>')
  121. def respond_list(self, query):
  122. """List stored data."""
  123. storage = self.server.storage
  124. self.send_headers()
  125. self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE</title></head>\n')
  126. self.wfile.write('<body>\n')
  127. self.wfile.write('<H1>BATCAVE - LIST</H1>\n')
  128. self.wfile.write('<table>\n')
  129. self.wfile.write('<thead><tr><th>ID</th><th>Name</th></tr></thead>\n')
  130. self.wfile.write('<tbody>\n')
  131. data = storage.data
  132. if 'sort' in query:
  133. if query['sort'] == 'name':
  134. sorteddata = sorted(data, key=lambda x: data[x]['hostname'].lower())
  135. data = sorteddata
  136. elif query['sort'] == 'id':
  137. sorteddata = sorted(data)
  138. data = sorteddata
  139. for nodeid in data:
  140. if nodeid.startswith('__'): continue
  141. nodename = storage.data[nodeid]['hostname'] if 'hostname' in storage.data[nodeid] else '&lt;?&gt;'
  142. self.wfile.write('<tr><td><a href="/node/' + nodeid + '.json">' + nodeid + '</a></td><td>' + nodename + '</td></tr>')
  143. self.wfile.write('</tbody>\n')
  144. self.wfile.write('</table>\n')
  145. def find_node(self, rawid):
  146. """Fetch node data from storage by given id, if necessary looking thorugh node aliases."""
  147. storage = self.server.storage
  148. # if we have a direct hit, return it immediately
  149. if rawid in storage.data:
  150. return storage.data[rawid]
  151. # no direct hit -> search via aliases
  152. nodeid = rawid
  153. for n in storage.data:
  154. if 'aliases' in storage.data[n] and rawid in storage.data[n]['aliases']:
  155. nodeid = n
  156. # return found node
  157. return storage.data[nodeid] if nodeid in storage.data else None
  158. def find_node_by_mac(self, mac):
  159. """Fetch node data from storage by given MAC address."""
  160. storage = self.server.storage
  161. needle = mac.lower()
  162. # iterate over all nodes
  163. for nodeid in storage.data:
  164. if nodeid.startswith('__'): continue
  165. node = storage.data[nodeid]
  166. # check node's primary MAC
  167. if 'mac' in node and needle == node['mac'].lower():
  168. return node
  169. # check alias MACs
  170. if 'macs' in node:
  171. haystack = [ x.lower() for x in node['macs'] ]
  172. if mac in haystack:
  173. return node
  174. # MAC address not found
  175. return None
  176. def respond_node(self, rawid):
  177. """Display node data."""
  178. # handle API example linked on index page
  179. if rawid == 'ff00ff00ff00':
  180. self.send_headers('text/json')
  181. self.wfile.write(json.dumps({
  182. 'name': 'API-Example',
  183. 'nodeid': rawid,
  184. 'META': 'Dies ist ein minimaler Beispiel-Datensatz. Herzlichen Glückwunsch, du hast das Prinzip der API kapiert.',
  185. }))
  186. return
  187. # search node by the given id
  188. node = self.find_node(rawid)
  189. # handle unknown nodes
  190. if node is None:
  191. self.send_error(404, 'No node with id \'' + rawid + '\' present.')
  192. return
  193. # dump node data as JSON
  194. self.send_headers('text/json')
  195. self.wfile.write(json.dumps(node))
  196. def respond_nodeidmac2name(self, ids):
  197. storage = self.server.storage
  198. self.send_headers('text/plain')
  199. for nodeid in ids:
  200. node = self.find_node(nodeid) if not ':' in nodeid else self.find_node_by_mac(nodeid)
  201. nodename = node['hostname'] if (not node is None) and 'hostname' in node else nodeid
  202. self.wfile.write('{0}={1}\n'.format(nodeid, nodename))
  203. def respond_nodedetail(self, nodeid, field):
  204. """Return a field from the given node - a string is returned as text, all other as JSON."""
  205. node = self.find_node(nodeid)
  206. if node is None:
  207. self.send_error(404, 'No node with id \'' + nodeid + '\' present.')
  208. return
  209. if not field in node:
  210. self.send_error(404, 'The node \'' + nodeid + '\' does not have a field named \'' + str(field) + '\'.')
  211. return
  212. value = node[field]
  213. self.send_headers('text/plain' if isinstance(value, basestring) else 'text/json')
  214. self.wfile.write(value if isinstance(value, basestring) else json.dumps(value))
  215. def respond_vpn(self, query):
  216. storage = self.server.storage
  217. peername = query['peer'] if 'peer' in query else None
  218. key = query['key'] if 'key' in query else None
  219. action = query['action'] if 'action' in query else None
  220. remote = query['remote'] if 'remote' in query else None
  221. gw = query['gw'] if 'gw' in query else None
  222. if action == 'list':
  223. self.respond_vpnlist()
  224. return
  225. if action != 'establish' and action != 'disestablish':
  226. self.logger.error('VPN: unknown action \'{0}\''.format(action))
  227. self.send_error(400, 'Invalid action.')
  228. return
  229. for k,v in { 'peername': peername, 'key': key, 'remote': remote, 'gw': gw }.items():
  230. if v is None or len(v.strip()) == 0:
  231. self.logger.error('VPN {0}: no or empty {1}'.format(action, k))
  232. self.send_error(400, 'Missing value for ' + str(k))
  233. return
  234. if key is None or re.match(r'^[a-fA-F0-9]+$', key) is None:
  235. self.logger.error('VPN peer \'{0}\' {1}: bad key \'{2}\''.format(peername, action, key))
  236. self.send_error(400, 'Bad key.')
  237. return
  238. if not self.DATAKEY_VPN in storage.data: storage.data[self.DATAKEY_VPN] = {}
  239. if not key in storage.data[self.DATAKEY_VPN]: storage.data[self.DATAKEY_VPN][key] = { 'active': {}, 'last': {} }
  240. item = storage.data[self.DATAKEY_VPN][key]
  241. if action == 'establish':
  242. item['active'][gw] = { 'establish': time.time(), 'peer': peername, 'remote': remote }
  243. elif action == 'disestablish':
  244. active = {}
  245. if gw in item['active']:
  246. active = item['active'][gw]
  247. del(item['active'][gw])
  248. active['disestablish'] = time.time()
  249. item['last'][gw] = active
  250. else:
  251. self.send_error(500, 'Unknown action not filtered (' + str(action) + ')')
  252. return
  253. self.send_headers('text/plain')
  254. self.wfile.write('OK')
  255. storage.save()
  256. def respond_vpnlist(self):
  257. storage = self.server.storage
  258. gateways = ['gw01','gw02','gw03','gw04']
  259. self.send_headers()
  260. self.wfile.write('<!DOCTYPE html>\n')
  261. self.wfile.write('<html><head><title>BATCAVE - VPN LIST</title></head>\n')
  262. self.wfile.write('<body>\n')
  263. self.wfile.write('<style type="text/css">\n')
  264. self.wfile.write('table { border: 2px solid #999; border-collapse: collapse; }\n')
  265. self.wfile.write('th, td { border: 1px solid #CCC; }\n')
  266. self.wfile.write('table tbody tr.online { background-color: #CFC; }\n')
  267. self.wfile.write('table tbody tr.offline { background-color: #FCC; }\n')
  268. self.wfile.write('</style>\n')
  269. self.wfile.write('<table>\n<thead>\n')
  270. self.wfile.write('<tr><th rowspan="2">names (key)</th><th colspan="' + str(len(gateways)) + '">active</th><th colspan="' + str(len(gateways)) + '">last</th></tr>\n')
  271. self.wfile.write('<tr><th>' + '</th><th>'.join(gateways) + '</th><th>' + '</th><th>'.join(gateways) + '</th></tr>\n')
  272. self.wfile.write('</thead>\n')
  273. if self.DATAKEY_VPN in storage.data:
  274. for key in storage.data[self.DATAKEY_VPN]:
  275. item = storage.data[self.DATAKEY_VPN][key]
  276. names = set()
  277. count = {}
  278. for t in [ 'active', 'last' ]:
  279. count[t] = 0
  280. if t in item:
  281. for gw in item[t]:
  282. if 'remote' in item[t][gw] and len(item[t][gw]['remote']) > 0:
  283. count[t] += 1
  284. if 'peer' in item[t][gw]:
  285. names.add(item[t][gw]['peer'])
  286. self.wfile.write('<tr class="online">' if count['active'] > 0 else '<tr class="offline">')
  287. self.wfile.write('<td title="' + str(key) + '">' + (' / '.join(names) if len(names) > 0 else '?') + '</td>')
  288. for t in [ 'active', 'last' ]:
  289. for gw in gateways:
  290. ip = ''
  291. details = ''
  292. if t in item and gw in item[t]:
  293. ip = item[t][gw]['remote'] if 'remote' in item[t][gw] else ''
  294. self.wfile.write('<td>' + ip + '</td>')
  295. self.wfile.write('</tr>\n')
  296. self.wfile.write('</table>\n')
  297. self.wfile.write('</body>')
  298. self.wfile.write('</html>')
  299. def respond_providers(self, query):
  300. """Return a summary of providers."""
  301. vpn = self.server.storage.data[self.DATAKEY_VPN]
  302. outputformat = query['format'].lower() if 'format' in query else 'html'
  303. geo = None
  304. try:
  305. geo = pygeoip.GeoIP('GeoIPISP.dat')
  306. except:
  307. self.return_error(500, 'The GeoIP-ISP database file could not be opened.')
  308. return
  309. isps = {}
  310. for key in vpn:
  311. if key is None: continue
  312. item = vpn[key]
  313. if not 'active' in item: continue
  314. ips = set()
  315. for gw in item['active']:
  316. if 'remote' in item['active'][gw]:
  317. ips.add(item['active'][gw]['remote'])
  318. if len(ips) == 0: continue
  319. if len(ips) > 1:
  320. self.logger.warn('VPN key \'{0}\' has {1} active ips. Possible cause is an in-progress re-dialin.'.format(key, len(ips)))
  321. item_isps = set()
  322. for ip in ips:
  323. try:
  324. isp = geo.org_by_addr(ip)
  325. if not isp is None: item_isps.add(isp)
  326. except Exception as err:
  327. self.logger.debug('Failed to resolve ISP for \'{0}\': {1}'.format(ip, str(err)))
  328. if len(item_isps) == 0:
  329. item_isps.add('unknown')
  330. elif len(item_isps) > 1:
  331. self.logger.warn('VPN key \'{0}\' has {1} active IPs which resolved to {2} ISPs: \'{3}\''.format(key, len(ips), len(item_isps), '\', \''.join(item_isps)))
  332. for isp in item_isps:
  333. if not isp in isps: isps[isp] = 0
  334. isps[isp] += 1
  335. isps_sum = sum([isps[x] for x in isps])
  336. if outputformat == 'csv':
  337. self.send_headers('text/csv')
  338. self.wfile.write('Count;Name\n')
  339. for isp in isps:
  340. self.wfile.write('{0};"{1}"\n'.format(isps[isp], isp))
  341. elif outputformat == 'json':
  342. self.send_headers('text/json')
  343. data = [ { 'name': x, 'count': isps[x], 'percentage': isps[x]*100.0/isps_sum } for x in isps ]
  344. self.wfile.write(json.dumps(data))
  345. elif outputformat == 'html':
  346. self.send_headers()
  347. self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE - PROVIDERS</title></head><body>\n')
  348. self.wfile.write('<table border="2"><thead><tr><th>Count</th><th>Percentage</th><th>Name</th></tr></thead><tbody>\n')
  349. for isp in isps:
  350. self.wfile.write('<tr><td>{0}</td><td>{1:.1f}%</td><td>{2}</td></tr>\n'.format(isps[isp], isps[isp]*100.0/isps_sum, isp))
  351. self.wfile.write('</tbody></table>\n')
  352. self.wfile.write('</body></html>')
  353. else:
  354. self.send_error(400, 'Unknown output format.')
  355. class ApiServer(ThreadingMixIn, HTTPServer):
  356. def __init__(self, endpoint, storage):
  357. if ':' in endpoint[0]: self.address_family = socket.AF_INET6
  358. HTTPServer.__init__(self, endpoint, BatcaveHttpRequestHandler)
  359. self.storage = storage
  360. def __str__(self):
  361. return 'ApiServer on {0}'.format(self.server_address)
  362. if __name__ == '__main__':
  363. dummystorage = Storage()
  364. server = ApiServer(('0.0.0.0', 8888), dummystorage)
  365. print("Server:", str(server))
  366. server.serve_forever()