server.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607
  1. #!/usr/bin/python
  2. # -*- coding: utf-8 -*-
  3. from __future__ import print_function, unicode_literals
  4. from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
  5. import cgi
  6. import json
  7. import logging
  8. import re
  9. import socket
  10. from SocketServer import ThreadingMixIn
  11. import time
  12. import ffstatus
  13. # each match of these regex is removed to normalize an ISP's name
  14. ISP_NORMALIZATIONS = [
  15. # normalize name: strip company indication
  16. re.compile(r'(AG|UG|G?mbH( ?& ?Co\.? ?(OH|K)G)?)$', flags=re.IGNORECASE),
  17. # normalize name: strip "pool" suffixes
  18. re.compile(r'(dynamic )?(customer |subscriber )?(ip )?(pool|(address )?range|addresses)$', flags=re.IGNORECASE),
  19. # normalize name: strip "B2B" and aggregation suffixes
  20. re.compile(r'(aggregate|aggregation)?$', flags=re.IGNORECASE),
  21. re.compile(r'(B2B)?$', flags=re.IGNORECASE),
  22. # normalize name: strip country suffixes (in Germany)
  23. re.compile(r'(' +
  24. 'DE|Deutschland|Germany|' +
  25. 'Nordrhein[- ]Westfalen|NRW|' +
  26. 'Baden[- ]Wuerttemburg|BW|' +
  27. 'Hessen|' +
  28. 'Niedersachsen|' +
  29. 'Rheinland[- ]Pfalz|RLP' +
  30. ')$',
  31. flags=re.IGNORECASE),
  32. ]
  33. REGEX_QUERYPARAM = re.compile(
  34. r'(?P<key>.+?)=(?P<value>.+?)(&|$)')
  35. REGEX_URL_NODEINFO = re.compile(
  36. r'node/(?P<id>[a-fA-F0-9]{12})(?P<cmd>\.json|/[a-zA-Z0-9_\-\.]+)$')
  37. REGEX_URL_NODESTATUS = re.compile(
  38. r'status/([a-f0-9]{12})$')
  39. def normalize_ispname(isp):
  40. """Removes all matches on ISP_NORMALIZATIONS."""
  41. isp = isp.strip()
  42. for regex in ISP_NORMALIZATIONS:
  43. isp = regex.sub('', isp).strip()
  44. return isp
  45. class BatcaveHttpRequestHandler(BaseHTTPRequestHandler):
  46. """Handles a single HTTP request to the BATCAVE."""
  47. def __init__(self, request, client_address, sockserver):
  48. self.logger = logging.getLogger('API')
  49. BaseHTTPRequestHandler.__init__(
  50. self, request, client_address, sockserver)
  51. def __parse_url_pathquery(self):
  52. """Extracts the query parameters from the request path."""
  53. url = re.match(r'^/(?P<path>.*?)(\?(?P<query>.+))?$', self.path.strip())
  54. if url is None:
  55. logging.warn('Failed to parse URL \'' + str(self.path) + '\'.')
  56. return (None, None)
  57. path = url.group('path')
  58. query = {}
  59. if not url.group('query') is None:
  60. for match in REGEX_QUERYPARAM.finditer(url.group('query')):
  61. query[match.group('key')] = match.group('value')
  62. return (path, query)
  63. def do_GET(self):
  64. """Handles all HTTP GET requests."""
  65. path, query = self.__parse_url_pathquery()
  66. if path is None:
  67. self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
  68. return
  69. # / - index page, shows generic help
  70. if path == '':
  71. self.__respond_index(query)
  72. return
  73. # /nodes.json
  74. if path == 'nodes.json':
  75. self.__respond_nodes(query)
  76. return
  77. # /list - list stored nodes
  78. if path == 'list':
  79. self.__respond_list(query)
  80. return
  81. # /vpn - notification endpoint for gateway's VPN connections
  82. if path == 'vpn':
  83. self.__respond_vpn(query)
  84. return
  85. # /providers
  86. if path == 'providers':
  87. self.__respond_providers(query)
  88. return
  89. # /node/<id>.json - node's data
  90. # /node/<id>/field - return specific field from node's data
  91. match = REGEX_URL_NODEINFO.match(path)
  92. if match is not None:
  93. cmd = match.group('cmd')
  94. nodeid = match.group('id').lower()
  95. if cmd == '.json':
  96. self.__respond_node(nodeid)
  97. else:
  98. self.__respond_nodedetail(nodeid, cmd[1:])
  99. return
  100. # /status - overall status (incl. node and client count)
  101. if path == 'status':
  102. self.__respond_status()
  103. return
  104. # /status/<id> - node's status
  105. match = REGEX_URL_NODESTATUS.match(path)
  106. if match is not None:
  107. self.__respond_nodestatus(match.group(1))
  108. return
  109. # no match -> 404
  110. self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))
  111. def do_POST(self):
  112. """Handles all HTTP POST requests."""
  113. path, query = self.__parse_url_pathquery()
  114. if path is None:
  115. self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
  116. return
  117. params = self.__parse_post_params()
  118. # node id/mac to name mapping
  119. if path == 'idmac2name':
  120. self.__respond_nodeidmac2name(params)
  121. return
  122. # no match -> 404
  123. self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))
  124. def __send_nocache_headers(self):
  125. """
  126. Sets HTTP headers indicating that this response shall not be cached.
  127. """
  128. self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
  129. self.send_header('Pragma', 'no-cache')
  130. self.send_header('Expires', '0')
  131. def __send_headers(self,
  132. content_type='text/html; charset=utf-8',
  133. nocache=True, extra={}):
  134. """Send HTTP 200 Response header with the given Content-Type.
  135. Optionally send no-caching headers, too."""
  136. self.send_response(200)
  137. self.send_header('Content-Type', content_type)
  138. if nocache:
  139. self.__send_nocache_headers()
  140. for key in extra:
  141. self.send_header(key, extra[key])
  142. self.end_headers()
  143. def __parse_post_params(self):
  144. ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
  145. if ctype == 'multipart/form-data':
  146. postvars = cgi.parse_multipart(self.rfile, pdict)
  147. elif ctype == 'application/x-www-form-urlencoded':
  148. length = int(self.headers.getheader('content-length'))
  149. postvars = cgi.parse_qs(
  150. self.rfile.read(length),
  151. keep_blank_values=1,
  152. )
  153. else:
  154. postvars = {}
  155. return postvars
  156. def __respond_index(self, query):
  157. """Display the index page."""
  158. self.__send_headers()
  159. index_page = '''<!DOCTYPE html>
  160. <html><head><title>BATCAVE</title></head>
  161. <body>
  162. <H1 title="Batman/Alfred Transmission Collection, Aggregation & Value Engine">
  163. BATCAVE
  164. </H1>
  165. <p>Dies ist ein interner Hintergrund-Dienst. Er wird nur von anderen Diensten
  166. angesprochen und sollte aus einer Mehrzahl von Gr&uuml;nden nicht
  167. &ouml;ffentlich zug&auml;nglich sein.</p>
  168. <H2>API</H2>
  169. <p>
  170. Grunds&auml;tzlich ist das Antwort-Format JSON und alle Daten sind
  171. Live-Daten (kein Cache) die ggf. etwas Bearbeitungs-Zeit erfordern.
  172. </p>
  173. <dl>
  174. <dt>GET <a href="/nodes.json">nodes.json</a></dt>
  175. <dd>zur Verwendung mit ffmap (MACs anonymisiert)</dd>
  176. <dt>GET /node/&lt;id&gt;.json</dt>
  177. <dd>alle Daten zu dem gew&uuml;nschten Knoten</dd>
  178. <dt>GET /providers?format=json</dt>
  179. <dd>Liste der Provider</dd>
  180. <dt>GET <a href="/status">/status</a></dt>
  181. <dd>Status der BATCAVE inkl. Zahl der Nodes+Clients (JSON)</dd>
  182. <dt>GET /status/&lt;id&gt;</dt>
  183. <dd>Status des Knotens</dd>
  184. </dl>
  185. </body></html>'''
  186. self.wfile.write(index_page)
  187. def __respond_list(self, query):
  188. """List stored data."""
  189. self.__send_headers()
  190. self.wfile.write('<!DOCTYPE html><html>\n')
  191. self.wfile.write('<head><title>BATCAVE</title></head>\n')
  192. self.wfile.write('<body>\n')
  193. self.wfile.write('<H1>BATCAVE - LIST</H1>\n')
  194. self.wfile.write('<table>\n')
  195. self.wfile.write('<thead><tr><th>ID</th><th>Name</th></tr></thead>\n')
  196. self.wfile.write('<tbody>\n')
  197. sortkey = query['sort'] if 'sort' in query else None
  198. data = self.server.storage.get_nodes(sortby=sortkey)
  199. for node in data:
  200. nodeid = node.get('node_id')
  201. nodename = node.get('hostname', '&lt;?&gt;')
  202. self.wfile.write('<tr>\n')
  203. self.wfile.write(' <td><a href="/node/{0}.json">{0}</a></td>\n'.format(nodeid))
  204. self.wfile.write(' <td>{0}</td>\n'.format(nodename))
  205. self.wfile.write('</tr>\n')
  206. self.wfile.write('</tbody>\n')
  207. self.wfile.write('</table>\n')
  208. def __map_item(self, haystack, needle, prefix=None):
  209. if not isinstance(haystack, dict):
  210. raise Exception("haystack must be a dict")
  211. if needle in haystack:
  212. return haystack[needle]
  213. idx = len(haystack) + 1
  214. name = prefix + str(idx)
  215. while name in haystack:
  216. idx += 1
  217. name = prefix + str(idx)
  218. haystack[needle] = name
  219. return name
  220. def __respond_nodes(self, query):
  221. indent = 2 if query.get('pretty', 0) == '1' else None
  222. nodes = []
  223. clientmapping = {}
  224. for node in self.server.storage.get_nodes():
  225. entry = {
  226. 'id': node.get('node_id'),
  227. 'name': node.get('hostname'),
  228. 'clients': [self.__map_item(clientmapping, x, "c")
  229. for x in node.get('clients', [])],
  230. }
  231. geo = node.get('location', None)
  232. if geo is not None:
  233. entry['geo'] = [geo['latitude'], geo['longitude']]
  234. nodes.append(entry)
  235. result = {'nodes': nodes}
  236. self.__send_headers(content_type='application/json',
  237. extra={'Content-Disposition': 'inline'})
  238. self.wfile.write(json.dumps(result, indent=indent))
  239. def __respond_node(self, rawid):
  240. """Display node data."""
  241. # search node by the given id
  242. node = self.server.storage.find_node(rawid)
  243. # handle unknown nodes
  244. if node is None:
  245. self.send_error(404, 'No node with id \'' + rawid + '\' present.')
  246. return
  247. # dump node data as JSON
  248. self.__send_headers('application/json',
  249. extra={'Content-Disposition': 'inline'})
  250. self.wfile.write(json.dumps(node))
  251. def __respond_nodestatus(self, rawid):
  252. """Display node status."""
  253. status = self.server.storage.get_nodestatus(rawid)
  254. if status is None:
  255. self.send_error(404, 'No node with id \'' + rawid + '\' present.')
  256. self.__send_headers('text/plain')
  257. self.wfile.write(status)
  258. def __respond_nodeidmac2name(self, ids):
  259. """Return a mapping of the given IDs (or MACs) into their hostname."""
  260. self.__send_headers('text/plain')
  261. for nodeid in ids:
  262. node = None
  263. if not ':' in nodeid:
  264. node = self.server.storage.find_node(nodeid)
  265. else:
  266. node = self.server.storage.find_node_by_mac(nodeid)
  267. nodename = node.get('hostname', nodeid) if node is not None else nodeid
  268. self.wfile.write('{0}={1}\n'.format(nodeid, nodename))
  269. def __respond_nodedetail(self, nodeid, field):
  270. """
  271. Return a field from the given node.
  272. String and integers are returned as text/plain,
  273. all other as JSON.
  274. """
  275. node = self.server.storage.find_node(nodeid)
  276. if node is None:
  277. self.send_error(404, 'No node with id \'' + nodeid + '\' present.')
  278. return
  279. return_count = False
  280. if field.endswith('.count'):
  281. return_count = True
  282. field = field[0:-6]
  283. if not field in node:
  284. self.send_error(
  285. 404,
  286. 'The node \'{0}\' does not have a field named \'{1}\'.'.format(
  287. nodeid, field
  288. )
  289. )
  290. return
  291. value = node[field]
  292. if return_count:
  293. value = len(value)
  294. no_json = isinstance(value, basestring) or isinstance(value, int)
  295. self.__send_headers('text/plain' if no_json else 'application/json',
  296. extra={'Content-Disposition': 'inline'})
  297. self.wfile.write(value if no_json else json.dumps(value))
  298. def __respond_status(self):
  299. status = self.server.storage.status
  300. self.__send_headers('application/json',
  301. extra={'Content-Disposition': 'inline'})
  302. self.wfile.write(json.dumps(status, indent=2))
  303. def __respond_vpn(self, query):
  304. storage = self.server.storage
  305. peername = query.get('peer')
  306. key = query.get('key')
  307. action = query.get('action')
  308. remote = query.get('remote')
  309. gateway = query.get('gw')
  310. timestamp = query.get('ts', time.time())
  311. if action == 'list':
  312. self.__respond_vpnlist()
  313. return
  314. if action != 'establish' and action != 'disestablish':
  315. self.logger.error('VPN: unknown action \'{0}\''.format(action))
  316. self.send_error(400, 'Invalid action.')
  317. return
  318. check = {
  319. 'peername': peername,
  320. 'key': key,
  321. 'remote': remote,
  322. 'gw': gateway,
  323. }
  324. for k, val in check.items():
  325. if val is None or len(val.strip()) == 0:
  326. self.logger.error('VPN {0}: no or empty {1}'.format(action, k))
  327. self.send_error(400, 'Missing value for ' + str(k))
  328. return
  329. try:
  330. if action == 'establish':
  331. self.server.storage.log_vpn_connect(
  332. key, peername, remote, gateway, timestamp)
  333. elif action == 'disestablish':
  334. self.server.storage.log_vpn_disconnect(key, gateway, timestamp)
  335. else:
  336. self.logger.error('Unknown VPN action \'%s\' not filtered.',
  337. action)
  338. self.send_error(500)
  339. return
  340. except ffstatus.exceptions.VpnKeyFormatError:
  341. self.logger.error('VPN peer \'{0}\' {1}: bad key \'{2}\''.format(
  342. peername, action, key,
  343. ))
  344. self.send_error(400, 'Bad key.')
  345. return
  346. self.__send_headers('text/plain')
  347. self.wfile.write('OK')
  348. storage.save()
  349. def __respond_vpnlist(self):
  350. self.__send_headers()
  351. self.wfile.write('''<!DOCTYPE html>
  352. <html><head><title>BATCAVE - VPN LIST</title></head>
  353. <body>
  354. <style type="text/css">
  355. table { border: 2px solid #999; border-collapse: collapse; }
  356. th, td { border: 1px solid #CCC; }
  357. table tbody tr.online { background-color: #CFC; }
  358. table tbody tr.offline { background-color: #FCC; }
  359. </style>
  360. <table>''')
  361. gateways = self.server.storage.get_vpn_gateways()
  362. gws_header = '<th>' + '</th><th>'.join(gateways) + '</th>'
  363. self.wfile.write('<thead>\n')
  364. self.wfile.write('<tr><th rowspan="2">names (key)</th>')
  365. self.wfile.write('<th colspan="' + str(len(gateways)) + '">active</th>')
  366. self.wfile.write('<th colspan="' + str(len(gateways)) + '">last</th>')
  367. self.wfile.write('</tr>\n')
  368. self.wfile.write('<tr>' + gws_header + gws_header + '</tr>\n')
  369. self.wfile.write('</thead>\n')
  370. for item in self.server.storage.get_vpn_connections():
  371. row_class = 'online' if item['online'] else 'offline'
  372. self.wfile.write('<tr class="{0}">'.format(row_class))
  373. self.wfile.write('<td title="{0}">{1}</td>'.format(
  374. item['key'],
  375. ' / '.join(item['names']) if len(item['names']) > 0 else '?',
  376. ))
  377. for conntype in ['active', 'last']:
  378. for gateway in gateways:
  379. remote = ''
  380. if conntype in item['remote'] and \
  381. gateway in item['remote'][conntype]:
  382. remote = item['remote'][conntype][gateway]
  383. if isinstance(remote, dict):
  384. remote = remote['name']
  385. symbol = '&check;' if len(remote) > 0 else '&times;'
  386. self.wfile.write('<td title="{0}">{1}</td>'.format(
  387. remote, symbol))
  388. self.wfile.write('</tr>\n')
  389. self.wfile.write('</table>\n')
  390. self.wfile.write('</body>')
  391. self.wfile.write('</html>')
  392. def __respond_providers(self, query):
  393. """Return a summary of providers."""
  394. outputformat = query['format'].lower() if 'format' in query else 'html'
  395. isps = {}
  396. ispblocks = {}
  397. for item in self.server.storage.get_vpn_connections():
  398. if item['count']['active'] == 0:
  399. continue
  400. remotes = []
  401. for gateway in item['remote']['active']:
  402. remote = item['remote']['active'][gateway]
  403. remotes.append(remote)
  404. if len(remotes) == 0:
  405. self.logger.warn(
  406. 'VPN key \'%s\' is marked with active remotes but 0 found?',
  407. item['key'])
  408. continue
  409. item_isps = set()
  410. for remote in remotes:
  411. isp = "UNKNOWN"
  412. ispblock = remote
  413. if isinstance(remote, dict):
  414. ispblock = remote['name']
  415. desc_lines = remote['description'].split('\n')
  416. isp = normalize_ispname(desc_lines[0])
  417. if not isp in ispblocks:
  418. ispblocks[isp] = set()
  419. ispblocks[isp].add(ispblock)
  420. item_isps.add(isp)
  421. if len(item_isps) == 0:
  422. item_isps.add('unknown')
  423. elif len(item_isps) > 1:
  424. self.logger.warn(
  425. 'VPN key \'%s\' has %d active IPs ' +
  426. 'which resolved to %d ISPs: \'%s\'',
  427. item['key'],
  428. len(remotes),
  429. len(item_isps),
  430. '\', \''.join(item_isps)
  431. )
  432. for isp in item_isps:
  433. if not isp in isps:
  434. isps[isp] = 0
  435. isps[isp] += 1.0 / len(item_isps)
  436. isps_sum = sum([isps[x] for x in isps])
  437. if outputformat == 'csv':
  438. self.__send_headers('text/csv')
  439. self.wfile.write('Count;Name\n')
  440. for isp in isps:
  441. self.wfile.write('{0};"{1}"\n'.format(isps[isp], isp))
  442. elif outputformat == 'json':
  443. self.__send_headers('application/json',
  444. extra={'Content-Disposition': 'inline'})
  445. data = [
  446. {
  447. 'name': isp,
  448. 'count': isps[isp],
  449. 'percentage': isps[isp]*100.0/isps_sum,
  450. 'blocks': [block for block in ispblocks[isp]],
  451. } for isp in isps
  452. ]
  453. self.wfile.write(json.dumps(data))
  454. elif outputformat == 'html':
  455. self.__send_headers()
  456. self.wfile.write('''<!DOCTYPE html>
  457. <html>
  458. <head><title>BATCAVE - PROVIDERS</title></head>
  459. <body>
  460. <table border="2">
  461. <thead>
  462. <tr><th>Count</th><th>Percentage</th><th>Name</th><th>Blocks</th></tr>
  463. </thead>
  464. <tbody>\n''')
  465. for isp in sorted(isps, key=lambda x: isps[x], reverse=True):
  466. self.wfile.write('<tr><td>{0}</td><td>{1:.1f}%</td><td>{2}</td><td>{3}</td></tr>\n'.format(
  467. isps[isp],
  468. isps[isp]*100.0/isps_sum,
  469. isp,
  470. ', '.join(sorted(ispblocks[isp])) if isp in ispblocks else '?',
  471. ))
  472. self.wfile.write('</tbody></table>\n')
  473. self.wfile.write('<p>Totals: {0} ISPs, {1} connections</p>\n'.format(len(isps), isps_sum))
  474. self.wfile.write('</body></html>')
  475. else:
  476. self.send_error(400, 'Unknown output format.')
  477. class ApiServer(ThreadingMixIn, HTTPServer):
  478. def __init__(self, endpoint, storage):
  479. if ':' in endpoint[0]:
  480. self.address_family = socket.AF_INET6
  481. HTTPServer.__init__(self, endpoint, BatcaveHttpRequestHandler)
  482. self.storage = storage
  483. def __str__(self):
  484. return 'ApiServer on {0}'.format(self.server_address)
  485. if __name__ == '__main__':
  486. dummystorage = ffstatus.basestorage.BaseStorage()
  487. server = ApiServer(('0.0.0.0', 8888), dummystorage)
  488. print("Server:", str(server))
  489. server.serve_forever()