server.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567
  1. #!/usr/bin/python
  2. # -*- coding: utf-8 -*-
  3. from __future__ import print_function, unicode_literals
  4. from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
  5. import cgi
  6. import json
  7. import logging
  8. import re
  9. import socket
  10. from SocketServer import ThreadingMixIn
  11. import time
  12. import ffstatus
  13. # each match of these regex is removed to normalize an ISP's name
  14. ISP_NORMALIZATIONS = [
  15. # normalize name: strip company indication
  16. re.compile(r'(AG|UG|G?mbH( ?& ?Co\.? ?(OH|K)G)?)$', flags=re.IGNORECASE),
  17. # normalize name: strip "pool" suffixes
  18. re.compile(r'(dynamic )?(customer |subscriber )?(ip )?(pool|(address )?range|addresses)$', flags=re.IGNORECASE),
  19. # normalize name: strip "B2B" and aggregation suffixes
  20. re.compile(r'(aggregate|aggregation)?$', flags=re.IGNORECASE),
  21. re.compile(r'(B2B)?$', flags=re.IGNORECASE),
  22. # normalize name: strip country suffixes (in Germany)
  23. re.compile(r'(' +
  24. 'DE|Deutschland|Germany|' +
  25. 'Nordrhein[- ]Westfalen|NRW|' +
  26. 'Baden[- ]Wuerttemburg|BW|' +
  27. 'Hessen|' +
  28. 'Niedersachsen|' +
  29. 'Rheinland[- ]Pfalz|RLP' +
  30. ')$',
  31. flags=re.IGNORECASE),
  32. ]
  33. REGEX_QUERYPARAM = re.compile(
  34. r'(?P<key>.+?)=(?P<value>.+?)(&|$)')
  35. REGEX_URL_NODEINFO = re.compile(
  36. r'node/(?P<id>[a-fA-F0-9]{12})(?P<cmd>\.json|/[a-zA-Z0-9_\-\.]+)$')
  37. REGEX_URL_NODESTATUS = re.compile(
  38. r'status/([a-f0-9]{12})$')
  39. def normalize_ispname(isp):
  40. """Removes all matches on ISP_NORMALIZATIONS."""
  41. isp = isp.strip()
  42. for regex in ISP_NORMALIZATIONS:
  43. isp = regex.sub('', isp).strip()
  44. return isp
  45. class BatcaveHttpRequestHandler(BaseHTTPRequestHandler):
  46. """Handles a single HTTP request to the BATCAVE."""
  47. def __init__(self, request, client_address, sockserver):
  48. self.logger = logging.getLogger('API')
  49. BaseHTTPRequestHandler.__init__(
  50. self, request, client_address, sockserver)
  51. def __parse_url_pathquery(self):
  52. """Extracts the query parameters from the request path."""
  53. url = re.match(r'^/(?P<path>.*?)(\?(?P<query>.+))?$', self.path.strip())
  54. if url is None:
  55. logging.warn('Failed to parse URL \'' + str(self.path) + '\'.')
  56. return (None, None)
  57. path = url.group('path')
  58. query = {}
  59. if not url.group('query') is None:
  60. for match in REGEX_QUERYPARAM.finditer(url.group('query')):
  61. query[match.group('key')] = match.group('value')
  62. return (path, query)
  63. def do_GET(self):
  64. """Handles all HTTP GET requests."""
  65. path, query = self.__parse_url_pathquery()
  66. if path is None:
  67. self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
  68. return
  69. # / - index page, shows generic help
  70. if path == '':
  71. self.__respond_index(query)
  72. return
  73. # /list - list stored nodes
  74. if path == 'list':
  75. self.__respond_list(query)
  76. return
  77. # /vpn - notification endpoint for gateway's VPN connections
  78. if path == 'vpn':
  79. self.__respond_vpn(query)
  80. return
  81. # /providers
  82. if path == 'providers':
  83. self.__respond_providers(query)
  84. return
  85. # /node/<id>.json - node's data
  86. # /node/<id>/field - return specific field from node's data
  87. match = REGEX_URL_NODEINFO.match(path)
  88. if match is not None:
  89. cmd = match.group('cmd')
  90. nodeid = match.group('id').lower()
  91. if cmd == '.json':
  92. self.__respond_node(nodeid)
  93. else:
  94. self.__respond_nodedetail(nodeid, cmd[1:])
  95. return
  96. # /status - overall status (incl. node and client count)
  97. if path == 'status':
  98. self.__respond_status()
  99. return
  100. # /status/<id> - node's status
  101. match = REGEX_URL_NODESTATUS.match(path)
  102. if match is not None:
  103. self.__respond_nodestatus(match.group(1))
  104. return
  105. # no match -> 404
  106. self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))
  107. def do_POST(self):
  108. """Handles all HTTP POST requests."""
  109. path, query = self.__parse_url_pathquery()
  110. if path is None:
  111. self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
  112. return
  113. params = self.__parse_post_params()
  114. # node id/mac to name mapping
  115. if path == 'idmac2name':
  116. self.__respond_nodeidmac2name(params)
  117. return
  118. # no match -> 404
  119. self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))
  120. def __send_nocache_headers(self):
  121. """
  122. Sets HTTP headers indicating that this response shall not be cached.
  123. """
  124. self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
  125. self.send_header('Pragma', 'no-cache')
  126. self.send_header('Expires', '0')
  127. def __send_headers(self,
  128. content_type='text/html; charset=utf-8',
  129. nocache=True, extra={}):
  130. """Send HTTP 200 Response header with the given Content-Type.
  131. Optionally send no-caching headers, too."""
  132. self.send_response(200)
  133. self.send_header('Content-Type', content_type)
  134. if nocache:
  135. self.__send_nocache_headers()
  136. for key in extra:
  137. self.send_header(key, extra[key])
  138. self.end_headers()
  139. def __parse_post_params(self):
  140. ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
  141. if ctype == 'multipart/form-data':
  142. postvars = cgi.parse_multipart(self.rfile, pdict)
  143. elif ctype == 'application/x-www-form-urlencoded':
  144. length = int(self.headers.getheader('content-length'))
  145. postvars = cgi.parse_qs(
  146. self.rfile.read(length),
  147. keep_blank_values=1,
  148. )
  149. else:
  150. postvars = {}
  151. return postvars
  152. def __respond_index(self, query):
  153. """Display the index page."""
  154. self.__send_headers()
  155. index_page = '''<!DOCTYPE html>
  156. <html><head><title>BATCAVE</title></head>
  157. <body>
  158. <H1 title="Batman/Alfred Transmission Collection, Aggregation & Value Engine">
  159. BATCAVE
  160. </H1>
  161. <p>Dies ist ein interner Hintergrund-Dienst. Er wird nur von anderen Diensten
  162. angesprochen und sollte aus einer Mehrzahl von Gr&uuml;nden nicht
  163. &ouml;ffentlich zug&auml;nglich sein.</p>
  164. <H2>API</H2>
  165. <p>
  166. Grunds&auml;tzlich ist das Antwort-Format JSON und alle Daten sind
  167. Live-Daten (kein Cache) die ggf. etwas Bearbeitungs-Zeit erfordern.
  168. </p>
  169. <dl>
  170. <dt>GET <a href="/nodes.json">nodes.json</a></dt>
  171. <dd>zur Verwendung mit ffmap (MACs anonymisiert)</dd>
  172. <dt>GET /node/&lt;id&gt;.json</dt>
  173. <dd>alle Daten zu dem gew&uuml;nschten Knoten</dd>
  174. <dt>GET /providers?format=json</dt>
  175. <dd>Liste der Provider</dd>
  176. <dt>GET <a href="/status">/status</a></dt>
  177. <dd>Status der BATCAVE inkl. Zahl der Nodes+Clients (JSON)</dd>
  178. <dt>GET /status/&lt;id&gt;</dt>
  179. <dd>Status des Knotens</dd>
  180. </dl>
  181. </body></html>'''
  182. self.wfile.write(index_page)
  183. def __respond_list(self, query):
  184. """List stored data."""
  185. self.__send_headers()
  186. self.wfile.write('<!DOCTYPE html><html>\n')
  187. self.wfile.write('<head><title>BATCAVE</title></head>\n')
  188. self.wfile.write('<body>\n')
  189. self.wfile.write('<H1>BATCAVE - LIST</H1>\n')
  190. self.wfile.write('<table>\n')
  191. self.wfile.write('<thead><tr><th>ID</th><th>Name</th></tr></thead>\n')
  192. self.wfile.write('<tbody>\n')
  193. sortkey = query['sort'] if 'sort' in query else None
  194. data = self.server.storage.get_nodes(sortby=sortkey)
  195. for node in data:
  196. nodeid = node.get('node_id')
  197. nodename = node.get('hostname', '&lt;?&gt;')
  198. self.wfile.write('<tr>\n')
  199. self.wfile.write(' <td><a href="/node/{0}.json">{0}</a></td>\n'.format(nodeid))
  200. self.wfile.write(' <td>{0}</td>\n'.format(nodename))
  201. self.wfile.write('</tr>\n')
  202. self.wfile.write('</tbody>\n')
  203. self.wfile.write('</table>\n')
  204. def __respond_node(self, rawid):
  205. """Display node data."""
  206. # search node by the given id
  207. node = self.server.storage.find_node(rawid)
  208. # handle unknown nodes
  209. if node is None:
  210. self.send_error(404, 'No node with id \'' + rawid + '\' present.')
  211. return
  212. # dump node data as JSON
  213. self.__send_headers('application/json',
  214. extra={'Content-Disposition': 'inline'})
  215. self.wfile.write(json.dumps(node))
  216. def __respond_nodestatus(self, rawid):
  217. """Display node status."""
  218. status = self.server.storage.get_nodestatus(rawid)
  219. if status is None:
  220. self.send_error(404, 'No node with id \'' + rawid + '\' present.')
  221. self.__send_headers('text/plain')
  222. self.wfile.write(status)
  223. def __respond_nodeidmac2name(self, ids):
  224. """Return a mapping of the given IDs (or MACs) into their hostname."""
  225. self.__send_headers('text/plain')
  226. for nodeid in ids:
  227. node = None
  228. if not ':' in nodeid:
  229. node = self.server.storage.find_node(nodeid)
  230. else:
  231. node = self.server.storage.find_node_by_mac(nodeid)
  232. nodename = node.get('hostname', nodeid) if node is not None else nodeid
  233. self.wfile.write('{0}={1}\n'.format(nodeid, nodename))
  234. def __respond_nodedetail(self, nodeid, field):
  235. """
  236. Return a field from the given node.
  237. String and integers are returned as text/plain,
  238. all other as JSON.
  239. """
  240. node = self.server.storage.find_node(nodeid)
  241. if node is None:
  242. self.send_error(404, 'No node with id \'' + nodeid + '\' present.')
  243. return
  244. return_count = False
  245. if field.endswith('.count'):
  246. return_count = True
  247. field = field[0:-6]
  248. if not field in node:
  249. self.send_error(
  250. 404,
  251. 'The node \'{0}\' does not have a field named \'{1}\'.'.format(
  252. nodeid, field
  253. )
  254. )
  255. return
  256. value = node[field]
  257. if return_count:
  258. value = len(value)
  259. no_json = isinstance(value, basestring) or isinstance(value, int)
  260. self.__send_headers('text/plain' if no_json else 'application/json',
  261. extra={'Content-Disposition': 'inline'})
  262. self.wfile.write(value if no_json else json.dumps(value))
  263. def __respond_status(self):
  264. status = self.server.storage.status
  265. self.__send_headers('application/json',
  266. extra={'Content-Disposition': 'inline'})
  267. self.wfile.write(json.dumps(status, indent=2))
  268. def __respond_vpn(self, query):
  269. storage = self.server.storage
  270. peername = query.get('peer')
  271. key = query.get('key')
  272. action = query.get('action')
  273. remote = query.get('remote')
  274. gateway = query.get('gw')
  275. timestamp = query.get('ts', time.time())
  276. if action == 'list':
  277. self.__respond_vpnlist()
  278. return
  279. if action != 'establish' and action != 'disestablish':
  280. self.logger.error('VPN: unknown action \'{0}\''.format(action))
  281. self.send_error(400, 'Invalid action.')
  282. return
  283. check = {
  284. 'peername': peername,
  285. 'key': key,
  286. 'remote': remote,
  287. 'gw': gateway,
  288. }
  289. for k, val in check.items():
  290. if val is None or len(val.strip()) == 0:
  291. self.logger.error('VPN {0}: no or empty {1}'.format(action, k))
  292. self.send_error(400, 'Missing value for ' + str(k))
  293. return
  294. try:
  295. if action == 'establish':
  296. self.server.storage.log_vpn_connect(
  297. key, peername, remote, gateway, timestamp)
  298. elif action == 'disestablish':
  299. self.server.storage.log_vpn_disconnect(key, gateway, timestamp)
  300. else:
  301. self.logger.error('Unknown VPN action \'%s\' not filtered.',
  302. action)
  303. self.send_error(500)
  304. return
  305. except ffstatus.exceptions.VpnKeyFormatError:
  306. self.logger.error('VPN peer \'{0}\' {1}: bad key \'{2}\''.format(
  307. peername, action, key,
  308. ))
  309. self.send_error(400, 'Bad key.')
  310. return
  311. self.__send_headers('text/plain')
  312. self.wfile.write('OK')
  313. storage.save()
  314. def __respond_vpnlist(self):
  315. self.__send_headers()
  316. self.wfile.write('''<!DOCTYPE html>
  317. <html><head><title>BATCAVE - VPN LIST</title></head>
  318. <body>
  319. <style type="text/css">
  320. table { border: 2px solid #999; border-collapse: collapse; }
  321. th, td { border: 1px solid #CCC; }
  322. table tbody tr.online { background-color: #CFC; }
  323. table tbody tr.offline { background-color: #FCC; }
  324. </style>
  325. <table>''')
  326. gateways = self.server.storage.get_vpn_gateways()
  327. gws_header = '<th>' + '</th><th>'.join(gateways) + '</th>'
  328. self.wfile.write('<thead>\n')
  329. self.wfile.write('<tr><th rowspan="2">names (key)</th>')
  330. self.wfile.write('<th colspan="' + str(len(gateways)) + '">active</th>')
  331. self.wfile.write('<th colspan="' + str(len(gateways)) + '">last</th>')
  332. self.wfile.write('</tr>\n')
  333. self.wfile.write('<tr>' + gws_header + gws_header + '</tr>\n')
  334. self.wfile.write('</thead>\n')
  335. for item in self.server.storage.get_vpn_connections():
  336. row_class = 'online' if item['online'] else 'offline'
  337. self.wfile.write('<tr class="{0}">'.format(row_class))
  338. self.wfile.write('<td title="{0}">{1}</td>'.format(
  339. item['key'],
  340. ' / '.join(item['names']) if len(item['names']) > 0 else '?',
  341. ))
  342. for conntype in ['active', 'last']:
  343. for gateway in gateways:
  344. remote = ''
  345. if conntype in item['remote'] and \
  346. gateway in item['remote'][conntype]:
  347. remote = item['remote'][conntype][gateway]
  348. if isinstance(remote, dict):
  349. remote = remote['name']
  350. symbol = '&check;' if len(remote) > 0 else '&times;'
  351. self.wfile.write('<td title="{0}">{1}</td>'.format(
  352. remote, symbol))
  353. self.wfile.write('</tr>\n')
  354. self.wfile.write('</table>\n')
  355. self.wfile.write('</body>')
  356. self.wfile.write('</html>')
  357. def __respond_providers(self, query):
  358. """Return a summary of providers."""
  359. outputformat = query['format'].lower() if 'format' in query else 'html'
  360. isps = {}
  361. ispblocks = {}
  362. for item in self.server.storage.get_vpn_connections():
  363. if item['count']['active'] == 0:
  364. continue
  365. remotes = []
  366. for gateway in item['remote']['active']:
  367. remote = item['remote']['active'][gateway]
  368. remotes.append(remote)
  369. if len(remotes) == 0:
  370. self.logger.warn(
  371. 'VPN key \'%s\' is marked with active remotes but 0 found?',
  372. item['key'])
  373. continue
  374. item_isps = set()
  375. for remote in remotes:
  376. isp = "UNKNOWN"
  377. ispblock = remote
  378. if isinstance(remote, dict):
  379. ispblock = remote['name']
  380. desc_lines = remote['description'].split('\n')
  381. isp = normalize_ispname(desc_lines[0])
  382. if not isp in ispblocks:
  383. ispblocks[isp] = set()
  384. ispblocks[isp].add(ispblock)
  385. item_isps.add(isp)
  386. if len(item_isps) == 0:
  387. item_isps.add('unknown')
  388. elif len(item_isps) > 1:
  389. self.logger.warn(
  390. 'VPN key \'%s\' has %d active IPs ' +
  391. 'which resolved to %d ISPs: \'%s\'',
  392. item['key'],
  393. len(remotes),
  394. len(item_isps),
  395. '\', \''.join(item_isps)
  396. )
  397. for isp in item_isps:
  398. if not isp in isps:
  399. isps[isp] = 0
  400. isps[isp] += 1.0 / len(item_isps)
  401. isps_sum = sum([isps[x] for x in isps])
  402. if outputformat == 'csv':
  403. self.__send_headers('text/csv')
  404. self.wfile.write('Count;Name\n')
  405. for isp in isps:
  406. self.wfile.write('{0};"{1}"\n'.format(isps[isp], isp))
  407. elif outputformat == 'json':
  408. self.__send_headers('application/json',
  409. extra={'Content-Disposition': 'inline'})
  410. data = [
  411. {
  412. 'name': isp,
  413. 'count': isps[isp],
  414. 'percentage': isps[isp]*100.0/isps_sum,
  415. 'blocks': [block for block in ispblocks[isp]],
  416. } for isp in isps
  417. ]
  418. self.wfile.write(json.dumps(data))
  419. elif outputformat == 'html':
  420. self.__send_headers()
  421. self.wfile.write('''<!DOCTYPE html>
  422. <html>
  423. <head><title>BATCAVE - PROVIDERS</title></head>
  424. <body>
  425. <table border="2">
  426. <thead>
  427. <tr><th>Count</th><th>Percentage</th><th>Name</th><th>Blocks</th></tr>
  428. </thead>
  429. <tbody>\n''')
  430. for isp in sorted(isps, key=lambda x: isps[x], reverse=True):
  431. self.wfile.write('<tr><td>{0}</td><td>{1:.1f}%</td><td>{2}</td><td>{3}</td></tr>\n'.format(
  432. isps[isp],
  433. isps[isp]*100.0/isps_sum,
  434. isp,
  435. ', '.join(sorted(ispblocks[isp])) if isp in ispblocks else '?',
  436. ))
  437. self.wfile.write('</tbody></table>\n')
  438. self.wfile.write('<p>Totals: {0} ISPs, {1} connections</p>\n'.format(len(isps), isps_sum))
  439. self.wfile.write('</body></html>')
  440. else:
  441. self.send_error(400, 'Unknown output format.')
  442. class ApiServer(ThreadingMixIn, HTTPServer):
  443. def __init__(self, endpoint, storage):
  444. if ':' in endpoint[0]:
  445. self.address_family = socket.AF_INET6
  446. HTTPServer.__init__(self, endpoint, BatcaveHttpRequestHandler)
  447. self.storage = storage
  448. def __str__(self):
  449. return 'ApiServer on {0}'.format(self.server_address)
  450. if __name__ == '__main__':
  451. dummystorage = ffstatus.basestorage.BaseStorage()
  452. server = ApiServer(('0.0.0.0', 8888), dummystorage)
  453. print("Server:", str(server))
  454. server.serve_forever()