#!/usr/bin/python # -*- coding: utf-8 -*- from __future__ import print_function from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer import cgi from storage import Storage import json import logging import pygeoip import re import socket from SocketServer import ThreadingMixIn import time class BatcaveHttpRequestHandler(BaseHTTPRequestHandler): DATAKEY_VPN = '__VPN__' def __init__(self, request, client_address, server): self.logger = logging.getLogger('API') BaseHTTPRequestHandler.__init__(self, request, client_address, server) def parse_url_pathquery(self): """Extracts the query parameters from the request path.""" url = re.match(r'^/(?P.*?)(\?(?P.+))?$', self.path.strip()) if url is None: logging.warn('Failed to parse URL \'' + str(self.path) + '\'.') return ( None, None ) path = url.group('path') query = {} if not url.group('query') is None: for m in re.finditer(r'(?P.+?)=(?P.+?)(&|$)', url.group('query')): query[m.group('key')] = m.group('value') return ( path, query ) def do_GET(self): """Handles all HTTP GET requests.""" path, query = self.parse_url_pathquery() if path is None: self.send_error(400, 'Could not parse URL (' + str(self.path) + ')') return # / - index page, shows generic help if path == '': self.respond_index(query) return # /list - list stored nodes if path == 'list': self.respond_list(query) return # /vpn - notification endpoint for gateway's VPN connections if path == 'vpn': self.respond_vpn(query) return # /providers if path == 'providers': self.respond_providers(query) return # /node/.json - node's data # /node//field - return specific field from node's data m = re.match(r'node/([a-f0-9]{12})(?P\.json|/[a-zA-Z0-9_\-]+)$', path) if m != None: cmd = m.group('cmd') if cmd == '.json': self.respond_node(m.group(1)) else: self.respond_nodedetail(m.group(1), cmd[1:]) return # no match -> 404 self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path)) def do_POST(self): """Handles all HTTP POST requests.""" path, query = self.parse_url_pathquery() if path is None: self.send_error(400, 'Could not parse URL (' + str(self.path) + ')') return params = self.parse_post_params() # node id/mac to name mapping if path == 'idmac2name': self.respond_nodeidmac2name(params) return # no match -> 404 self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path)) def send_nocache_headers(self): """Sets HTTP headers indicating that this response shall not be cached.""" self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate') self.send_header('Pragma', 'no-cache') self.send_header('Expires', '0') def send_headers(self, content_type='text/html; charset=utf-8', nocache=True): """Send HTTP 200 Response header with the given Content-Type. Optionally send no-caching headers, too.""" self.send_response(200) self.send_header('Content-Type', content_type) if nocache: self.send_nocache_headers() self.end_headers() def parse_post_params(self): ctype, pdict = cgi.parse_header(self.headers.getheader('content-type')) if ctype == 'multipart/form-data': postvars = cgi.parse_multipart(self.rfile, pdict) elif ctype == 'application/x-www-form-urlencoded': length = int(self.headers.getheader('content-length')) postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1) else: postvars = {} return postvars def respond_index(self, query): """Display the index page.""" storage = self.server.storage self.send_headers() self.wfile.write('BATCAVE\n') self.wfile.write('\n') self.wfile.write('

BATCAVE

\n') self.wfile.write('

Dies ist ein interner Hintergrund-Dienst. Er wird nur von anderen Diensten\n') self.wfile.write('angesprochen und sollte aus einer Mehrzahl von Gründen nicht öffentlich\n') self.wfile.write('zugänglich sein.

\n') self.wfile.write('

Status

\n') self.wfile.write('Daten: ') self.wfile.write(len(storage.data)) self.wfile.write('\n') self.wfile.write('

API

\n') self.wfile.write('

Grundsätzlich ist das Antwort-Format JSON und alle Daten sind Live-Daten (kein Cache) die ggf. etwas Bearbeitungs-Zeit erfordern.

') self.wfile.write('

nodes.json: zur Verwendung mit ffmap (MACs anonymisiert)
/node/<id>.json: alle vorhandenen Information zu der gewünschten Node

\n') self.wfile.write('') def respond_list(self, query): """List stored data.""" storage = self.server.storage self.send_headers() self.wfile.write('BATCAVE\n') self.wfile.write('\n') self.wfile.write('

BATCAVE - LIST

\n') self.wfile.write('\n') self.wfile.write('\n') self.wfile.write('\n') data = storage.data if 'sort' in query: if query['sort'] == 'name': sorteddata = sorted(data, key=lambda x: data[x]['hostname'].lower()) data = sorteddata elif query['sort'] == 'id': sorteddata = sorted(data) data = sorteddata for nodeid in data: if nodeid.startswith('__'): continue nodename = storage.data[nodeid]['hostname'] if 'hostname' in storage.data[nodeid] else '<?>' self.wfile.write('') self.wfile.write('\n') self.wfile.write('

ID	Name
' + nodeid + '	' + nodename + '

\n') def find_node(self, rawid): """Fetch node data from storage by given id, if necessary looking thorugh node aliases.""" storage = self.server.storage # if we have a direct hit, return it immediately if rawid in storage.data: return storage.data[rawid] # no direct hit -> search via aliases nodeid = rawid for n in storage.data: if 'aliases' in storage.data[n] and rawid in storage.data[n]['aliases']: nodeid = n # return found node return storage.data[nodeid] if nodeid in storage.data else None def find_node_by_mac(self, mac): """Fetch node data from storage by given MAC address.""" storage = self.server.storage needle = mac.lower() # iterate over all nodes for nodeid in storage.data: if nodeid.startswith('__'): continue node = storage.data[nodeid] # check node's primary MAC if 'mac' in node and needle == node['mac'].lower(): return node # check alias MACs if 'macs' in node: haystack = [ x.lower() for x in node['macs'] ] if mac in haystack: return node # MAC address not found return None def respond_node(self, rawid): """Display node data.""" # handle API example linked on index page if rawid == 'ff00ff00ff00': self.send_headers('text/json') self.wfile.write(json.dumps({ 'name': 'API-Example', 'nodeid': rawid, 'META': 'Dies ist ein minimaler Beispiel-Datensatz. Herzlichen Glückwunsch, du hast das Prinzip der API kapiert.', })) return # search node by the given id node = self.find_node(rawid) # handle unknown nodes if node is None: self.send_error(404, 'No node with id \'' + rawid + '\' present.') return # dump node data as JSON self.send_headers('text/json') self.wfile.write(json.dumps(node)) def respond_nodeidmac2name(self, ids): storage = self.server.storage self.send_headers('text/plain') for nodeid in ids: node = self.find_node(nodeid) if not ':' in nodeid else self.find_node_by_mac(nodeid) nodename = node['hostname'] if (not node is None) and 'hostname' in node else nodeid self.wfile.write('{0}={1}\n'.format(nodeid, nodename)) def respond_nodedetail(self, nodeid, field): """Return a field from the given node - a string is returned as text, all other as JSON.""" node = self.find_node(nodeid) if node is None: self.send_error(404, 'No node with id \'' + nodeid + '\' present.') return if not field in node: self.send_error(404, 'The node \'' + nodeid + '\' does not have a field named \'' + str(field) + '\'.') return value = node[field] self.send_headers('text/plain' if isinstance(value, basestring) else 'text/json') self.wfile.write(value if isinstance(value, basestring) else json.dumps(value)) def respond_vpn(self, query): storage = self.server.storage peername = query['peer'] if 'peer' in query else None key = query['key'] if 'key' in query else None action = query['action'] if 'action' in query else None remote = query['remote'] if 'remote' in query else None gw = query['gw'] if 'gw' in query else None if action == 'list': self.respond_vpnlist() return if action != 'establish' and action != 'disestablish': self.logger.error('VPN: unknown action \'{0}\''.format(action)) self.send_error(400, 'Invalid action.') return for k,v in { 'peername': peername, 'key': key, 'remote': remote, 'gw': gw }.items(): if v is None or len(v.strip()) == 0: self.logger.error('VPN {0}: no or empty {1}'.format(action, k)) self.send_error(400, 'Missing value for ' + str(k)) return if key is None or re.match(r'^[a-fA-F0-9]+$', key) is None: self.logger.error('VPN peer \'{0}\' {1}: bad key \'{2}\''.format(peername, action, key)) self.send_error(400, 'Bad key.') return if not self.DATAKEY_VPN in storage.data: storage.data[self.DATAKEY_VPN] = {} if not key in storage.data[self.DATAKEY_VPN]: storage.data[self.DATAKEY_VPN][key] = { 'active': {}, 'last': {} } item = storage.data[self.DATAKEY_VPN][key] if action == 'establish': item['active'][gw] = { 'establish': time.time(), 'peer': peername, 'remote': remote } elif action == 'disestablish': active = {} if gw in item['active']: active = item['active'][gw] del(item['active'][gw]) active['disestablish'] = time.time() item['last'][gw] = active else: self.send_error(500, 'Unknown action not filtered (' + str(action) + ')') return self.send_headers('text/plain') self.wfile.write('OK') storage.save() def respond_vpnlist(self): storage = self.server.storage gateways = ['gw01','gw02','gw03','gw04'] self.send_headers() self.wfile.write('\n') self.wfile.write('BATCAVE - VPN LIST\n') self.wfile.write('\n') self.wfile.write('\n') self.wfile.write('\n\n') self.wfile.write('\n') self.wfile.write('\n') self.wfile.write('\n') if self.DATAKEY_VPN in storage.data: for key in storage.data[self.DATAKEY_VPN]: item = storage.data[self.DATAKEY_VPN][key] names = set() count = {} for t in [ 'active', 'last' ]: count[t] = 0 if t in item: for gw in item[t]: if 'remote' in item[t][gw] and len(item[t][gw]['remote']) > 0: count[t] += 1 if 'peer' in item[t][gw]: names.add(item[t][gw]['peer']) self.wfile.write('' if count['active'] > 0 else '') self.wfile.write('') for t in [ 'active', 'last' ]: for gw in gateways: ip = '' details = '' if t in item and gw in item[t]: ip = item[t][gw]['remote'] if 'remote' in item[t][gw] else '' self.wfile.write('') self.wfile.write('\n') self.wfile.write('

names (key)	active	last
names (key)	' + '	'.join(gateways) + '	' + '	'.join(gateways) + '
' + (' / '.join(names) if len(names) > 0 else '?') + '	' + ip + '

\n') self.wfile.write('') self.wfile.write('') def respond_providers(self, query): """Return a summary of providers.""" vpn = self.server.storage.data[self.DATAKEY_VPN] outputformat = query['format'].lower() if 'format' in query else 'html' geo = None try: geo = pygeoip.GeoIP('GeoIPISP.dat') except: self.return_error(500, 'The GeoIP-ISP database file could not be opened.') return isps = {} for key in vpn: if key is None: continue item = vpn[key] if not 'active' in item: continue ips = set() for gw in item['active']: if 'remote' in item['active'][gw]: ips.add(item['active'][gw]['remote']) if len(ips) == 0: continue if len(ips) > 1: self.logger.warn('VPN key \'{0}\' has {1} active ips. Possible cause is an in-progress re-dialin.'.format(key, len(ips))) item_isps = set() for ip in ips: try: isp = geo.org_by_addr(ip) if not isp is None: item_isps.add(isp) except Exception as err: self.logger.debug('Failed to resolve ISP for \'{0}\': {1}'.format(ip, str(err))) if len(item_isps) == 0: item_isps.add('unknown') elif len(item_isps) > 1: self.logger.warn('VPN key \'{0}\' has {1} active IPs which resolved to {2} ISPs: \'{3}\''.format(key, len(ips), len(item_isps), '\', \''.join(item_isps))) for isp in item_isps: if not isp in isps: isps[isp] = 0 isps[isp] += 1 isps_sum = sum([isps[x] for x in isps]) if outputformat == 'csv': self.send_headers('text/csv') self.wfile.write('Count;Name\n') for isp in isps: self.wfile.write('{0};"{1}"\n'.format(isps[isp], isp)) elif outputformat == 'json': self.send_headers('text/json') data = [ { 'name': x, 'count': isps[x], 'percentage': isps[x]*100.0/isps_sum } for x in isps ] self.wfile.write(json.dumps(data)) elif outputformat == 'html': self.send_headers() self.wfile.write('BATCAVE - PROVIDERS\n') self.wfile.write('\n') for isp in isps: self.wfile.write('\n'.format(isps[isp], isps[isp]*100.0/isps_sum, isp)) self.wfile.write('

Count	Percentage	Name
{0}	{1:.1f}%	{2}

\n') self.wfile.write('') else: self.send_error(400, 'Unknown output format.') class ApiServer(ThreadingMixIn, HTTPServer): def __init__(self, endpoint, storage): if ':' in endpoint[0]: self.address_family = socket.AF_INET6 HTTPServer.__init__(self, endpoint, BatcaveHttpRequestHandler) self.storage = storage def __str__(self): return 'ApiServer on {0}'.format(self.server_address) if __name__ == '__main__': dummystorage = Storage() server = ApiServer(('0.0.0.0', 8888), dummystorage) print("Server:", str(server)) server.serve_forever()