#!/usr/bin/python # -*- coding: utf-8 -*- from __future__ import print_function from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer import cgi from storage import Storage import json import logging import pygeoip import re import socket from SocketServer import ThreadingMixIn import time import ffstatus class BatcaveHttpRequestHandler(BaseHTTPRequestHandler): DATAKEY_VPN = '__VPN__' FIELDKEY_UPDATED = '__UPDATED__' def __init__(self, request, client_address, server): self.logger = logging.getLogger('API') BaseHTTPRequestHandler.__init__(self, request, client_address, server) def parse_url_pathquery(self): """Extracts the query parameters from the request path.""" url = re.match(r'^/(?P.*?)(\?(?P.+))?$', self.path.strip()) if url is None: logging.warn('Failed to parse URL \'' + str(self.path) + '\'.') return ( None, None ) path = url.group('path') query = {} if not url.group('query') is None: for m in re.finditer(r'(?P.+?)=(?P.+?)(&|$)', url.group('query')): query[m.group('key')] = m.group('value') return ( path, query ) def do_GET(self): """Handles all HTTP GET requests.""" path, query = self.parse_url_pathquery() if path is None: self.send_error(400, 'Could not parse URL (' + str(self.path) + ')') return # / - index page, shows generic help if path == '': self.respond_index(query) return # /list - list stored nodes if path == 'list': self.respond_list(query) return # /vpn - notification endpoint for gateway's VPN connections if path == 'vpn': self.respond_vpn(query) return # /providers if path == 'providers': self.respond_providers(query) return # /node/.json - node's data # /node//field - return specific field from node's data m = re.match(r'node/(?P[a-fA-F0-9]{12})(?P\.json|/[a-zA-Z0-9_\-\.]+)$', path) if m != None: cmd = m.group('cmd') nodeid = m.group('id').lower() if cmd == '.json': self.respond_node(nodeid) else: self.respond_nodedetail(nodeid, cmd[1:]) return # /status/ - node's status m = re.match(r'status/([a-f0-9]{12})$', path) if m != None: self.respond_nodestatus(m.group(1)) return # no match -> 404 self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path)) def do_POST(self): """Handles all HTTP POST requests.""" path, query = self.parse_url_pathquery() if path is None: self.send_error(400, 'Could not parse URL (' + str(self.path) + ')') return params = self.parse_post_params() # node id/mac to name mapping if path == 'idmac2name': self.respond_nodeidmac2name(params) return # no match -> 404 self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path)) def send_nocache_headers(self): """Sets HTTP headers indicating that this response shall not be cached.""" self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate') self.send_header('Pragma', 'no-cache') self.send_header('Expires', '0') def send_headers(self, content_type='text/html; charset=utf-8', nocache=True): """Send HTTP 200 Response header with the given Content-Type. Optionally send no-caching headers, too.""" self.send_response(200) self.send_header('Content-Type', content_type) if nocache: self.send_nocache_headers() self.end_headers() def parse_post_params(self): ctype, pdict = cgi.parse_header(self.headers.getheader('content-type')) if ctype == 'multipart/form-data': postvars = cgi.parse_multipart(self.rfile, pdict) elif ctype == 'application/x-www-form-urlencoded': length = int(self.headers.getheader('content-length')) postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1) else: postvars = {} return postvars def respond_index(self, query): """Display the index page.""" storage = self.server.storage self.send_headers() self.wfile.write('BATCAVE\n') self.wfile.write('\n') self.wfile.write('

BATCAVE

\n') self.wfile.write('

Dies ist ein interner Hintergrund-Dienst. Er wird nur von anderen Diensten\n') self.wfile.write('angesprochen und sollte aus einer Mehrzahl von Gründen nicht öffentlich\n') self.wfile.write('zugänglich sein.

\n') self.wfile.write('

Status

\n') self.wfile.write('Daten: ') self.wfile.write(len(storage.data)) self.wfile.write('\n') self.wfile.write('

API

\n') self.wfile.write('

Grundsätzlich ist das Antwort-Format JSON und alle Daten sind Live-Daten (kein Cache) die ggf. etwas Bearbeitungs-Zeit erfordern.

') self.wfile.write('

nodes.json: zur Verwendung mit ffmap (MACs anonymisiert)
/node/<id>.json: alle vorhandenen Information zu der gewünschten Node

\n') self.wfile.write('') def respond_list(self, query): """List stored data.""" storage = self.server.storage self.send_headers() self.wfile.write('BATCAVE\n') self.wfile.write('\n') self.wfile.write('

BATCAVE - LIST

\n') self.wfile.write('\n') self.wfile.write('\n') self.wfile.write('\n') data = storage.data if 'sort' in query: if query['sort'] == 'name': sorteddata = sorted(data, key=lambda x: data[x]['hostname'].lower()) data = sorteddata elif query['sort'] == 'id': sorteddata = sorted(data) data = sorteddata for nodeid in data: if nodeid.startswith('__'): continue nodename = storage.data[nodeid]['hostname'] if 'hostname' in storage.data[nodeid] else '<?>' self.wfile.write('') self.wfile.write('\n') self.wfile.write('

ID	Name
' + nodeid + '	' + nodename + '

\n') def find_node(self, rawid): """Fetch node data from storage by given id, if necessary looking thorugh node aliases.""" storage = self.server.storage # if we have a direct hit, return it immediately if rawid in storage.data: return storage.data[rawid] # no direct hit -> search via aliases nodeid = rawid for n in storage.data: if 'aliases' in storage.data[n] and rawid in storage.data[n]['aliases']: nodeid = n # return found node return storage.data[nodeid] if nodeid in storage.data else None def find_node_by_mac(self, mac): """Fetch node data from storage by given MAC address.""" storage = self.server.storage needle = mac.lower() # iterate over all nodes for nodeid in storage.data: if nodeid.startswith('__'): continue node = storage.data[nodeid] # check node's primary MAC if 'mac' in node and needle == node['mac'].lower(): return node # check alias MACs if 'macs' in node: haystack = [ x.lower() for x in node['macs'] ] if mac in haystack: return node # MAC address not found return None def respond_node(self, rawid): """Display node data.""" # handle API example linked on index page if rawid == 'ff00ff00ff00': self.send_headers('text/json') self.wfile.write(json.dumps({ 'name': 'API-Example', 'nodeid': rawid, 'META': 'Dies ist ein minimaler Beispiel-Datensatz. Herzlichen Glückwunsch, du hast das Prinzip der API kapiert.', })) return # search node by the given id node = self.find_node(rawid) # handle unknown nodes if node is None: self.send_error(404, 'No node with id \'' + rawid + '\' present.') return # remove fields from output: __RAW__ export = ffstatus.dict_merge({}, node) if '__RAW__' in export: del(export['__RAW__']) # dump node data as JSON self.send_headers('text/json') self.wfile.write(json.dumps(export)) def get_nodestatus(self, rawid): """Determine node's status.""" # search node by the given id node = self.find_node(rawid) # handle unknown nodes if node is None: return None # check that the last batadv update is noted in the data updated = node[self.FIELDKEY_UPDATED] if self.FIELDKEY_UPDATED in node else None if updated is None or not 'batadv' in updated: return 'unknown' # make decision based on time of last batadv update diff = time.time() - updated['batadv'] if diff < 150: return 'active' elif diff < 300: return 'stale' else: return 'offline' def respond_nodestatus(self, rawid): """Display node status.""" status = self.get_nodestatus(rawid) if status is None: self.send_error(404, 'No node with id \'' + rawid + '\' present.') self.send_headers('text/plain') self.wfile.write(status) def respond_nodeidmac2name(self, ids): """Return a mapping of the given IDs (or MACs) into their hostname.""" self.send_headers('text/plain') for nodeid in ids: node = self.find_node(nodeid) if not ':' in nodeid else self.find_node_by_mac(nodeid) nodename = node['hostname'] if (not node is None) and 'hostname' in node else nodeid self.wfile.write('{0}={1}\n'.format(nodeid, nodename)) def respond_nodedetail(self, nodeid, field): """Return a field from the given node - a string is returned as text, all other as JSON.""" node = self.find_node(nodeid) if node is None: self.send_error(404, 'No node with id \'' + nodeid + '\' present.') return return_count = False if field.endswith('.count'): return_count = True field = field[0:-6] if not field in node: self.send_error(404, 'The node \'' + nodeid + '\' does not have a field named \'' + str(field) + '\'.') return value = node[field] if return_count: value = len(value) self.send_headers('text/plain' if isinstance(value, basestring) or isinstance(value, int) else 'text/json') self.wfile.write(value if isinstance(value, basestring) else json.dumps(value)) def respond_vpn(self, query): storage = self.server.storage peername = query['peer'] if 'peer' in query else None key = query['key'] if 'key' in query else None action = query['action'] if 'action' in query else None remote = query['remote'] if 'remote' in query else None gw = query['gw'] if 'gw' in query else None ts = query['ts'] if 'ts' in query else time.time() if action == 'list': self.respond_vpnlist() return if action != 'establish' and action != 'disestablish': self.logger.error('VPN: unknown action \'{0}\''.format(action)) self.send_error(400, 'Invalid action.') return for k,v in { 'peername': peername, 'key': key, 'remote': remote, 'gw': gw }.items(): if v is None or len(v.strip()) == 0: self.logger.error('VPN {0}: no or empty {1}'.format(action, k)) self.send_error(400, 'Missing value for ' + str(k)) return if key is None or re.match(r'^[a-fA-F0-9]+$', key) is None: self.logger.error('VPN peer \'{0}\' {1}: bad key \'{2}\''.format(peername, action, key)) self.send_error(400, 'Bad key.') return if not self.DATAKEY_VPN in storage.data: storage.data[self.DATAKEY_VPN] = {} if not key in storage.data[self.DATAKEY_VPN]: storage.data[self.DATAKEY_VPN][key] = { 'active': {}, 'last': {} } item = storage.data[self.DATAKEY_VPN][key] # resolve remote addr to its netblock remote_raw = remote remote_resolved = None if not remote is None: remote_resolved = ffstatus.resolve_ipblock(remote) if not remote_resolved is None: self.logger.debug('Resolved IP \'{0}\' to block \'{1}\'.'.format(remote, remote_resolved['name'])) remote = remote_resolved if action == 'establish': item['active'][gw] = { 'establish': ts, 'peer': peername, 'remote': remote, 'remote_raw': remote_raw, } elif action == 'disestablish': active = {} if gw in item['active']: active = item['active'][gw] del(item['active'][gw]) active['disestablish'] = ts item['last'][gw] = active else: self.send_error(500, 'Unknown action not filtered (' + str(action) + ')') return self.send_headers('text/plain') self.wfile.write('OK') storage.save() def respond_vpnlist(self): storage = self.server.storage gateways = ['gw01','gw02','gw03','gw04','gw05','gw06'] self.send_headers() self.wfile.write('\n') self.wfile.write('BATCAVE - VPN LIST\n') self.wfile.write('\n') self.wfile.write('\n') self.wfile.write('\n\n') self.wfile.write('\n') self.wfile.write('\n') self.wfile.write('\n') if self.DATAKEY_VPN in storage.data: for key in storage.data[self.DATAKEY_VPN]: item = storage.data[self.DATAKEY_VPN][key] if not isinstance(item, dict): continue names = set() count = {} for t in [ 'active', 'last' ]: count[t] = 0 if t in item: for gw in item[t]: if 'remote' in item[t][gw] and len(item[t][gw]['remote']) > 0: count[t] += 1 if 'peer' in item[t][gw]: names.add(item[t][gw]['peer']) self.wfile.write('' if count['active'] > 0 else '') self.wfile.write('') for t in [ 'active', 'last' ]: for gw in gateways: ip = '' if t in item and gw in item[t]: ip = item[t][gw]['remote'] if 'remote' in item[t][gw] else '' if isinstance(ip, dict): ip = ip['name'] self.wfile.write('') self.wfile.write('\n') self.wfile.write('

names (key)	active	last
names (key)	' + '	'.join(gateways) + '	' + '	'.join(gateways) + '
' + (' / '.join(names) if len(names) > 0 else '?') + '	' + ('✓' if len(ip) > 0 else '×') + '

\n') self.wfile.write('') self.wfile.write('') def respond_providers(self, query): """Return a summary of providers.""" vpn = self.server.storage.data[self.DATAKEY_VPN] outputformat = query['format'].lower() if 'format' in query else 'html' isps = {} ispblocks = {} vpnstorage_updated = False vpnstorage_update_allowed = 'update' in query and query['update'] == 'allowed' for key in vpn: if key is None: continue item = vpn[key] if not isinstance(item, dict): continue if not 'active' in item: continue ips = [] for gw in item['active']: if 'remote' in item['active'][gw]: ip = item['active'][gw]['remote'] if vpnstorage_update_allowed and not isinstance(ip, dict): # try to resolve ip now resolved = ffstatus.resolve_ipblock(ip) if not resolved is None: self.logger.debug('Resolved IP \'{0}\' to block \'{1}\'.'.format(ip, resolved)) item['active'][gw]['remote'] = resolved vpnstorage_updated = True ip = resolved else: self.logger.debug('Failed to resolve IP \'{0}\'.'.format(ip)) ips.append(ip) if len(ips) == 0: # no active dialins -> no need to process this key any further continue item_isps = set() for ip in ips: isp = "UNKNOWN" ispblock = ip if isinstance(ip, dict): ispblock = ip['name'] desc_lines = ip['description'].split('\n') isp = desc_lines[0].strip() # normalize name: strip company indication isp = re.sub(r'(AG|UG|G?mbH( ?& ?Co\.? ?(OH|K)G)?)$', '', isp, flags=re.IGNORECASE).strip() # normalize name: strip "pool" suffixes isp = re.sub(r'(dynamic )?(customer |subscriber )?(ip )?(pool|(address )?range|addresses)$', '', isp, flags=re.IGNORECASE).strip() # normalize name: strip "B2B" and aggregation suffixes isp = re.sub(r'(aggregate|aggregation)?$', '', isp, flags=re.IGNORECASE).strip() isp = re.sub(r'(B2B)?$', '', isp, flags=re.IGNORECASE).strip() # normalize name: strip country suffixes (in Germany) isp = re.sub(r'(DE|Deutschland|Germany|Nordrhein[- ]Westfalen|NRW|Baden[- ]Wuerttemburg|BW|Hessen|Niedersachsen|Rheinland[- ]Pfalz|RLP)$', '', isp, flags=re.IGNORECASE).strip() isp = str(isp) if not isp in ispblocks: ispblocks[isp] = set() ispblocks[isp].add(ispblock) item_isps.add(isp) if len(item_isps) == 0: item_isps.add('unknown') elif len(item_isps) > 1: self.logger.warn('VPN key \'{0}\' has {1} active IPs which resolved to {2} ISPs: \'{3}\''.format(key, len(ips), len(item_isps), '\', \''.join(item_isps))) for isp in item_isps: if not isp in isps: isps[isp] = 0 isps[isp] += 1.0 / len(item_isps) isps_sum = sum([isps[x] for x in isps]) if vpnstorage_updated: self.server.storage.save() if outputformat == 'csv': self.send_headers('text/csv') self.wfile.write('Count;Name\n') for isp in isps: self.wfile.write('{0};"{1}"\n'.format(isps[isp], isp)) elif outputformat == 'json': self.send_headers('text/json') data = [ { 'name': isp, 'count': isps[isp], 'percentage': isps[isp]*100.0/isps_sum, 'blocks': [block for block in ispblocks[isp]] } for isp in isps ] self.wfile.write(json.dumps(data)) elif outputformat == 'html': self.send_headers() self.wfile.write('BATCAVE - PROVIDERS\n') self.wfile.write('\n') for isp in sorted(isps, key=lambda x: isps[x], reverse=True): self.wfile.write('\n'.format( isps[isp], isps[isp]*100.0/isps_sum, isp, ', '.join(sorted(ispblocks[isp])) if isp in ispblocks else '?', )) self.wfile.write('

Count	Percentage	Name	Blocks
{0}	{1:.1f}%	{2}	{3}

\n') self.wfile.write('

Totals: {0} ISPs, {1} connections

\n'.format(len(isps), isps_sum)) self.wfile.write('') else: self.send_error(400, 'Unknown output format.') class ApiServer(ThreadingMixIn, HTTPServer): def __init__(self, endpoint, storage): if ':' in endpoint[0]: self.address_family = socket.AF_INET6 HTTPServer.__init__(self, endpoint, BatcaveHttpRequestHandler) self.storage = storage # check all entries for a proper 'remote' entry vpn = storage.data[BatcaveHttpRequestHandler.DATAKEY_VPN] if BatcaveHttpRequestHandler.DATAKEY_VPN in storage.data else {} init_vpn_cache = {} for key in vpn: if not isinstance(vpn[key], dict): continue for mode in vpn[key]: if not isinstance(vpn[key][mode], dict): continue for gw in vpn[key][mode]: if not isinstance(vpn[key][mode][gw], dict): continue item = vpn[key][mode][gw] if 'remote' in item and not 'remote_raw' in item: item['remote_raw'] = item['remote'] resolved = None if item['remote'] in init_vpn_cache: resolved = init_vpn_cache[item['remote']] else: resolved = ffstatus.resolve_ipblock(item['remote']) init_vpn_cache[item['remote']] = resolved if not resolved is None: self.logger.info('Startup: resolved VPN entry \'{0}\' to net \'{1}\'.'.format(item['remote'], resolved['name'])) if not resolved is None: item['remote'] = resolved storage.save() def __str__(self): return 'ApiServer on {0}'.format(self.server_address) if __name__ == '__main__': dummystorage = Storage() server = ApiServer(('0.0.0.0', 8888), dummystorage) print("Server:", str(server)) server.serve_forever()