ffpb-archiv
/
BATCAVE


			
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607
							#!/usr/bin/python
# -*- coding: utf-8 -*-

from __future__ import print_function
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer

import cgi
from storage import Storage
import json
import logging
import pygeoip
import re
import socket
from SocketServer import ThreadingMixIn
import time

import ffstatus

class BatcaveHttpRequestHandler(BaseHTTPRequestHandler):
	DATAKEY_VPN = '__VPN__'
	FIELDKEY_UPDATED = '__UPDATED__'

	def __init__(self, request, client_address, server):
		self.logger = logging.getLogger('API')
		BaseHTTPRequestHandler.__init__(self, request, client_address, server)

	def parse_url_pathquery(self):
		"""Extracts the query parameters from the request path."""
		url = re.match(r'^/(?P<path>.*?)(\?(?P<query>.+))?$', self.path.strip())
		if url is None:
			logging.warn('Failed to parse URL \'' + str(self.path) + '\'.')
			return ( None, None )

		path = url.group('path')
		query = {}
		if not url.group('query') is None:
			for m in re.finditer(r'(?P<key>.+?)=(?P<value>.+?)(&|$)', url.group('query')):
				query[m.group('key')] = m.group('value')
		return ( path, query )

	def do_GET(self):
		"""Handles all HTTP GET requests."""

		path, query = self.parse_url_pathquery()
		if path is None:
			self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
                        return

		# / - index page, shows generic help
		if path == '':
			self.respond_index(query)
			return

		# /list - list stored nodes
		if path == 'list':
			self.respond_list(query)
			return

		# /vpn - notification endpoint for gateway's VPN connections
		if path == 'vpn':
			self.respond_vpn(query)
			return

		# /providers
		if path == 'providers':
			self.respond_providers(query)
			return

		# /node/<id>.json - node's data
		# /node/<id>/field - return specific field from node's data
		m = re.match(r'node/(?P<id>[a-fA-F0-9]{12})(?P<cmd>\.json|/[a-zA-Z0-9_\-\.]+)$', path)
		if m != None:
			cmd = m.group('cmd')
			nodeid = m.group('id').lower()
			if cmd == '.json':
				self.respond_node(nodeid)
			else:
				self.respond_nodedetail(nodeid, cmd[1:])
			return

		# /status/<id> - node's status
		m = re.match(r'status/([a-f0-9]{12})$', path)
		if m != None:
			self.respond_nodestatus(m.group(1))
			return

		# no match -> 404
		self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))

	def do_POST(self):
		"""Handles all HTTP POST requests."""

		path, query = self.parse_url_pathquery()
		if path is None:
			self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
                        return
		params = self.parse_post_params()

		# node id/mac to name mapping
		if path == 'idmac2name':
			self.respond_nodeidmac2name(params)
			return

		# no match -> 404
		self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))

	def send_nocache_headers(self):
		"""Sets HTTP headers indicating that this response shall not be cached."""

		self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
		self.send_header('Pragma', 'no-cache')
		self.send_header('Expires', '0')

	def send_headers(self, content_type='text/html; charset=utf-8', nocache=True):
		"""Send HTTP 200 Response header with the given Content-Type.
		Optionally send no-caching headers, too."""

		self.send_response(200)
		self.send_header('Content-Type', content_type)
		if nocache: self.send_nocache_headers()
		self.end_headers()

	def parse_post_params(self):
		ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
		if ctype == 'multipart/form-data':
			postvars = cgi.parse_multipart(self.rfile, pdict)
		elif ctype == 'application/x-www-form-urlencoded':
			length = int(self.headers.getheader('content-length'))
			postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1)
		else:
			postvars = {}
		return postvars

	def respond_index(self, query):
		"""Display the index page."""

		storage = self.server.storage
		self.send_headers()

		self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE</title></head>\n')
		self.wfile.write('<body>\n')
		self.wfile.write('<H1 title="Batman/Alfred Transmission Collection, Aggregation & Value Engine">BATCAVE</H1>\n')

		self.wfile.write('<p>Dies ist ein interner Hintergrund-Dienst. Er wird nur von anderen Diensten\n')
		self.wfile.write('angesprochen und sollte aus einer Mehrzahl von Gr&uuml;nden nicht &ouml;ffentlich\n')
		self.wfile.write('zug&auml;nglich sein.</p>\n')

		self.wfile.write('<H2>Status</H2>\n')
		self.wfile.write('Daten: <span id="datacount" class="value">')
		self.wfile.write(len(storage.data))
		self.wfile.write('</span>\n')

		self.wfile.write('<H2>API</H2>\n')
		self.wfile.write('<p>Grundsätzlich ist das Antwort-Format JSON und alle Daten sind Live-Daten (kein Cache) die ggf. etwas Bearbeitungs-Zeit erfordern.</p>')
		self.wfile.write('<dl>\n')
		self.wfile.write('<dt><a href="/nodes.json">nodes.json</a></dt><dd>zur Verwendung mit ffmap (MACs anonymisiert)</dd>\n')
		self.wfile.write('<dt><a href="/node/ff00ff00ff00.json">/node/&lt;id&gt;.json</a></dt><dd><u>alle</u> vorhandenen Information zu der gewünschten Node</dd>\n')
		self.wfile.write('</dl>\n')
		self.wfile.write('</body></html>')

	def respond_list(self, query):
		"""List stored data."""

		storage = self.server.storage
		self.send_headers()

		self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE</title></head>\n')
		self.wfile.write('<body>\n')
		self.wfile.write('<H1>BATCAVE - LIST</H1>\n')

		self.wfile.write('<table>\n')
		self.wfile.write('<thead><tr><th>ID</th><th>Name</th></tr></thead>\n')
		self.wfile.write('<tbody>\n')

		data = storage.data
		if 'sort' in query:
			if query['sort'] == 'name':
				sorteddata = sorted(data, key=lambda x: data[x]['hostname'].lower())
				data = sorteddata
			elif query['sort'] == 'id':
				sorteddata = sorted(data)
				data = sorteddata

		for nodeid in data:
			if nodeid.startswith('__'): continue
			nodename = storage.data[nodeid]['hostname'] if 'hostname' in storage.data[nodeid] else '&lt;?&gt;'
			self.wfile.write('<tr><td><a href="/node/' + nodeid + '.json">' + nodeid + '</a></td><td>' + nodename + '</td></tr>')

		self.wfile.write('</tbody>\n')
		self.wfile.write('</table>\n')

	def find_node(self, rawid):
		"""Fetch node data from storage by given id, if necessary looking thorugh node aliases."""

		storage = self.server.storage

		# if we have a direct hit, return it immediately
		if rawid in storage.data:
			return storage.data[rawid]

		# no direct hit -> search via aliases
		nodeid = rawid
		for n in storage.data:
			if 'aliases' in storage.data[n] and rawid in storage.data[n]['aliases']:
				nodeid = n

		# return found node
		return storage.data[nodeid] if nodeid in storage.data else None

	def find_node_by_mac(self, mac):
		"""Fetch node data from storage by given MAC address."""

		storage = self.server.storage
		needle = mac.lower()

		# iterate over all nodes
		for nodeid in storage.data:
			if nodeid.startswith('__'): continue
			node = storage.data[nodeid]

			# check node's primary MAC
			if 'mac' in node and needle == node['mac'].lower():
				return node

			# check alias MACs
			if 'macs' in node:
				haystack = [ x.lower() for x in node['macs'] ]
				if mac in haystack:
					return node

		# MAC address not found
		return None

	def respond_node(self, rawid):
		"""Display node data."""

		# handle API example linked on index page
		if rawid == 'ff00ff00ff00':
			self.send_headers('text/json')
			self.wfile.write(json.dumps({
				'name': 'API-Example',
				'nodeid': rawid,
				'META': 'Dies ist ein minimaler Beispiel-Datensatz. Herzlichen Glückwunsch, du hast das Prinzip der API kapiert.',
			}))
			return

		# search node by the given id
		node = self.find_node(rawid)

		# handle unknown nodes
		if node is None:
			self.send_error(404, 'No node with id \'' + rawid + '\' present.')
			return

		# remove fields from output: __RAW__
		export = ffstatus.dict_merge({}, node)
		if '__RAW__' in export:
			del(export['__RAW__'])

		# dump node data as JSON
		self.send_headers('text/json')
		self.wfile.write(json.dumps(export))

	def get_nodestatus(self, rawid):
		"""Determine node's status."""

		# search node by the given id
		node = self.find_node(rawid)

		# handle unknown nodes
		if node is None:
			return None

		# check that the last batadv update is noted in the data
		updated = node[self.FIELDKEY_UPDATED] if self.FIELDKEY_UPDATED in node else None
		if updated is None or not 'batadv' in updated:
			return 'unknown'

		# make decision based on time of last batadv update
		diff = time.time() - updated['batadv']
		if diff < 150:
			return 'active'
		elif diff < 300:
			return 'stale'
		else:
			return 'offline'

	def respond_nodestatus(self, rawid):
		"""Display node status."""

		status = self.get_nodestatus(rawid)

		if status is None:
			self.send_error(404, 'No node with id \'' + rawid + '\' present.')

		self.send_headers('text/plain')
		self.wfile.write(status)

	def respond_nodeidmac2name(self, ids):
		"""Return a mapping of the given IDs (or MACs) into their hostname."""

		self.send_headers('text/plain')
		for nodeid in ids:
			node = self.find_node(nodeid) if not ':' in nodeid else self.find_node_by_mac(nodeid)
			nodename = node['hostname'] if (not node is None) and 'hostname' in node else nodeid
			self.wfile.write('{0}={1}\n'.format(nodeid, nodename))
			
	def respond_nodedetail(self, nodeid, field):
		"""Return a field from the given node - a string is returned as text, all other as JSON."""

		node = self.find_node(nodeid)
		if node is None:
			self.send_error(404, 'No node with id \'' + nodeid + '\' present.')
			return

		return_count = False
		if field.endswith('.count'):
			return_count = True
			field = field[0:-6]

		if not field in node:
			self.send_error(404, 'The node \'' + nodeid + '\' does not have a field named \'' + str(field) + '\'.')
			return

		value = node[field]
		if return_count: value = len(value)

		self.send_headers('text/plain' if isinstance(value, basestring) or isinstance(value, int) else 'text/json')
		self.wfile.write(value if isinstance(value, basestring) else json.dumps(value))

	def respond_vpn(self, query):
		storage = self.server.storage
		peername = query['peer'] if 'peer' in query else None
		key = query['key'] if 'key' in query else None
		action = query['action'] if 'action' in query else None
		remote = query['remote'] if 'remote' in query else None
		gw = query['gw'] if 'gw' in query else None
		ts = query['ts'] if 'ts' in query else time.time()

		if action == 'list':
			self.respond_vpnlist()
			return

		if action != 'establish' and action != 'disestablish':
			self.logger.error('VPN: unknown action \'{0}\''.format(action))
			self.send_error(400, 'Invalid action.')
			return

		for k,v in { 'peername': peername, 'key': key, 'remote': remote, 'gw': gw }.items():
			if v is None or len(v.strip()) == 0:
				self.logger.error('VPN {0}: no or empty {1}'.format(action, k))
				self.send_error(400, 'Missing value for ' + str(k))
				return

		if key is None or re.match(r'^[a-fA-F0-9]+$', key) is None:
			self.logger.error('VPN peer \'{0}\' {1}: bad key \'{2}\''.format(peername, action, key))
			self.send_error(400, 'Bad key.')
			return

		if not self.DATAKEY_VPN in storage.data: storage.data[self.DATAKEY_VPN] = {}
		if not key in storage.data[self.DATAKEY_VPN]: storage.data[self.DATAKEY_VPN][key] = { 'active': {}, 'last': {} }
		item = storage.data[self.DATAKEY_VPN][key]

		# resolve remote addr to its netblock
		remote_raw = remote
		remote_resolved = None
		if not remote is None:
			remote_resolved = ffstatus.resolve_ipblock(remote)
			if not remote_resolved is None:
				self.logger.debug('Resolved IP \'{0}\' to block \'{1}\'.'.format(remote, remote_resolved['name']))
				remote = remote_resolved

		if action == 'establish':
			item['active'][gw] = {
				'establish': ts,
				'peer': peername,
				'remote': remote,
				'remote_raw': remote_raw,
			}

		elif action == 'disestablish':
			active = {}
			if gw in item['active']:
				active = item['active'][gw]
				del(item['active'][gw])
			active['disestablish'] = ts
			item['last'][gw] = active
		else:
			self.send_error(500, 'Unknown action not filtered (' + str(action) + ')')
			return

		self.send_headers('text/plain')
		self.wfile.write('OK')

		storage.save()

	def respond_vpnlist(self):
		storage = self.server.storage

		gateways = ['gw01','gw02','gw03','gw04','gw05','gw06']

		self.send_headers()
		self.wfile.write('<!DOCTYPE html>\n')
		self.wfile.write('<html><head><title>BATCAVE - VPN LIST</title></head>\n')
		self.wfile.write('<body>\n')
		self.wfile.write('<style type="text/css">\n')
		self.wfile.write('table { border: 2px solid #999; border-collapse: collapse; }\n')
		self.wfile.write('th, td { border: 1px solid #CCC; }\n')
		self.wfile.write('table tbody tr.online { background-color: #CFC; }\n')
		self.wfile.write('table tbody tr.offline { background-color: #FCC; }\n')
		self.wfile.write('</style>\n')
		self.wfile.write('<table>\n<thead>\n')
		self.wfile.write('<tr><th rowspan="2">names (key)</th><th colspan="' + str(len(gateways)) + '">active</th><th colspan="' + str(len(gateways)) + '">last</th></tr>\n')
		self.wfile.write('<tr><th>' + '</th><th>'.join(gateways) + '</th><th>' + '</th><th>'.join(gateways) + '</th></tr>\n')
		self.wfile.write('</thead>\n')

		if self.DATAKEY_VPN in storage.data:
			for key in storage.data[self.DATAKEY_VPN]:
				item = storage.data[self.DATAKEY_VPN][key]
				if not isinstance(item, dict):
					continue

				names = set()
				count = {}
				for t in [ 'active', 'last' ]:
					count[t] = 0
					if t in item:
						for gw in item[t]:
							if 'remote' in item[t][gw] and len(item[t][gw]['remote']) > 0:
								count[t] += 1
							if 'peer' in item[t][gw]:
								names.add(item[t][gw]['peer'])

				self.wfile.write('<tr class="online">' if count['active'] > 0 else '<tr class="offline">')
				self.wfile.write('<td title="' + str(key) + '">' + (' / '.join(names) if len(names) > 0 else '?') + '</td>')
				for t in [ 'active', 'last' ]:
					for gw in gateways:
						ip = ''
						if t in item and gw in item[t]:
							ip = item[t][gw]['remote'] if 'remote' in item[t][gw] else ''
							if isinstance(ip, dict):
								ip = ip['name']
						self.wfile.write('<td title="' + ip + '">' + ('&check;' if len(ip) > 0 else '&times;') + '</td>')

				self.wfile.write('</tr>\n')

		self.wfile.write('</table>\n')
		self.wfile.write('</body>')
		self.wfile.write('</html>')

	def respond_providers(self, query):
		"""Return a summary of providers."""

		vpn = self.server.storage.data[self.DATAKEY_VPN]
		outputformat = query['format'].lower() if 'format' in query else 'html'

		isps = {}
		ispblocks = {}
		vpnstorage_updated = False
		vpnstorage_update_allowed = 'update' in query and query['update'] == 'allowed'
		for key in vpn:
			if key is None: continue
			item = vpn[key]
			if not isinstance(item, dict): continue
			if not 'active' in item: continue

			ips = []
			for gw in item['active']:
				if 'remote' in item['active'][gw]:
					ip = item['active'][gw]['remote']
					if vpnstorage_update_allowed and not isinstance(ip, dict):
						# try to resolve ip now
						resolved = ffstatus.resolve_ipblock(ip)
						if not resolved is None:
							self.logger.debug('Resolved IP \'{0}\' to block \'{1}\'.'.format(ip, resolved))
							item['active'][gw]['remote'] = resolved
							vpnstorage_updated = True
							ip = resolved
						else:
							self.logger.debug('Failed to resolve IP \'{0}\'.'.format(ip))
					ips.append(ip)

			if len(ips) == 0:
				# no active dialins -> no need to process this key any further
				continue

			item_isps = set()
			for ip in ips:
				isp = "UNKNOWN"
				ispblock = ip
				if isinstance(ip, dict):
					ispblock = ip['name']
					desc_lines = ip['description'].split('\n')
					isp = desc_lines[0].strip()

					# normalize name: strip company indication
					isp = re.sub(r'(AG|UG|G?mbH( ?& ?Co\.? ?(OH|K)G)?)$', '', isp, flags=re.IGNORECASE).strip()

					# normalize name: strip "pool" suffixes
					isp = re.sub(r'(dynamic )?(customer |subscriber )?(ip )?(pool|(address )?range|addresses)$', '', isp, flags=re.IGNORECASE).strip()

					# normalize name: strip "B2B" and aggregation suffixes
					isp = re.sub(r'(aggregate|aggregation)?$', '', isp, flags=re.IGNORECASE).strip()
					isp = re.sub(r'(B2B)?$', '', isp, flags=re.IGNORECASE).strip()

					# normalize name: strip country suffixes (in Germany)
					isp = re.sub(r'(DE|Deutschland|Germany|Nordrhein[- ]Westfalen|NRW|Baden[- ]Wuerttemburg|BW|Hessen|Niedersachsen|Rheinland[- ]Pfalz|RLP)$', '', isp, flags=re.IGNORECASE).strip()

				isp = str(isp)
				if not isp in ispblocks:
					ispblocks[isp] = set()
				ispblocks[isp].add(ispblock)

				item_isps.add(isp)

			if len(item_isps) == 0:
				item_isps.add('unknown')

			elif len(item_isps) > 1:
				self.logger.warn('VPN key \'{0}\' has {1} active IPs which resolved to {2} ISPs: \'{3}\''.format(key, len(ips), len(item_isps), '\', \''.join(item_isps)))

			for isp in item_isps:
				if not isp in isps: isps[isp] = 0
				isps[isp] += 1.0 / len(item_isps)

		isps_sum = sum([isps[x] for x in isps])

		if vpnstorage_updated:
			self.server.storage.save()
 
		if outputformat == 'csv':
			self.send_headers('text/csv')

			self.wfile.write('Count;Name\n')
			for isp in isps:
				self.wfile.write('{0};"{1}"\n'.format(isps[isp], isp))

		elif outputformat == 'json':
			self.send_headers('text/json')

			data = [ { 'name': isp, 'count': isps[isp], 'percentage': isps[isp]*100.0/isps_sum, 'blocks': [block for block in ispblocks[isp]] } for isp in isps ]
			self.wfile.write(json.dumps(data))

		elif outputformat == 'html':
			self.send_headers()

			self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE - PROVIDERS</title></head><body>\n')
			self.wfile.write('<table border="2"><thead><tr><th>Count</th><th>Percentage</th><th>Name</th><th>Blocks</th></tr></thead><tbody>\n')

			for isp in sorted(isps, key=lambda x: isps[x], reverse=True):
				self.wfile.write('<tr><td>{0}</td><td>{1:.1f}%</td><td>{2}</td><td>{3}</td></tr>\n'.format(
					isps[isp],
					isps[isp]*100.0/isps_sum,
					isp,
					', '.join(sorted(ispblocks[isp])) if isp in ispblocks else '?',
				))

			self.wfile.write('</tbody></table>\n')
			self.wfile.write('<p>Totals: {0} ISPs, {1} connections</p>\n'.format(len(isps), isps_sum))
			self.wfile.write('</body></html>')

		else:
			self.send_error(400, 'Unknown output format.')

class ApiServer(ThreadingMixIn, HTTPServer):
	def __init__(self, endpoint, storage):
		if ':' in endpoint[0]: self.address_family = socket.AF_INET6
		HTTPServer.__init__(self, endpoint, BatcaveHttpRequestHandler)
		self.storage = storage

		# check all entries for a proper 'remote' entry
		vpn = storage.data[BatcaveHttpRequestHandler.DATAKEY_VPN] if BatcaveHttpRequestHandler.DATAKEY_VPN in storage.data else {}
		init_vpn_cache = {}
		for key in vpn:
			if not isinstance(vpn[key], dict):
				continue
			for mode in vpn[key]:
				if not isinstance(vpn[key][mode], dict):
					continue
				for gw in vpn[key][mode]:
					if not isinstance(vpn[key][mode][gw], dict):
						continue
					item = vpn[key][mode][gw]
					if 'remote' in item and not 'remote_raw' in item:
						item['remote_raw'] = item['remote']
						resolved = None
						if item['remote'] in init_vpn_cache:
							resolved = init_vpn_cache[item['remote']]
						else:
							resolved = ffstatus.resolve_ipblock(item['remote'])
							init_vpn_cache[item['remote']] = resolved
							if not resolved is None:
								self.logger.info('Startup: resolved VPN entry \'{0}\' to net \'{1}\'.'.format(item['remote'], resolved['name']))
						if not resolved is None:
							item['remote'] = resolved
		storage.save()


	def __str__(self):
		return 'ApiServer on {0}'.format(self.server_address)

if __name__ == '__main__':
	dummystorage = Storage()
	server = ApiServer(('0.0.0.0', 8888), dummystorage)

	print("Server:", str(server))
	server.serve_forever()