ffpb-archiv
/
BATCAVE


			
				
					
						
						
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466
							#!/usr/bin/python
# -*- coding: utf-8 -*-

from __future__ import print_function
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer

import cgi
from storage import Storage
import json
import logging
import pygeoip
import re
import socket
from SocketServer import ThreadingMixIn
import time

class BatcaveHttpRequestHandler(BaseHTTPRequestHandler):
	DATAKEY_VPN = '__VPN__'

	def __init__(self, request, client_address, server):
		self.logger = logging.getLogger('API')
		BaseHTTPRequestHandler.__init__(self, request, client_address, server)

	def parse_url_pathquery(self):
		"""Extracts the query parameters from the request path."""
		url = re.match(r'^/(?P<path>.*?)(\?(?P<query>.+))?$', self.path.strip())
		if url is None:
			logging.warn('Failed to parse URL \'' + str(self.path) + '\'.')
			return ( None, None )

		path = url.group('path')
		query = {}
		if not url.group('query') is None:
			for m in re.finditer(r'(?P<key>.+?)=(?P<value>.+?)(&|$)', url.group('query')):
				query[m.group('key')] = m.group('value')
		return ( path, query )

	def do_GET(self):
		"""Handles all HTTP GET requests."""

		path, query = self.parse_url_pathquery()
		if path is None:
			self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
                        return

		# / - index page, shows generic help
		if path == '':
			self.respond_index(query)
			return

		# /list - list stored nodes
		if path == 'list':
			self.respond_list(query)
			return

		# /vpn - notification endpoint for gateway's VPN connections
		if path == 'vpn':
			self.respond_vpn(query)
			return

		# /providers
		if path == 'providers':
			self.respond_providers(query)
			return

		# /node/<id>.json - node's data
		# /node/<id>/field - return specific field from node's data
		m = re.match(r'node/([a-f0-9]{12})(?P<cmd>\.json|/[a-zA-Z0-9_\-]+)$', path)
		if m != None:
			cmd = m.group('cmd')
			if cmd == '.json':
				self.respond_node(m.group(1))
			else:
				self.respond_nodedetail(m.group(1), cmd[1:])
			return

		# no match -> 404
		self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))

	def do_POST(self):
		"""Handles all HTTP POST requests."""

		path, query = self.parse_url_pathquery()
		if path is None:
			self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
                        return
		params = self.parse_post_params()

		# node id/mac to name mapping
		if path == 'idmac2name':
			self.respond_nodeidmac2name(params)
			return

		# no match -> 404
		self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))

	def send_nocache_headers(self):
		"""Sets HTTP headers indicating that this response shall not be cached."""

		self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
		self.send_header('Pragma', 'no-cache')
		self.send_header('Expires', '0')

	def send_headers(self, content_type='text/html; charset=utf-8', nocache=True):
		"""Send HTTP 200 Response header with the given Content-Type.
		Optionally send no-caching headers, too."""

		self.send_response(200)
		self.send_header('Content-Type', content_type)
		if nocache: self.send_nocache_headers()
		self.end_headers()

	def parse_post_params(self):
		ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
		if ctype == 'multipart/form-data':
			postvars = cgi.parse_multipart(self.rfile, pdict)
		elif ctype == 'application/x-www-form-urlencoded':
			length = int(self.headers.getheader('content-length'))
			postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1)
		else:
			postvars = {}
		return postvars

	def respond_index(self, query):
		"""Display the index page."""

		storage = self.server.storage
		self.send_headers()

		self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE</title></head>\n')
		self.wfile.write('<body>\n')
		self.wfile.write('<H1 title="Batman/Alfred Transmission Collection, Aggregation & Value Engine">BATCAVE</H1>\n')

		self.wfile.write('<p>Dies ist ein interner Hintergrund-Dienst. Er wird nur von anderen Diensten\n')
		self.wfile.write('angesprochen und sollte aus einer Mehrzahl von Gr&uuml;nden nicht &ouml;ffentlich\n')
		self.wfile.write('zug&auml;nglich sein.</p>\n')

		self.wfile.write('<H2>Status</H2>\n')
		self.wfile.write('Daten: <span id="datacount" class="value">')
		self.wfile.write(len(storage.data))
		self.wfile.write('</span>\n')

		self.wfile.write('<H2>API</H2>\n')
		self.wfile.write('<p>Grundsätzlich ist das Antwort-Format JSON und alle Daten sind Live-Daten (kein Cache) die ggf. etwas Bearbeitungs-Zeit erfordern.</p>')
		self.wfile.write('<dl>\n')
		self.wfile.write('<dt><a href="/nodes.json">nodes.json</a></dt><dd>zur Verwendung mit ffmap (MACs anonymisiert)</dd>\n')
		self.wfile.write('<dt><a href="/node/ff00ff00ff00.json">/node/&lt;id&gt;.json</a></dt><dd><u>alle</u> vorhandenen Information zu der gewünschten Node</dd>\n')
		self.wfile.write('</dl>\n')
		self.wfile.write('</body></html>')

	def respond_list(self, query):
		"""List stored data."""

		storage = self.server.storage
		self.send_headers()

		self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE</title></head>\n')
		self.wfile.write('<body>\n')
		self.wfile.write('<H1>BATCAVE - LIST</H1>\n')

		self.wfile.write('<table>\n')
		self.wfile.write('<thead><tr><th>ID</th><th>Name</th></tr></thead>\n')
		self.wfile.write('<tbody>\n')

		data = storage.data
		if 'sort' in query:
			if query['sort'] == 'name':
				sorteddata = sorted(data, key=lambda x: data[x]['hostname'].lower())
				data = sorteddata
			elif query['sort'] == 'id':
				sorteddata = sorted(data)
				data = sorteddata

		for nodeid in data:
			if nodeid.startswith('__'): continue
			nodename = storage.data[nodeid]['hostname'] if 'hostname' in storage.data[nodeid] else '&lt;?&gt;'
			self.wfile.write('<tr><td><a href="/node/' + nodeid + '.json">' + nodeid + '</a></td><td>' + nodename + '</td></tr>')

		self.wfile.write('</tbody>\n')
		self.wfile.write('</table>\n')

	def find_node(self, rawid):
		"""Fetch node data from storage by given id, if necessary looking thorugh node aliases."""

		storage = self.server.storage

		# if we have a direct hit, return it immediately
		if rawid in storage.data:
			return storage.data[rawid]

		# no direct hit -> search via aliases
		nodeid = rawid
		for n in storage.data:
			if 'aliases' in storage.data[n] and rawid in storage.data[n]['aliases']:
				nodeid = n

		# return found node
		return storage.data[nodeid] if nodeid in storage.data else None

	def find_node_by_mac(self, mac):
		"""Fetch node data from storage by given MAC address."""

		storage = self.server.storage
		needle = mac.lower()

		# iterate over all nodes
		for nodeid in storage.data:
			if nodeid.startswith('__'): continue
			node = storage.data[nodeid]

			# check node's primary MAC
			if 'mac' in node and needle == node['mac'].lower():
				return node

			# check alias MACs
			if 'macs' in node:
				haystack = [ x.lower() for x in node['macs'] ]
				if mac in haystack:
					return node

		# MAC address not found
		return None

	def respond_node(self, rawid):
		"""Display node data."""

		# handle API example linked on index page
		if rawid == 'ff00ff00ff00':
			self.send_headers('text/json')
			self.wfile.write(json.dumps({
				'name': 'API-Example',
				'nodeid': rawid,
				'META': 'Dies ist ein minimaler Beispiel-Datensatz. Herzlichen Glückwunsch, du hast das Prinzip der API kapiert.',
			}))
			return

		# search node by the given id
		node = self.find_node(rawid)

		# handle unknown nodes
		if node is None:
			self.send_error(404, 'No node with id \'' + rawid + '\' present.')
			return

		# dump node data as JSON
		self.send_headers('text/json')
		self.wfile.write(json.dumps(node))

	def respond_nodeidmac2name(self, ids):
		storage = self.server.storage
		
		self.send_headers('text/plain')
		for nodeid in ids:
			node = self.find_node(nodeid) if not ':' in nodeid else self.find_node_by_mac(nodeid)
			nodename = node['hostname'] if (not node is None) and 'hostname' in node else nodeid
			self.wfile.write('{0}={1}\n'.format(nodeid, nodename))
			
	def respond_nodedetail(self, nodeid, field):
		"""Return a field from the given node - a string is returned as text, all other as JSON."""

		node = self.find_node(nodeid)
		if node is None:
			self.send_error(404, 'No node with id \'' + nodeid + '\' present.')
			return

		if not field in node:
			self.send_error(404, 'The node \'' + nodeid + '\' does not have a field named \'' + str(field) + '\'.')
			return

		value = node[field]

		self.send_headers('text/plain' if isinstance(value, basestring) else 'text/json')
		self.wfile.write(value if isinstance(value, basestring) else json.dumps(value))

	def respond_vpn(self, query):
		storage = self.server.storage
		peername = query['peer'] if 'peer' in query else None
		key = query['key'] if 'key' in query else None
		action = query['action'] if 'action' in query else None
		remote = query['remote'] if 'remote' in query else None
		gw = query['gw'] if 'gw' in query else None

		if action == 'list':
			self.respond_vpnlist()
			return

		if action != 'establish' and action != 'disestablish':
			self.logger.error('VPN: unknown action \'{0}\''.format(action))
			self.send_error(400, 'Invalid action.')
			return

		for k,v in { 'peername': peername, 'key': key, 'remote': remote, 'gw': gw }.items():
			if v is None or len(v.strip()) == 0:
				self.logger.error('VPN {0}: no or empty {1}'.format(action, k))
				self.send_error(400, 'Missing value for ' + str(k))
				return

		if key is None or re.match(r'^[a-fA-F0-9]+$', key) is None:
			self.logger.error('VPN peer \'{0}\' {1}: bad key \'{2}\''.format(peername, action, key))
			self.send_error(400, 'Bad key.')
			return

		if not self.DATAKEY_VPN in storage.data: storage.data[self.DATAKEY_VPN] = {}
		if not key in storage.data[self.DATAKEY_VPN]: storage.data[self.DATAKEY_VPN][key] = { 'active': {}, 'last': {} }
		item = storage.data[self.DATAKEY_VPN][key]

		if action == 'establish':
			item['active'][gw] = { 'establish': time.time(), 'peer': peername, 'remote': remote }
		elif action == 'disestablish':
			active = {}
			if gw in item['active']:
				active = item['active'][gw]
				del(item['active'][gw])
			active['disestablish'] = time.time()
			item['last'][gw] = active
		else:
			self.send_error(500, 'Unknown action not filtered (' + str(action) + ')')
			return

		self.send_headers('text/plain')
		self.wfile.write('OK')

		storage.save()

	def respond_vpnlist(self):
		storage = self.server.storage

		gateways = ['gw01','gw02','gw03','gw04']

		self.send_headers()
		self.wfile.write('<!DOCTYPE html>\n')
		self.wfile.write('<html><head><title>BATCAVE - VPN LIST</title></head>\n')
		self.wfile.write('<body>\n')
		self.wfile.write('<style type="text/css">\n')
		self.wfile.write('table { border: 2px solid #999; border-collapse: collapse; }\n')
		self.wfile.write('th, td { border: 1px solid #CCC; }\n')
		self.wfile.write('table tbody tr.online { background-color: #CFC; }\n')
		self.wfile.write('table tbody tr.offline { background-color: #FCC; }\n')
		self.wfile.write('</style>\n')
		self.wfile.write('<table>\n<thead>\n')
		self.wfile.write('<tr><th rowspan="2">names (key)</th><th colspan="' + str(len(gateways)) + '">active</th><th colspan="' + str(len(gateways)) + '">last</th></tr>\n')
		self.wfile.write('<tr><th>' + '</th><th>'.join(gateways) + '</th><th>' + '</th><th>'.join(gateways) + '</th></tr>\n')
		self.wfile.write('</thead>\n')

		if self.DATAKEY_VPN in storage.data:
			for key in storage.data[self.DATAKEY_VPN]:
				item = storage.data[self.DATAKEY_VPN][key]

				names = set()
				count = {}
				for t in [ 'active', 'last' ]:
					count[t] = 0
					if t in item:
						for gw in item[t]:
							if 'remote' in item[t][gw] and len(item[t][gw]['remote']) > 0:
								count[t] += 1
							if 'peer' in item[t][gw]:
								names.add(item[t][gw]['peer'])

				self.wfile.write('<tr class="online">' if count['active'] > 0 else '<tr class="offline">')
				self.wfile.write('<td title="' + str(key) + '">' + (' / '.join(names) if len(names) > 0 else '?') + '</td>')
				for t in [ 'active', 'last' ]:
					for gw in gateways:
						ip = ''
						details = ''
						if t in item and gw in item[t]:
							ip = item[t][gw]['remote'] if 'remote' in item[t][gw] else ''
						self.wfile.write('<td>' + ip + '</td>')

				self.wfile.write('</tr>\n')

		self.wfile.write('</table>\n')
		self.wfile.write('</body>')
		self.wfile.write('</html>')

	def respond_providers(self, query):
		"""Return a summary of providers."""

		vpn = self.server.storage.data[self.DATAKEY_VPN]
		outputformat = query['format'].lower() if 'format' in query else 'html'

		geo = None
		try:
			geo = pygeoip.GeoIP('GeoIPISP.dat')
		except:
			self.return_error(500, 'The GeoIP-ISP database file could not be opened.')
			return

		isps = {}
		for key in vpn:
			if key is None: continue
			item = vpn[key]
			if not 'active' in item: continue

			ips = set()
			for gw in item['active']:
				if 'remote' in item['active'][gw]:
					ips.add(item['active'][gw]['remote'])

			if len(ips) == 0: continue
			if len(ips) > 1:
				self.logger.warn('VPN key \'{0}\' has {1} active ips. Possible cause is an in-progress re-dialin.'.format(key, len(ips)))

			item_isps = set()
			for ip in ips:
				try:
					isp = geo.org_by_addr(ip)
					if not isp is None: item_isps.add(isp)
				except Exception as err:
					self.logger.debug('Failed to resolve ISP for \'{0}\': {1}'.format(ip, str(err)))

			if len(item_isps) == 0:
				item_isps.add('unknown')

			elif len(item_isps) > 1:
				self.logger.warn('VPN key \'{0}\' has {1} active IPs which resolved to {2} ISPs: \'{3}\''.format(key, len(ips), len(item_isps), '\', \''.join(item_isps)))

			for isp in item_isps:
				if not isp in isps: isps[isp] = 0
				isps[isp] += 1

		isps_sum = sum([isps[x] for x in isps])

		if outputformat == 'csv':
			self.send_headers('text/csv')

			self.wfile.write('Count;Name\n')
			for isp in isps:
				self.wfile.write('{0};"{1}"\n'.format(isps[isp], isp))

		elif outputformat == 'json':
			self.send_headers('text/json')

			data = [ { 'name': x, 'count': isps[x], 'percentage': isps[x]*100.0/isps_sum } for x in isps ]
			self.wfile.write(json.dumps(data))

		elif outputformat == 'html':
			self.send_headers()

			self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE - PROVIDERS</title></head><body>\n')
			self.wfile.write('<table border="2"><thead><tr><th>Count</th><th>Percentage</th><th>Name</th></tr></thead><tbody>\n')

			for isp in isps:
				self.wfile.write('<tr><td>{0}</td><td>{1:.1f}%</td><td>{2}</td></tr>\n'.format(isps[isp], isps[isp]*100.0/isps_sum, isp))

			self.wfile.write('</tbody></table>\n')
			self.wfile.write('</body></html>')

		else:
			self.send_error(400, 'Unknown output format.')

class ApiServer(ThreadingMixIn, HTTPServer):
	def __init__(self, endpoint, storage):
		if ':' in endpoint[0]: self.address_family = socket.AF_INET6
		HTTPServer.__init__(self, endpoint, BatcaveHttpRequestHandler)
		self.storage = storage

	def __str__(self):
		return 'ApiServer on {0}'.format(self.server_address)

if __name__ == '__main__':
	dummystorage = Storage()
	server = ApiServer(('0.0.0.0', 8888), dummystorage)

	print("Server:", str(server))
	server.serve_forever()