BATCAVE/ffstatus/server.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

from __future__ import print_function, unicode_literals
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer

import cgi
import json
import logging
import re
import socket
from SocketServer import ThreadingMixIn
import time

import ffstatus

class BatcaveHttpRequestHandler(BaseHTTPRequestHandler):

    def __init__(self, request, client_address, server):
        self.logger = logging.getLogger('API')
        BaseHTTPRequestHandler.__init__(self, request, client_address, server)

    def parse_url_pathquery(self):
        """Extracts the query parameters from the request path."""
        url = re.match(r'^/(?P<path>.*?)(\?(?P<query>.+))?$', self.path.strip())
        if url is None:
            logging.warn('Failed to parse URL \'' + str(self.path) + '\'.')
            return ( None, None )

        path = url.group('path')
        query = {}
        if not url.group('query') is None:
            for m in re.finditer(r'(?P<key>.+?)=(?P<value>.+?)(&|$)', url.group('query')):
                query[m.group('key')] = m.group('value')
        return ( path, query )

    def do_GET(self):
        """Handles all HTTP GET requests."""

        path, query = self.parse_url_pathquery()
        if path is None:
            self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
            return

        # / - index page, shows generic help
        if path == '':
            self.respond_index(query)
            return

        # /list - list stored nodes
        if path == 'list':
            self.respond_list(query)
            return

        # /vpn - notification endpoint for gateway's VPN connections
        if path == 'vpn':
            self.respond_vpn(query)
            return

        # /providers
        if path == 'providers':
            self.respond_providers(query)
            return

        # /node/<id>.json - node's data
        # /node/<id>/field - return specific field from node's data
        m = re.match(r'node/(?P<id>[a-fA-F0-9]{12})(?P<cmd>\.json|/[a-zA-Z0-9_\-\.]+)$', path)
        if m != None:
            cmd = m.group('cmd')
            nodeid = m.group('id').lower()
            if cmd == '.json':
                self.respond_node(nodeid)
            else:
                self.respond_nodedetail(nodeid, cmd[1:])
            return

        # /status/<id> - node's status
        m = re.match(r'status/([a-f0-9]{12})$', path)
        if m != None:
            self.respond_nodestatus(m.group(1))
            return

        # no match -> 404
        self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))

    def do_POST(self):
        """Handles all HTTP POST requests."""

        path, query = self.parse_url_pathquery()
        if path is None:
            self.send_error(400, 'Could not parse URL (' + str(self.path) + ')')
            return
        params = self.parse_post_params()

        # node id/mac to name mapping
        if path == 'idmac2name':
            self.respond_nodeidmac2name(params)
            return

        # no match -> 404
        self.send_error(404, 'The URL \'{0}\' was not found here.'.format(path))

    def send_nocache_headers(self):
        """Sets HTTP headers indicating that this response shall not be cached."""

        self.send_header('Cache-Control', 'no-cache, no-store, must-revalidate')
        self.send_header('Pragma', 'no-cache')
        self.send_header('Expires', '0')

    def send_headers(self, content_type='text/html; charset=utf-8', nocache=True):
        """Send HTTP 200 Response header with the given Content-Type.
        Optionally send no-caching headers, too."""

        self.send_response(200)
        self.send_header('Content-Type', content_type)
        if nocache:
            self.send_nocache_headers()
        self.end_headers()

    def parse_post_params(self):
        ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
        if ctype == 'multipart/form-data':
            postvars = cgi.parse_multipart(self.rfile, pdict)
        elif ctype == 'application/x-www-form-urlencoded':
            length = int(self.headers.getheader('content-length'))
            postvars = cgi.parse_qs(self.rfile.read(length), keep_blank_values=1)
        else:
            postvars = {}
        return postvars

    def respond_index(self, query):
        """Display the index page."""

        storage = self.server.storage
        self.send_headers()

        self.wfile.write('<!DOCTYPE html><html><head><title>BATCAVE</title></head>\n')
        self.wfile.write('<body>\n')
        self.wfile.write('<H1 title="Batman/Alfred Transmission Collection, Aggregation & Value Engine">BATCAVE</H1>\n')

        self.wfile.write('<p>Dies ist ein interner Hintergrund-Dienst. Er wird nur von anderen Diensten\n')
        self.wfile.write('angesprochen und sollte aus einer Mehrzahl von Gr&uuml;nden nicht &ouml;ffentlich\n')
        self.wfile.write('zug&auml;nglich sein.</p>\n')

        self.wfile.write('<H2>Status</H2>\n')
        self.wfile.write('Daten: <span id="datacount" class="value">')
        self.wfile.write(len(storage.data))
        self.wfile.write('</span>\n')

        self.wfile.write('<H2>API</H2>\n')
        self.wfile.write('<p>Grundsätzlich ist das Antwort-Format JSON und alle Daten sind Live-Daten (kein Cache) die ggf. etwas Bearbeitungs-Zeit erfordern.</p>')
        self.wfile.write('<dl>\n')
        self.wfile.write('<dt><a href="/nodes.json">nodes.json</a></dt><dd>zur Verwendung mit ffmap (MACs anonymisiert)</dd>\n')
        self.wfile.write('<dt><a href="/node/ff00ff00ff00.json">/node/&lt;id&gt;.json</a></dt><dd><u>alle</u> vorhandenen Information zu der gewünschten Node</dd>\n')
        self.wfile.write('</dl>\n')
        self.wfile.write('</body></html>')

    def respond_list(self, query):
        """List stored data."""

        self.send_headers()

        self.wfile.write('<!DOCTYPE html><html>\n')
        self.wfile.write('<head><title>BATCAVE</title></head>\n')
        self.wfile.write('<body>\n')
        self.wfile.write('<H1>BATCAVE - LIST</H1>\n')

        self.wfile.write('<table>\n')
        self.wfile.write('<thead><tr><th>ID</th><th>Name</th></tr></thead>\n')
        self.wfile.write('<tbody>\n')

        sortkey = query['sort'] if 'sort' in query else None
        data = self.server.storage.get_nodes(sortby=sortkey)
        for node in data:
            nodeid = node['node_id']
            nodename = node['hostname'] if 'hostname' in node else '&lt;?&gt;'

            self.wfile.write('<tr><td><a href="/node/' + nodeid + '.json">' + nodeid + '</a></td><td>' + nodename + '</td></tr>')

        self.wfile.write('</tbody>\n')
        self.wfile.write('</table>\n')

    def respond_node(self, rawid):
        """Display node data."""

        # handle API example linked on index page
        if rawid == 'ff00ff00ff00':
            self.send_headers('text/json')
            self.wfile.write(json.dumps({
                'name': 'API-Example',
                'nodeid': rawid,
                'META': 'Dies ist ein minimaler Beispiel-Datensatz. Herzlichen Glückwunsch, du hast das Prinzip der API kapiert.',
            }))
            return

        # search node by the given id
        node = self.server.storage.find_node(rawid)

        # handle unknown nodes
        if node is None:
            self.send_error(404, 'No node with id \'' + rawid + '\' present.')
            return

        # dump node data as JSON
        self.send_headers('text/json')
        self.wfile.write(json.dumps(node))

    def respond_nodestatus(self, rawid):
        """Display node status."""

        status = self.server.storage.get_nodestatus(rawid)

        if status is None:
            self.send_error(404, 'No node with id \'' + rawid + '\' present.')

        self.send_headers('text/plain')
        self.wfile.write(status)

    def respond_nodeidmac2name(self, ids):
        """Return a mapping of the given IDs (or MACs) into their hostname."""

        self.send_headers('text/plain')
        for nodeid in ids:
            node = self.server.storage.find_node(nodeid) if not ':' in nodeid else self.server.storage.find_node_by_mac(nodeid)
            nodename = node['hostname'] if (not node is None) and 'hostname' in node else nodeid
            self.wfile.write('{0}={1}\n'.format(nodeid, nodename))

    def respond_nodedetail(self, nodeid, field):
        """Return a field from the given node - a string is returned as text, all other as JSON."""

        node = self.server.storage.find_node(nodeid)
        if node is None:
            self.send_error(404, 'No node with id \'' + nodeid + '\' present.')
            return

        return_count = False
        if field.endswith('.count'):
            return_count = True
            field = field[0:-6]

        if not field in node:
            self.send_error(404, 'The node \'' + nodeid + '\' does not have a field named \'' + str(field) + '\'.')
            return

        value = node[field]
        if return_count:
            value = len(value)

        self.send_headers('text/plain' if isinstance(value, basestring) or isinstance(value, int) else 'text/json')
        self.wfile.write(value if isinstance(value, basestring) else json.dumps(value))

    def respond_vpn(self, query):
        storage = self.server.storage
        peername = query['peer'] if 'peer' in query else None
        key = query['key'] if 'key' in query else None
        action = query['action'] if 'action' in query else None
        remote = query['remote'] if 'remote' in query else None
        gw = query['gw'] if 'gw' in query else None
        ts = query['ts'] if 'ts' in query else time.time()

        if action == 'list':
            self.respond_vpnlist()
            return

        if action != 'establish' and action != 'disestablish':
            self.logger.error('VPN: unknown action \'{0}\''.format(action))
            self.send_error(400, 'Invalid action.')
            return

        check = {'peername': peername, 'key': key, 'remote': remote, 'gw': gw}
        for k, val in check.items():
            if val is None or len(val.strip()) == 0:
                self.logger.error('VPN {0}: no or empty {1}'.format(action, k))
                self.send_error(400, 'Missing value for ' + str(k))
                return

        try:
            if action == 'establish':
                self.server.storage.log_vpn_connect(
                    key, peername, remote, gw, ts)

            elif action == 'disestablish':
                self.server.storage.log_vpn_connect(key, gw, ts)

            else:
                self.logger.error('Unknown VPN action \'%s\' not filtered.',
                                  action)
                self.send_error(500)
                return

        except ffstatus.exceptions.VpnKeyFormatError:
            self.logger.error('VPN peer \'{0}\' {1}: bad key \'{2}\''.format(
                peername, action, key,
            ))
            self.send_error(400, 'Bad key.')
            return

        self.send_headers('text/plain')
        self.wfile.write('OK')

        storage.save()

    def respond_vpnlist(self):
        self.send_headers()
        self.wfile.write('<!DOCTYPE html>\n')
        self.wfile.write('<html><head><title>BATCAVE - VPN LIST</title></head>\n')
        self.wfile.write('<body>\n')
        self.wfile.write('<style type="text/css">\n')
        self.wfile.write('table { border: 2px solid #999; border-collapse: collapse; }\n')
        self.wfile.write('th, td { border: 1px solid #CCC; }\n')
        self.wfile.write('table tbody tr.online { background-color: #CFC; }\n')
        self.wfile.write('table tbody tr.offline { background-color: #FCC; }\n')
        self.wfile.write('</style>\n')
        self.wfile.write('<table>\n<thead>\n')
        gateways = self.server.storage.get_vpn_gateways()
        self.wfile.write('<tr><th rowspan="2">names (key)</th><th colspan="' + str(len(gateways)) + '">active</th><th colspan="' + str(len(gateways)) + '">last</th></tr>\n')
        self.wfile.write('<tr><th>' + '</th><th>'.join(gateways) + '</th><th>' + '</th><th>'.join(gateways) + '</th></tr>\n')
        self.wfile.write('</thead>\n')

        for item in self.server.storage.get_vpn_connections():
            self.wfile.write('<tr class="{0}">'.format('online' if item['online'] else 'offline'))
            self.wfile.write('<td title="{0}">{1}</td>'.format(
                item['key'],
                ' / '.join(item['names']) if len(item['names']) > 0 else '?',
            ))

            for conntype in ['active', 'last']:
                for gateway in gateways:
                    remote = ''
                    if conntype in item['remote'] and gateway in item['remote'][conntype]:
                        remote = item['remote'][conntype][gateway]
                        if isinstance(remote, dict):
                            remote = remote['name']
                    symbol = '&check;' if len(remote) > 0 else '&times;'
                    self.wfile.write('<td title="{0}">{1}</td>'.format(
                        remote, symbol))

            self.wfile.write('</tr>\n')

        self.wfile.write('</table>\n')
        self.wfile.write('</body>')
        self.wfile.write('</html>')

    def respond_providers(self, query):
        """Return a summary of providers."""

        outputformat = query['format'].lower() if 'format' in query else 'html'

        isps = {}
        ispblocks = {}
        for item in self.server.storage.get_vpn_connections():
            if item['count']['active'] == 0:
                continue

            remotes = []
            for gateway in item['remote']['active']:
                remote = item['remote']['active'][gateway]
                remotes.append(remote)

            if len(remotes) == 0:
                self.logger.warn(
                    'VPN key \'%s\' is marked with active remotes but 0 found?',
                    item['key'])
                continue

            item_isps = set()
            for remote in remotes:
                isp = "UNKNOWN"
                ispblock = remote
                if isinstance(remote, dict):
                    ispblock = remote['name']
                    desc_lines = remote['description'].split('\n')
                    isp = desc_lines[0].strip()

                    # normalize name: strip company indication
                    isp = re.sub(r'(AG|UG|G?mbH( ?& ?Co\.? ?(OH|K)G)?)$', '', isp, flags=re.IGNORECASE).strip()

                    # normalize name: strip "pool" suffixes
                    isp = re.sub(r'(dynamic )?(customer |subscriber )?(ip )?(pool|(address )?range|addresses)$', '', isp, flags=re.IGNORECASE).strip()

                    # normalize name: strip "B2B" and aggregation suffixes
                    isp = re.sub(r'(aggregate|aggregation)?$', '', isp, flags=re.IGNORECASE).strip()
                    isp = re.sub(r'(B2B)?$', '', isp, flags=re.IGNORECASE).strip()

                    # normalize name: strip country suffixes (in Germany)
                    isp = re.sub(r'(DE|Deutschland|Germany|Nordrhein[- ]Westfalen|NRW|Baden[- ]Wuerttemburg|BW|Hessen|Niedersachsen|Rheinland[- ]Pfalz|RLP)$', '', isp, flags=re.IGNORECASE).strip()

                isp = str(isp)
                if not isp in ispblocks:
                    ispblocks[isp] = set()
                ispblocks[isp].add(ispblock)

                item_isps.add(isp)

            if len(item_isps) == 0:
                item_isps.add('unknown')

            elif len(item_isps) > 1:
                self.logger.warn('VPN key \'{0}\' has {1} active IPs which resolved to {2} ISPs: \'{3}\''.format(key, len(ips), len(item_isps), '\', \''.join(item_isps)))

            for isp in item_isps:
                if not isp in isps:
                    isps[isp] = 0
                isps[isp] += 1.0 / len(item_isps)

        isps_sum = sum([isps[x] for x in isps])

        if outputformat == 'csv':
            self.send_headers('text/csv')

            self.wfile.write('Count;Name\n')
            for isp in isps:
                self.wfile.write('{0};"{1}"\n'.format(isps[isp], isp))

        elif outputformat == 'json':
            self.send_headers('text/json')

            data = [
                {
                    'name': isp,
                    'count': isps[isp],
                    'percentage': isps[isp]*100.0/isps_sum,
                    'blocks': [block for block in ispblocks[isp]],
                } for isp in isps
            ]
            self.wfile.write(json.dumps(data))

        elif outputformat == 'html':
            self.send_headers()

            self.wfile.write('<!DOCTYPE html><html>\n')
            self.wfile.write('<head><title>BATCAVE - PROVIDERS</title></head>\n')
            self.wfile.write('<body>\n')
            self.wfile.write('<table border="2">\n')
            self.wfile.write('<thead><tr><th>Count</th><th>Percentage</th><th>Name</th><th>Blocks</th></tr></thead>\n')
            self.wfile.write('<tbody>\n')

            for isp in sorted(isps, key=lambda x: isps[x], reverse=True):
                self.wfile.write('<tr><td>{0}</td><td>{1:.1f}%</td><td>{2}</td><td>{3}</td></tr>\n'.format(
                    isps[isp],
                    isps[isp]*100.0/isps_sum,
                    isp,
                    ', '.join(sorted(ispblocks[isp])) if isp in ispblocks else '?',
                ))

            self.wfile.write('</tbody></table>\n')
            self.wfile.write('<p>Totals: {0} ISPs, {1} connections</p>\n'.format(len(isps), isps_sum))
            self.wfile.write('</body></html>')

        else:
            self.send_error(400, 'Unknown output format.')


class ApiServer(ThreadingMixIn, HTTPServer):
    def __init__(self, endpoint, storage):
        if ':' in endpoint[0]:
            self.address_family = socket.AF_INET6
        HTTPServer.__init__(self, endpoint, BatcaveHttpRequestHandler)
        self.storage = storage

    def __str__(self):
        return 'ApiServer on {0}'.format(self.server_address)

if __name__ == '__main__':
    dummystorage = ffstatus.basestorage.BaseStorage()
    server = ApiServer(('0.0.0.0', 8888), dummystorage)

    print("Server:", str(server))
    server.serve_forever()