#!/usr/bin/python # -*- coding: utf-8 -*- from __future__ import print_function, unicode_literals import logging import re import time import ffstatus from .exceptions import VpnKeyFormatError def sanitize_node(data, include_raw_data=False): """ Filters potentially harmful entries from the node's data. """ export = ffstatus.dict_merge({}, data) if export is None: return {} # remove fields from output: __RAW__ if '__RAW__' in export and not include_raw_data: del export['__RAW__'] return export class BaseStorage(object): """ Provides operations on the storage data. This class gets subclassed to actually write the data to a file, database, whatever. """ DATAKEY_VPN = '__VPN__' FIELDKEY_UPDATED = '__UPDATED__' metric_handler = None def open(self): """ When overridden in a subclass, opens the persistent storage. """ pass def save(self): """ When overriden in a subclass, stores the data to a persistent storage. """ pass def close(self): """ When overridden in a subclass, closes the persistent storage. """ pass @property def status(self): """Gets status information on the storage.""" nodes = 0 nodes_active = 0 gateways = 0 gateways_active = 0 sum_clients = 0 clients = set() for node in self.get_nodes(): nodetype = node.get('type', 'node') if nodetype == 'gateway': gateways += 1 if self.get_nodestatus(node=node) == 'active': gateways_active += 1 continue nodes += 1 nodemacs = [x for x in node.get('macs', [])] if 'mac' in node: nodemacs.append(node['mac']) if self.get_nodestatus(node=node) == 'active': nodes_active += 1 sum_clients += node.get('clientcount', 0) for client in node.get('clients', []): if client in nodemacs: continue clients.add(client) return { 'clients_sum': sum_clients, 'clients_unique': len(clients), 'gateways': gateways, 'gateways_active': gateways_active, 'nodes': nodes, 'nodes_active': nodes_active, 'now': int(time.time()), } def __merge_alias_node(self, item, alias): # start by using standard dict_merge() update = ffstatus.dict_merge(item, alias, overwrite_lists=False) # extract some fields for further inspection update_macs = update.get('macs', []) or [] # field 'node_id': keep original value update['node_id'] = item['node_id'] # field 'mac': keep original value if 'mac' in item: update['mac'] = item['mac'] if 'mac' in alias: update_macs.append(alias['mac']) # field 'macs': get rid of duplicates and primary mac primary_mac = update.get('mac') macs = [] for x in update_macs: if x != primary_mac and x not in macs: macs.append(x) update['macs'] = update_macs = macs # field 'type': keep special node type item_type = item.get('type', 'node') if item_type != 'node' and update.get('type', 'node') == 'node': update['type'] = item_type return update def __send_metric(self, key, value, ts=None): if ts is None: ts = time.time() func = self.metric_handler if func is None or not callable(func): # no handler -> do nothing return func(self, key, value, ts) def merge_new_data(self, newdata): """Updates data in the storage by merging the new data.""" ts = time.time() if newdata is None or not isinstance(newdata, dict): raise ValueError("Expected a dict as new data.") # keep a list of aliased nodes so they can be removed from the result aliased_nodes = {} # start merge on a copy of the current data current = {} for node in self.get_nodes(): item_id = node['node_id'] current[item_id] = ffstatus.dict_merge(node, {}) current[item_id]['aliases'] = [] current[item_id]['clients.bak'] = current[item_id].get('clients', []) current[item_id]['clients'] = [] current[item_id]['neighbours'] = {} current[item_id]['type'] = 'node' if not item_id in newdata: continue if not '__RAW__' in current[item_id]: current[item_id]['__RAW__'] = {} if '__RAW__' in newdata[item_id]: for key in newdata[item_id]['__RAW__']: if key in current[item_id]['__RAW__']: del current[item_id]['__RAW__'][key] # merge the dictionaries updated = {} for itemid in newdata: if not itemid in current: # new element which did not exist in storage before, that's easy updated[itemid] = newdata[itemid] else: # merge the old and new element update = ffstatus.dict_merge(current[itemid], newdata[itemid]) updated[itemid] = update for alias_id in updated[itemid].get('aliases', []): if alias_id in aliased_nodes: aliased_nodes[alias_id].append(itemid) else: aliased_nodes[alias_id] = [itemid] # merge aliased nodes for alias_id in aliased_nodes: if len(aliased_nodes[alias_id]) != 1: logging.warn("Node '%s' is aliased by multiple nodes: %s", alias_id, aliased_nodes[alias_id]) continue # target's id is the single entry of the alias list item_id = aliased_nodes[alias_id][0] if alias_id == item_id: # the node has itself as alias -> remove the alias entry if alias_id in updated and 'aliases' in updated[alias_id]: updated[alias_id]['aliases'].remove(alias_id) logging.debug("Removed self-alias of '%s'.", alias_id) continue # look for alias node alias = updated.get(alias_id, current.get(alias_id)) if alias is None: # no alias node present already, as we're trying to achieve here continue # look for target node item = updated.get(item_id, current.get(item_id)) if item is None: logging.warn("Alias node '%s' is missing its target '%s.", alias_id, item_id) continue # ensure both target and alias node have 'node_id' field set if not 'node_id' in item: item['node_id'] = item_id alias['node_id'] = alias_id # merge data update = self.__merge_alias_node(item, alias) updated[item_id] = update logging.debug("Merged alias '%s' into '%s'.", alias_id, item_id) # mark alias node for deletion updated[alias_id] = None # sanitize each item's data for itemid in updated: if itemid.startswith('__'): continue item = updated[itemid] # delete node if requested if item is None: self.set_node_data(itemid, None) continue # ensure 'node_id' is set if not 'node_id' in item: item['node_id'] = itemid # remove node's MACs from clients list item_mac = item.get('mac') or ffstatus.guess_mac_from_nodeid(itemid) clients = [x for x in item.get('clients', [])] if item_mac is not None and item_mac in clients: clients.remove(item_mac) for mac in item.get('macs', []): if mac in clients: clients.remove(mac) # set clientcount item['clientcount'] = len(clients) # compute client delta prev_clients = item.get('clients.bak', []) new_clients = item.get('clients', []) if 'clients.bak' in item: del(item['clients.bak']) diff_added = [x for x in new_clients if x not in prev_clients] diff_removed = [x for x in prev_clients if x not in new_clients] self.__send_metric("nodes." + item_mac + ".clients_added", len(diff_added), ts) self.__send_metric("nodes." + item_mac + ".clients_removed", len(diff_removed), ts) # finally, set each new data self.set_node_data(itemid, item) def get_nodes(self, sortby=None, include_raw_data=False): """Gets a list of all known nodes.""" nodes = self.get_all_nodes_raw() sorted_ids = [x for x in nodes] if sortby is not None: if sortby == 'name': sortkey = lambda x: nodes[x]['hostname'].lower() sorted_ids = sorted(sorted_ids, key=sortkey) elif sortby == 'id': sorted_ids = sorted(sorted_ids) result = [] for nodeid in sorted_ids: if nodeid.startswith('__'): continue node = sanitize_node(nodes[nodeid], include_raw_data) result.append(node) return result def find_node(self, rawid, include_raw_data=False, search_aliases=True): """ Fetch node data by given id. If necessary, look through node aliases. """ # try direct match, first node = self.get_node(rawid) if node is not None: return sanitize_node(node, include_raw_data=include_raw_data) # look through all nodes found = None nodes = self.get_all_nodes_raw() for nodeid in nodes: node = nodes[nodeid] # if we have a direct hit, return it immediately if nodeid == rawid: return sanitize_node(node, include_raw_data=include_raw_data) # search through aliases if search_aliases and rawid in node.get('aliases', []): found = node # return found node if not found is None: return sanitize_node(found, include_raw_data=include_raw_data) else: return None def find_node_by_mac(self, mac): """Fetch node data by given MAC address.""" needle = mac.lower() # iterate over all nodes for node in self.get_nodes(): # check node's primary MAC if 'mac' in node and needle == node['mac'].lower(): return sanitize_node(node) # check alias MACs if 'macs' in node: haystack = [x.lower() for x in node['macs']] if mac in haystack: return sanitize_node(node) # MAC address not found return None def get_nodestatus(self, rawid=None, node=None): """Determine node's status.""" # search node by the given id if node is None and not rawid is None: node = self.find_node(rawid) # handle unknown nodes if node is None: return None # check that the last batadv update is noted in the data updated = node.get(self.FIELDKEY_UPDATED, None) if updated is None: return 'unknown' u = updated.get('batadv', updated.get('batctl')) if u is None: return 'unknown' # make decision based on time of last batadv update diff = time.time() - u if diff < 150: return 'active' elif diff < 300: return 'stale' else: return 'offline' def set_node_data(self, key, data): """ Overwrite data for the node with the given key. Specifying 'None' as data effectively means deleting the key. """ raise NotImplementedError("set_node_data was not overridden") def check_vpn_key(self, key): if key is None or re.match(r'^[a-fA-F0-9]+$', key) is None: raise VpnKeyFormatError(key) def get_vpn_keys(self): """Gets a list of VPN keys.""" raise NotImplementedError("get_vpn_keys was not overriden") def get_vpn_item(self, key, create=False): self.check_vpn_key(key) raise NotImplementedError("store_vpn_item was not overriden") def store_vpn_item(self, key, data): raise NotImplementedError("store_vpn_item was not overriden") def resolve_vpn_remotes(self): """Iterates all remotes and resolves IP blocks not yet resolved.""" vpn = self.get_vpn_keys() init_vpn_cache = {} for key in vpn: entry = self.get_vpn_item(key) entry_modified = False for mode in entry: if not isinstance(entry[mode], dict): continue for gateway in entry[mode]: if not isinstance(entry[mode][gateway], dict): continue item = entry[mode][gateway] if 'remote' in item and not 'remote_raw' in item: item['remote_raw'] = item['remote'] resolved = None if item['remote'] in init_vpn_cache: resolved = init_vpn_cache[item['remote']] else: resolved = ffstatus.resolve_ipblock(item['remote']) init_vpn_cache[item['remote']] = resolved if resolved is not None: logging.info( 'Resolved VPN entry \'%s\' to net \'%s\'.', item['remote'], resolved['name'], ) if resolved is not None: item['remote'] = resolved entry_modified = True if entry_modified: self.store_vpn_item(key, entry) def get_vpn_gateways(self): gateways = set() vpn = self.get_vpn_keys() for key in vpn: entry = self.get_vpn_item(key) for conntype in entry: for gateway in entry[conntype]: gateways.add(gateway) return sorted(gateways) def get_vpn_connections(self): conntypes = ['active', 'last'] result = [] vpnkeys = self.get_vpn_keys() for key in vpnkeys: vpn_entry = self.get_vpn_item(key) if not isinstance(vpn_entry, dict): continue item = { 'key': key, 'count': {}, 'remote': {}, } names = set() for conntype in conntypes: item['count'][conntype] = 0 item['remote'][conntype] = {} if conntype in vpn_entry: for gateway in vpn_entry[conntype]: if 'remote' in vpn_entry[conntype][gateway]: remote = vpn_entry[conntype][gateway]['remote'] if remote is None or remote == '': continue item['count'][conntype] += 1 item['remote'][conntype][gateway] = remote if 'peer' in vpn_entry[conntype][gateway]: names.add(vpn_entry[conntype][gateway]['peer']) item['names'] = sorted(names) item['online'] = item['count']['active'] > 0 result.append(item) return result def log_vpn_connect(self, key, peername, remote, gateway, timestamp): item = self.get_vpn_item(key, create=True) # resolve remote addr to its netblock remote_raw = remote remote_resolved = None if remote is not None: remote_resolved = ffstatus.resolve_ipblock(remote) if remote_resolved is not None: logging.debug('Resolved IP \'{0}\' to block \'{1}\'.'.format( remote, remote_resolved['name'], )) remote = remote_resolved # store connection info item['active'][gateway] = { 'establish': timestamp, 'peer': peername, 'remote': remote, 'remote_raw': remote_raw, } self.store_vpn_item(key, item) def log_vpn_disconnect(self, key, gateway, timestamp): item = self.get_vpn_item(key, create=True) active = {} if gateway in item['active']: active = item['active'][gateway] del item['active'][gateway] active['disestablish'] = timestamp item['last'][gateway] = active self.store_vpn_item(key, item)