status-bot/modules/ffpb.py

# -*- coding: utf-8 -*-
from __future__ import print_function
import willie

import datetime
import difflib
from email.utils import mktime_tz
from fnmatch import fnmatch
import git
import netaddr
import json
import urllib2
import re
import os
import random
import shelve
import subprocess
import time

import dns.resolver,dns.reversename
import socket
import SocketServer
import threading

msgserver = None
peers_repo = None
monitored_nodes = None
highscores = None

nodeaccess = None

alfred_method = None
alfred_data = None
alfred_update = datetime.datetime(1970,1,1,23,42)

ffpb_resolver = dns.resolver.Resolver ()
ffpb_resolver.nameservers = ['10.132.254.53']

class MsgHandler(SocketServer.BaseRequestHandler):
	"""Reads line from TCP stream and forwards it to configured IRC channels."""

	def handle(self):
		data = self.request.recv(2048).strip()
		sender = self._resolve_name (self.client_address[0])

		bot = self.server.bot
		if bot is None:
			print("ERROR: No bot in handle() :-(")
			return

		target = bot.config.core.owner
		if bot.config.has_section('ffpb'):
			is_public = data.lstrip().lower().startswith("public:")

			if is_public and not (bot.config.ffpb.msg_target_public is None):
				data = data[7:].lstrip()
				target = bot.config.ffpb.msg_target_public
			elif not (bot.config.ffpb.msg_target is None):
				target = bot.config.ffpb.msg_target

		bot.msg(target, "[{0}] {1}".format(sender, str(data)))

	def _resolve_name (self, ip):
		"""Resolves the host name of the given IP address
		and strips away the suffix (.infra)?.ffpb"""

		if ip.startswith ("127."):
			return "localhost"

		try:
			addr = dns.reversename.from_address (ip)
			return re.sub ("(.infra)?.ffpb.", "", str (ffpb_resolver.query (addr, "PTR")[0]))
		except dns.resolver.NXDOMAIN:
			return ip

class ThreadingTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
	pass

def setup(bot):
	global msgserver, peers_repo, alfred_method, highscores, monitored_nodes, nodeaccess

	# signal begin of setup routine
	bot.memory['ffpb_in_setup'] = True

	# load highscores from disk
	highscores = shelve.open('highscoredata', writeback=True)
	if not 'nodes' in highscores:
		highscores['nodes'] = 0
		highscores['nodes_ts'] = time.time()
	if not 'clients' in highscores:
		highscores['clients'] = 0
		highscores['clients_ts'] = time.time()

	# load list of monitored nodes and their last status from disk
	monitored_nodes = shelve.open('nodes.monitored', writeback=True)

	# load list of seen nodes from disk
	seen_nodes = shelve.open('nodes.seen', writeback=True)
	bot.memory['seen_nodes'] = seen_nodes

	# load list of node ACL from disk (used in playitsafe())
	nodeaccess = shelve.open('nodes.acl', writeback=True)

	# no need to configure anything else if the ffpb config section is missing
	if not bot.config.has_section('ffpb'):
		bot.memory['ffpb_in_setup'] = False
		return

	# open the git repository containing the peers files
	if not bot.config.ffpb.peers_directory is None:
		peers_repo = git.Repo(bot.config.ffpb.peers_directory)
		assert peers_repo.bare == False

	# if configured, start the messaging server
	if int(bot.config.ffpb.msg_enable) == 1:
		host = "localhost"
		port = 2342
		if not bot.config.ffpb.msg_host is None: host = bot.config.ffpb.msg_host
		if not bot.config.ffpb.msg_port is None: port = int(bot.config.ffpb.msg_port)

		msgserver = ThreadingTCPServer((host,port), MsgHandler)
		msgserver.bot = bot
		ip, port = msgserver.server_address
		print("Messaging server listening on {}:{}".format(ip,port))

		msgserver_thread = threading.Thread(target=msgserver.serve_forever)
		msgserver_thread.daemon = True
		msgserver_thread.start()

	# initially fetch ALFRED data
	alfred_method = bot.config.ffpb.alfred_method
	ffpb_updatealfred(bot)

	# signal end of setup routine
	bot.memory['ffpb_in_setup'] = False

def shutdown(bot):
	global msgserver, highscores, monitored_nodes, nodeaccess

	# store highscores
	if not highscores is None:
		highscores.sync()
		highscores.close()
		highscores = None

	# store monitored nodes
	if not monitored_nodes is None:
		monitored_nodes.sync()
		monitored_nodes.close()
		monitored_nodes = None

	# store node acl
	if not nodeaccess is None:
		nodeaccess.sync()
		nodeaccess.close()
		nodeaccess = None

	# store seen nodes
	if 'seen_nodes' in bot.memory and bot.memory['seen_nodes'] != None:
		bot.memory['seen_nodes'].close()
		bot.memory['seen_nodes'] = None
		del(bot.memory['seen_nodes'])

	# shutdown messaging server
	if not msgserver is None:
		msgserver.shutdown()
		print("Closed messaging server.")
		msgserver = None

@willie.module.commands("help")
@willie.module.commands("hilfe")
@willie.module.commands("man")
def ffpb_help(bot, trigger):
	"""Display commony ulsed functions."""

	functions = {
		"!ping <knoten>": "Prüfe ob der Knoten erreichbar ist.",
		"!status": "Aktuellen Status des Netzwerks (insb. Anzahl Knoten und Clients) ausgegeben.",
		"!info <knoten>": "Allgemeine Information zu dem Knoten anzeigen.",
		"!link <knoten>": "MAC-Adresse und Link zur Status-Seite des Knotens anzeigen.",
		"!exec-on-peer <knoten> <kommando>": "Befehl auf dem Knoten ausführen (nur möglich bei eigenen Knoten oder als Admin, in beiden Fällen auch nur wenn der SSH-Key des Bots hinterlegt wurde)",
		"!mesh <knoten>": "Zeige Mesh-Partner eines Knotens",
	}

	param = trigger.group(2)
	if param is None:
		bot.say("Funktionen: " + str.join(", ", sorted(functions.keys())))
		return

	if param.startswith("!"): param = param[1:]

	for fun in functions.keys():
		if fun.startswith("!" + param + " "):
			bot.say("Hilfe zu '" + fun + "': " + functions[fun])
			return

	bot.say("Allgemeine Hilfe gibt's mit !help - ohne Parameter.")

def playitsafe(bot, trigger,
	botadmin=False, admin_channel=False, via_channel=False, via_privmsg=False, need_op=False, node=None,
	reply_directly=True, debug_user=None, debug_ignorebotadmin=False):
	"""helper: checks that the triggering user has the necessary rights

	Returns true if everything is okay. If it's not, a reply is send via the bot and false is returned.
	"""

	if via_channel and via_privmsg:
		raise Exception('Der Entwickler ist ein dummer, dummer Junge (playitsafe hat via_channel und via_privmsg gleichzeitig gesetzt).')

	user = trigger.nick if debug_user is None else debug_user
	user = user.lower()

	# botadmin: you need to be configured as a bot admin
	if botadmin and not trigger.admin:
		if reply_directly: bot.say('Du brauchst Super-Kuh-Kräfte um dieses Kommando auszuführen.')
		return False

	# via_channel: the request must not be a private conversation
	if via_channel and trigger.is_privmsg:
		if reply_directly: bot.say('Bitte per Channel - mehr Transparenz wagen und so!')
		return False

	# via_privmsg: the request must be a private conversation
	if via_privmsg and not trigger.is_privmsg:
		if reply_directly: bot.say('Solche Informationen gibt es nur per PM, da bin ich ja schon ein klein wenig sensibel ...')
		return False

	# need_op: if the message is in a channel, check that the user has OP there
	if need_op and (not trigger.is_privmsg) and (not user in bot.ops[trigger.sender]):
		if reply_directly: bot.say('Keine Zimtschnecke, keine Kekse.')
		return False

	# node: check that the user is whitelisted (or is admin)
	if not node is None and (debug_ignorebotadmin or not trigger.admin):
		acluser = [ x for x in nodeaccess if x.lower() == user ]
		acluser = acluser[0] if len(acluser) == 1 else None
		if nodeaccess is None or acluser is None:
			if reply_directly: bot.reply('You! Shall! Not! Access!')
			return False

		nodeid = node['node_id'] if 'node_id' in node else None
		matched = False
		for x in nodeaccess[acluser]:
			if x == nodeid or fnmatch(node['hostname'], x):
				matched = True
				break

		if not matched:
			if reply_directly: bot.reply('Mach das doch bitte auf deinen Knoten, kthxbye.')
			return False

	return True

@willie.module.commands('nodeacl')
def ffpb_nodeacl(bot, trigger):
	"""Configure ACL for nodes."""

	if not playitsafe(bot, trigger, botadmin=True):
		# the check function already gives a bot reply, just exit here
		return

	# ensure the user gave arguments (group 2 is the concatenation of all following groups)
	if trigger.group(2) is None or len(trigger.group(2)) == 0:
		bot.say('Sag doch was du willst ... einmal mit Profis arbeiten, ey -.-')
		return

	# read additional arguments
	cmd = trigger.group(3).lower()

	if cmd == 'list':
		user = trigger.group(4)
		if user is None:
			bot.say('ACLs gesetzt für die User: ' + ', '.join([x for x in nodeaccess]))
			return

		user = user.lower()
		uid = [ x for x in nodeaccess if x.lower() == user ]
		if len(uid) == 0:
			bot.say('Für \'{0}\' ist keine Node ACL gesetzt.'.format(user))
			return

		bot.say('Node ACL für \'{0}\' = \'{1}\''.format(uid[0], '\', \''.join(nodeaccess[uid[0]])))
		return

	if cmd in [ 'add', 'del', 'check' ]:
		user = trigger.group(4)
		value = trigger.group(5)

		if user is None or value is None:
			bot.say('Du bist eine Pappnase - User und Knoten, bitte.')
			return

		user = str(user)
		print('NodeACL ' + cmd + ' \'' + value + '\' for user \'' + user + '\'')
		uid = [ x for x in nodeaccess if x == user or x.lower() == user ]
		if cmd == 'add':
			uid = uid[0] if len(uid) > 0 else user
			if not uid in nodeaccess:
				nodeaccess[uid] = []
			if not value in nodeaccess[uid]:
				nodeaccess[uid].append(value)
				bot.say('201 nodeACL \'{0}\' +\'{1}\''.format(uid, value))
			else:
				bot.say('304 nodeACL \'{0}\' contains \'{1}\''.format(uid, value))

		elif cmd == 'del':
			if len(uid) == 0:
				bot.say('404 nodeACL \'{0}\''.format(uid))
				return
			if value in nodeaccess[uid]:
				nodeaccess[uid].remove(value)
				bot.say('200 nodeACL \'{0}\' -\'{1}\''.format(uid, value))
			else:
				bot.say('404 nodeACL \'{0}\' does not contain \'{1}\''.format(uid, value))

		elif cmd == 'check':
			if len(uid) == 0:
				bot.say('Nope, keine ACL gesetzt.')
				return

			node = ffpb_findnode(value)
			if node is None:
				bot.say('Nope, kein Plan was für ein Knoten das ist.')
				return

			result = playitsafe(bot, trigger, debug_user=uid[0], debug_ignorebotadmin=True, node=node, reply_directly=False)
			if result == True:
				bot.say('Jupp.')
			elif result == False:
				bot.say('Nope.')
			else:
				bot.say('Huh? result=' + str(result))

		return

	bot.say('Unbekanntes Kommando. Probier "list [user]", "add user value" oder "del user value". Value kann node_id oder hostname-Maske sein.')

def ffpb_findnode(name):
	"""helper: try to identify the node the user meant by the given name"""

	# no name, no node
	if name is None or len(name) == 0:
		return None

	name = str(name).strip()

	names = {}

	# try to match MAC
	m = re.search("^([0-9a-fA-F][0-9a-fA-F]:){5}[0-9a-fA-F][0-9a-fA-F]$", name)
	if (not m is None):
		mac = m.group(0).lower()
		if mac in alfred_data:
			return alfred_data[mac]

		# try to find alias MAC in ALFRED data
		for nodeid in alfred_data:
			node = alfred_data[nodeid]
			if "network" in node:
				if "mac" in node["network"] and node["network"]["mac"].lower() == mac:
					return node
				if "mesh_interfaces" in node["network"]:
					for mim in node["network"]["mesh_interfaces"]:
						if mim.lower() == mac:
							return node

		return {
			'hostname': '?-' + mac.replace(':','').lower(),
			'network': { 'addresses': [ mac2ipv6(mac, 'fdca:ffee:ff12:132:') ], 'mac': mac, },
			'hardware': { 'model': 'derived-from-mac' },
		}

	# look through the ALFRED peers
	for nodeid in alfred_data:
		node = alfred_data[nodeid]
		if 'hostname' in node:
			h = node['hostname']
			if h.lower() == name.lower():
				return node
			else:
				names[h] = nodeid

	# not found in ALFRED data -> try peers_repo
	if not peers_repo is None:
		peer_name = None
		peer_mac = None
		peer_file = None

		for b in peers_repo.heads.master.commit.tree.blobs:
			if b.name.lower() == name.lower():
				peer_name = b.name
				peer_file = b.abspath
				break

		if (not peer_file is None) and os.path.exists(peer_file):
			peerfile = open(peer_file, "r")
			for line in peerfile:
				if line.startswith("# MAC:"):
					peer_mac = line[6:].strip()
			peerfile.close()

		if not (peer_mac is None):
			return {
				'hostname': peer_name,
				'network': { 'addresses': [ mac2ipv6(peer_mac, 'fdca:ffee:ff12:132:') ], 'mac': peer_mac },
				'hardware': { 'model': 'derived-from-vpnkeys' },
			}

	# do a similar name lookup in the ALFRED data
	possibilities = difflib.get_close_matches(name, [ x for x in names ], cutoff=0.75)
	print('findnode: Fuzzy matching \'{0}\' got {1} entries: {2}'.format(name, len(possibilities), ', '.join(possibilities)))
	if len(possibilities) == 1:
		# if we got exactly one candidate that might be it
		return alfred_data[names[possibilities[0]]]

	# none of the above was able to identify the requested node
	return None

def ffpb_findnode_from_botparam(bot, name, ensure_recent_alfreddata = True):
	"""helper: call ffpb_findnode() and give common answers via bot if nothing has been found"""

	if (name is None or len(name) == 0):
		if not bot is None: bot.reply("Grün.")
		return None

	if ensure_recent_alfreddata and alfred_data is None:
		if not bot is None: bot.say("Informationen sind ausverkauft, kommen erst morgen wieder rein.")
		return None

	if ensure_recent_alfreddata and ffpb_alfred_data_outdated():
		if not bot is None: bot.say("Ich habe gerade keine aktuellen Informationen, daher sage ich mal lieber nichts zu '" + name + "'.")
		return None

	node = ffpb_findnode(name)
	if node is None:
		if not bot is None: bot.say("Kein Plan wer oder was mit '" + name + "' gemeint ist :(")

	return node

def mac2ipv6(mac, prefix=None):
	"""Calculate IPv6 address from given MAC,
	optionally replacing the fe80:: prefix with a given one."""

	result = str(netaddr.EUI(mac).ipv6_link_local())
	if (not prefix is None) and (result.startswith("fe80::")):
		result = prefix + result[6:]
	return result

@willie.module.interval(30)
def ffpb_updatealfred(bot):
	"""Aktualisiere ALFRED-Daten"""
	global alfred_data, alfred_update

	if alfred_method is None or alfred_method == "None":
		return

	updated = None
	if alfred_method == "exec":
		rawdata = subprocess.check_output(['alfred-json', '-z', '-r', '158'])
		updated = datetime.datetime.now()
	elif alfred_method.startswith("http"):
		try:
			rawdata = urllib2.urlopen(alfred_method)
		except:
			print("Failed to download ALFRED data.")
			return
		updated = datetime.datetime.fromtimestamp(mktime_tz(rawdata.info().getdate_tz("Last-Modified")))
	else:
		print("Unknown ALFRED data method '", alfred_method, "', cannot load new data.", sep="")
		alfred_data = None
		return

	try:
		alfred_data = json.load(rawdata)
		#print("Fetched new ALFRED data:", len(alfred_data), "entries")
		alfred_update = updated

	except ValueError as e:
		print("Failed to parse ALFRED data: " + str(e))
		return

	seen_nodes = bot.memory['seen_nodes'] if 'seen_nodes' in bot.memory else None
	if not seen_nodes is None:
		new = []
		for nodeid in alfred_data:
			nodeid = str(nodeid)
			if not nodeid in seen_nodes:
				seen_nodes[nodeid] = updated
				new.append((nodeid,alfred_data[nodeid]['hostname']))
				print('First time seen: ' + str(nodeid))
		if len(new) > 0 and not bot.memory['ffpb_in_setup']:
			action_msg = None
			if len(new) == 1:
				action_msg = random.choice((
					'bemerkt den neuen Knoten {0}',
					'entdeckt {0}',
					'reibt sich die Augen und erblickt einen verpackungsfrischen Knoten {0}',
					u'heißt {0} im Mesh willkommen',
					'freut sich, dass {0} aufgetaucht ist',
					'traut seinen Augen kaum. {0} sagt zum ersten Mal: Hallo Freifunk Paderborn',
					u'sieht die ersten Herzschläge von {0}',
					u'stellt einen großen Pott Heißgetränk zu {0} und fragt ob es hier Meshpartner gibt.',
				)).format('\'' + str(new[0][1]) + '\'')
			else:
				action_msg = random.choice((
					'bemerkt die neuen Knoten {0} und {1}',
					'hat {0} und {1} entdeckt',
					'bewundert {0} sowie {1}',
					'freut sich, dass {0} und {1} nun auch online sind',
					u'heißt {0} und {1} im Mesh willkommen',
					'fragt sich ob die noch jungen Herzen von {0} und {1} synchron schlagen',
				)).format('\'' + '\', \''.join([ str(x[1]) for x in new[0:-1] ]) + '\'', '\'' + str(new[-1][1]) + '\'')
			action_target = bot.config.ffpb.msg_target
			bot.msg(action_target, '\x01ACTION %s\x01' % action_msg)

def ffpb_alfred_data_outdated():
	timeout = datetime.datetime.now() - datetime.timedelta(minutes=5)
	is_outdated = timeout > alfred_update
	#print("ALFRED outdated? {0} (timeout={1} vs. lastupdate={2})".format(is_outdated, timeout, alfred_update))
	return is_outdated

def ffpb_get_batcave_nodefield(nodeid, field):
	raw_data = None
	try:
	        # query BATCAVE for node's field
		raw_data = urllib2.urlopen('http://[fdca:ffee:ff12:a255::253]:8888/node/{0}/{1}'.format(nodeid, field))
	except Exception as err:
		print('Failed to contact BATCAVE for \'{0}\'->\'{1}\': {2}'.format(nodeid, field, err))
		return None

	try:
        	return json.load(raw_data)

	except:
		print('Could not parse BATCAVE\'s response as JSON for \'{0}\'->\'{1}\''.format(nodeid, field))
		return None

@willie.module.commands('debug-alfred')
def ffpb_debug_alfred(bot, trigger):
	"""Show statistics of available ALFRED data."""

	if alfred_data is None:
		bot.say("Keine ALFRED-Daten vorhanden.")
	else:
		bot.say("ALFRED Daten: count={0} lastupdate={1}".format(len(alfred_data), alfred_update))

@willie.module.commands('alfred-data')
def ffpb_peerdata(bot, trigger):
	"""Show ALFRED data of the given node."""

	# identify node or bail out
	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name)
	if node is None: return

	# query must be a PM or as OP in the channel
	if not playitsafe(bot, trigger, need_op=True, node=node):
		# the check function already gives a bot reply, just exit here
		return

	# reply each key in the node's data
	for key in node:
		if key in [ 'hostname' ]: continue
		bot.say("{0}.{1} = {2}".format(node['hostname'], key, str(node[key])))

@willie.module.commands('info')
def ffpb_peerinfo(bot, trigger):
	"""Show information of the given node."""

	# identify node or bail out
	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name)
	if node is None: return

	# read node information
	info_mac = node['network']['mac'] if 'network' in node and 'mac' in node['network'] else '??:??:??:??:??:??'
	info_id = node['node_id'] if 'node_id' in node else info_mac.replace(':','')
	info_name = node['hostname'] if 'hostname' in node else '?-' + info_id

	info_hw = ""
	if "hardware" in node:
		if "model" in node["hardware"]:
			model = node["hardware"]["model"]
			info_hw = " model='" + model + "'"

	info_fw = ""
	info_update = ""
	if "software" in node:
		if "firmware" in node["software"]:
			fwinfo = str(node["software"]["firmware"]["release"]) if "release" in node["software"]["firmware"] else "unknown"
			info_fw = " firmware=" + fwinfo

		if "autoupdater" in node["software"]:
			autoupdater = node["software"]["autoupdater"]["branch"] if node["software"]["autoupdater"]["enabled"] else "off"
			info_update = " (autoupdater="+autoupdater+")"

	info_uptime = ""
	u = -1
	if "statistics" in node and "uptime" in node["statistics"]:
		u = int(float(node["statistics"]["uptime"]))
	elif 'uptime' in node:
		u = int(float(node['uptime']))

	if u > 0:
		d, r1 = divmod(u, 86400)
		h, r2 = divmod(r1, 3600)
		m, s = divmod(r2, 60)
		if d > 0:
			info_uptime = ' up {0}d {1}h'.format(d,h)
		elif h > 0:
			info_uptime = ' up {0}h {1}m'.format(h,m)
		else:
			info_uptime = ' up {0}m'.format(m)

	info_clients = ""
	clientcount = ffpb_get_batcave_nodefield(info_id, 'clientcount')
	if not clientcount is None:
		clientcount = int(clientcount)
		info_clients = ' clients={0}'.format(clientcount)

	bot.say('[{1}]{2}{3}{4}{5}{6}'.format(info_mac, info_name, info_hw, info_fw, info_update, info_uptime, info_clients))

@willie.module.commands('uptime')
def ffpb_peeruptime(bot, trigger):
	"""Display the uptime of the given node."""

	# identify node or bail out
	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name)
	if node is None: return

	# get name and raw uptime from node
	info_name = node["hostname"]
	info_uptime = ''
	u_raw = None
	if 'statistics' in node and 'uptime' in node['statistics']:
		u_raw = node['statistics']['uptime']
	elif 'uptime' in node:
		u_raw = node['uptime']

	# pretty print uptime
	if not u_raw is None:
		u = int(float(u_raw))
		d, r1 = divmod(u, 86400)
		h, r2 = divmod(r1, 3600)
		m, s = divmod(r2, 60)
		if d > 0:
			info_uptime += '{0}d '.format(d)
		if h > 0:
			info_uptime += '{0}h '.format(h)
		info_uptime += '{0}m'.format(m)
		info_uptime += ' # raw: \'{0}\''.format(u_raw)
	else:
		info_uptime += '?'

	# reply to user
	bot.say('uptime(\'{0}\') = {1}'.format(info_name, info_uptime))

@willie.module.commands('link')
def ffpb_peerlink(bot, trigger):
	"""Display MAC and link to statuspage for the given node."""

	# identify node or bail out
	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name)
	if node is None: return

	# get node's MAC
	info_mac = node["network"]["mac"]
	info_name = node["hostname"]

	# get node's v6 address in the mesh (derived from MAC address)
	info_v6 = mac2ipv6(info_mac, 'fdca:ffee:ff12:132:')

	# reply to user
	bot.say('[{1}] mac {0} -> http://[{2}]/'.format(info_mac, info_name, info_v6))

@willie.module.interval(60)
def ffpb_updatepeers(bot):
	"""Refresh list of peers and message the diff."""

	if peers_repo is None:
		print('WARNING: peers_repo is None')
		return

	old_head = peers_repo.head.commit
	peers_repo.remotes.origin.pull()
	new_head = peers_repo.head.commit

	if new_head != old_head:
		print('git pull: from ' + str(old_head) + ' to ' + str(new_head))

		added = []
		changed = []
		renamed = []
		deleted = []
		for f in old_head.diff(new_head):
			if f.new_file:
				added.append(f.b_blob.name)

			elif f.deleted_file:
				deleted.append(f.a_blob.name)

			elif f.renamed:
				renamed.append([f.rename_from, f.rename_to])

			else:
				changed.append(f.a_blob.name)

		response = "Knoten-Update (VPN +{0} %{1} -{2}): ".format(len(added), len(renamed)+len(changed), len(deleted))
		for f in added:
			response += " +'{}'".format(f)
		for f in changed:
			response += " %'{}'".format(f)
		for f in renamed:
			response += " '{}'->'{}'".format(f[0],f[1])
		for f in deleted:
			response += " -'{}'".format(f)

		bot.msg(bot.config.ffpb.msg_target, response)

def ffpb_fetch_stats(bot, url, memoryid):
	"""Fetch a ffmap-style nodes.json from the given URL and
	store it in the bot's memory."""

	response = urllib2.urlopen(url)

	data = json.load(response)
	nodes_active = 0
	nodes_total = 0
	clients_count = 0

	for node in data['nodes']:
		if node['flags']['gateway'] or node['flags']['client']:
			continue

		nodes_total += 1
		if node['flags']['online']:
			nodes_active += 1

		if 'legacy' in node['flags'] and node['flags']['legacy']:
			clients_count -= 1

	for link in data['links']:
		if link['type'] == 'client':
			clients_count += 1

	if not memoryid in bot.memory:
		bot.memory[memoryid] = { }
	stats = bot.memory[memoryid]
	stats["fetchtime"] = time.time()
	stats["nodes_active"] = nodes_active
	stats["nodes_total"] = nodes_total
	stats["clients"] = clients_count

	return (nodes_active, nodes_total, clients_count)

@willie.module.interval(15)
def ffpb_get_stats(bot):
	"""Fetch current statistics, if the highscore changes signal this."""

	(nodes_active, nodes_total, clients_count) = ffpb_fetch_stats(bot, 'http://map.paderborn.freifunk.net/nodes.json', 'ffpb_stats')

	highscore_changed = False
	if nodes_active > highscores['nodes']:
		highscores['nodes'] = nodes_active
		highscores['nodes_ts'] = time.time()
		highscore_changed = True

	if clients_count > highscores['clients']:
		highscores['clients'] = clients_count
		highscores['clients_ts'] = time.time()
		highscore_changed = True

	if highscore_changed:
		print('HIGHSCORE changed: {0} nodes ({1}), {2} clients ({3})'.format(highscores['nodes'], highscores['nodes_ts'], highscores['clients'], highscores['clients_ts']))
		if not (bot.config.ffpb.msg_target is None):
			action_msg = 'notiert sich den neuen Highscore: {0} Knoten ({1}), {2} Clients ({3})'.format(highscores['nodes'], pretty_date(int(highscores['nodes_ts'])), highscores['clients'], pretty_date(int(highscores['clients_ts'])))
			action_target = bot.config.ffpb.msg_target
			if (not bot.config.ffpb.msg_target_public is None):
				action_target = bot.config.ffpb.msg_target_public
			bot.msg(action_target, '\x01ACTION %s\x01' % action_msg)

@willie.module.commands('status')
def ffpb_status(bot, trigger):
	"""State of the network: count of nodes + clients"""

	stats = bot.memory['ffpb_stats'] if 'ffpb_stats' in bot.memory else None
	if stats is None:
		bot.say('Uff, kein Plan wo der Zettel ist. Fragst du später nochmal?')
		return

	bot.say('Es sind {0} Knoten und ca. {1} Clients online.'.format(stats["nodes_active"], stats["clients"]))

def pretty_date(time=False):
    """
    Get a datetime object or a int() Epoch timestamp and return a
    pretty string like 'an hour ago', 'Yesterday', '3 months ago',
    'just now', etc
    """
    from datetime import datetime
    now = datetime.now()
    compare = None
    if type(time) is int:
        compare = datetime.fromtimestamp(time)
    elif type(time) is float:
        compare = datetime.fromtimestamp(int(time))
    elif isinstance(time,datetime):
        compare = time
    elif not time:
        compare = now
    diff = now - compare
    second_diff = diff.seconds
    day_diff = diff.days

    if day_diff < 0:
        return ''

    if day_diff == 0:
        if second_diff < 10:
            return "gerade eben"
        if second_diff < 60:
            return "vor " + str(second_diff) + " Sekunden"
        if second_diff < 120:
            return "vor einer Minute"
        if second_diff < 3600:
            return "vor " + str(second_diff / 60) + " Minuten"
        if second_diff < 7200:
            return "vor einer Stunde"
        if second_diff < 86400:
            return "vor " + str(second_diff / 3600) + " Stunden"
    if day_diff == 1:
        return "gestern"
    if day_diff < 7:
        return "vor " + str(day_diff) + " Tagen"
    return "am " + compare.strftime('%d.%m.%Y um %H:%M Uhr') 

@willie.module.commands('highscore')
def ffpb_highscore(bot, trigger):
	bot.say('Highscore: {0} Knoten ({1}), {2} Clients ({3})'.format(
		highscores['nodes'], pretty_date(int(highscores['nodes_ts'])),
		highscores['clients'], pretty_date(int(highscores['clients_ts']))))

@willie.module.commands('rollout-status')
def ffpb_rolloutstatus(bot, trigger):
	"""Display statistic on how many nodes have installed the given firmware version."""

	# initialize results dictionary
	result = { }
	for branch in [ 'stable', 'testing' ]:
		result[branch] = None
	skipped = 0

	# command is restricted to bot-admins via PM or OPS in the channel
	if (not (trigger.admin and trigger.is_privmsg)) and (not trigger.nick in bot.ops[trigger.sender]):
		bot.say('Geh zur dunklen Seite, die haben Kekse - ohne Keks kein Rollout-Status.')
		return

	# read expected firmware version from command arguments
	expected_release = trigger.group(2)
	if expected_release is None or len(expected_release) == 0:
		bot.say('Von welcher Firmware denn?')
		return

	# check each node in ALFRED data
	for nodeid in alfred_data:
		item = alfred_data[nodeid]
		if (not 'software' in item) or (not 'firmware' in item['software']) or (not 'autoupdater' in item['software']):
			skipped+=1
			continue

		release = item['software']['firmware']['release']
		branch = item['software']['autoupdater']['branch']
		enabled = item['software']['autoupdater']['enabled']
		if not branch in result or result[branch] is None:
			result[branch] = { 'auto_count': 0, 'auto_not': 0, 'manual_count': 0, 'manual_not': 0, 'total': 0 }

		result[branch]['total'] += 1
		match = 'count' if release == expected_release else 'not'
		mode = 'auto' if enabled else 'manual'
		result[branch][mode+'_'+match] += 1

	# respond to user
	output = "Rollout von '{0}':".format(expected_release)
	for branch in result:
		auto_count = result[branch]['auto_count']
		auto_total = auto_count + result[branch]['auto_not']
		manual_count = result[branch]['manual_count']
		manual_total = manual_count + result[branch]['manual_not']
		bot.say("Rollout von '{0}': {1} = {2}/{3} per Auto-Update, {4}/{5} manuell".format(expected_release, branch, auto_count, auto_total, manual_count, manual_total))

	# output count of nodes for which the autoupdater's branch and/or 
	# firmware version could not be retrieved
	if skipped > 0:
		bot.say("Rollout von '{0}': {1} Knoten unklar".format(expected_release, skipped))

@willie.module.commands('ping')
def ffpb_ping(bot, trigger=None, target_name=None):
	"""Ping the given node"""

	# identify node or bail out
	if target_name is None: target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name, ensure_recent_alfreddata=False)
	if node is None: return None

	# get the first non-linklocal address from the node
	target = [x for x in node["network"]["addresses"] if not x.lower().startswith("fe80:")][0]
	target_alias = node["hostname"]

	# execute the actual ping and reply the result
	print("pinging '{0}' at {1} ...".format(target_name, target))
	result = os.system('ping6 -c 2 -W 1 ' + target + ' >/dev/null')
	if result == 0:
		print("ping to '{0}' succeeded".format(target_name))
		if not bot is None: bot.say('Knoten "' + target_alias + '" antwortet \o/')
		return True
	elif result == 1 or result == 256:
		print("ping to '{0}' failed".format(target_name))
		if not bot is None: bot.say('Keine Antwort von "' + target_alias + '" :-(')
		return False
	else:
		print("ping to '{0}' broken: result='{1}'".format(target_name, result))
		if not bot is None: bot.say('Uh oh, irgendwas ist kaputt. Chef, ping result = ' + str(result) + ' - darf ich das essen?')
		return None

@willie.module.interval(3*60)
def ffpb_monitor_ping(bot):
	"""Ping each node currently under surveillance."""

	# determine where-to to send alerts
	notify_target = bot.config.core.owner
	if (not bot.config.ffpb.msg_target is None):
		notify_target = bot.config.ffpb.msg_target

	# check each node under surveillance
	for node in monitored_nodes:
		mon = monitored_nodes[node]
		added = mon['added']
		last_status = mon['status']
		last_check = mon['last_check']
		last_success = mon['last_success']

		current_status = ffpb_ping(bot=None, target_name=node)
		if current_status is None: current_status = False
		mon['status'] = current_status
		mon['last_check'] = time.time()
		if current_status == True: mon['last_success'] = time.time()
		print("Monitoring ({0}) {1} (last: {2} at {3})".format(node, current_status, last_status, time.strftime('%Y-%m-%d %H:%M', time.localtime(last_check))))
		if last_status != current_status and (last_status or current_status):
			if last_check is None:
				# erster Check, keine Ausgabe
				continue

			if current_status == True:
				bot.msg(notify_target, 'Monitoring: Knoten \'{0}\' pingt wieder (zuletzt {1})'.format(node, pretty_date(last_success) if not last_success is None else '[nie]'))
			else:
				bot.msg(notify_target, 'Monitoring: Knoten \'{0}\' DOWN'.format(node))

@willie.module.commands('monitor')
def ffpb_monitor(bot, trigger):
	"""Monitoring capability of the bot, try subcommands add, del, info and list."""

	# command is restricted to bot admins
	if not trigger.admin:
		bot.say('Ich ping hier nicht für jeden durch die Weltgeschichte.')
		return

	# ensure the user gave arguments (group 2 is the concatenation of all following groups)
	if trigger.group(2) is None or len(trigger.group(2)) == 0:
		bot.say('Das Monitoring sagt du hast doofe Ohren.')
		return

	# read additional arguments
	cmd = trigger.group(3)
	node = trigger.group(4)
	if not node is None: node = str(node)

	# subcommand 'add': add a node to monitoring
	if cmd == "add":
		if node in monitored_nodes:
			bot.say('Knoten \'{0}\' wird bereits gemonitored.'.format(node))
			return

		monitored_nodes[node] = {
			'added': trigger.sender, 
			'status': None,
			'last_check': None,
			'last_success': None,
		}

		bot.say('Knoten \'{0}\' wird jetzt ganz genau beobachtet.'.format(node))
		return

	# subcommand 'del': remote a node from monitoring
	if cmd == "del":
		if not node in monitored_nodes:
			bot.say('Knoten \'{0}\' war gar nicht im Monitoring?!?'.format(node))
			return
		del monitored_nodes[node]
		bot.say('Okidoki, \'{0}\' lasse ich jetzt links liegen.'.format(node))
		return

	# subcommand 'info': monitoring status of a node
	if cmd == "info":
		if node in monitored_nodes:
			info = monitored_nodes[node]
			bot.say('Knoten \'{0}\' wurde zuletzt {1} gepingt (Ergebnis: {2})'.format(node, pretty_date(info['last_check']) if not info['last_check'] is None else "^W noch nie", info['status']))
		else:
			bot.say('Knoten \'{0}\' ist nicht im Monitoring.'.format(node))
		return

	# subcommand 'list': enumerate all monitored nodes
	if cmd == "list":
		nodes = ""
		for node in monitored_nodes:
			nodes = nodes + " " + node
		bot.say('Monitoring aktiv für:' + nodes)
		return

	# subcommand 'help': give some hints what the user can do
	if cmd == "help":
		bot.say('Entweder "!monitor list" oder "!monitor {add|del|info} <node>"')
		return

	# no valid subcommand given: complain
	bot.say('Mit "' + str(cmd) + '" kann ich nix anfangen, probier doch mal "!monitor help".')

@willie.module.commands('providers')
def ffpb_providers(bot, trigger):
	"""Fetch the top 5 providers from BATCAVE."""

	providers = json.load(urllib2.urlopen('http://[fdca:ffee:ff12:a255::253]:8888/providers?format=json'))
	providers.sort(key=lambda x: x['count'], reverse=True)
	bot.say('Unsere Top 5 Provider: ' + ', '.join(['{0} ({1:.0f}%)'.format(x['name'], x['percentage']) for x in providers[:5]]))

@willie.module.commands('mesh')
def ffpb_nodemesh(bot, trigger):
	"""Display mesh partners of the given node."""

	# identify node or bail out
	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name, ensure_recent_alfreddata=False)
	if node is None: return None

	# derive node's id
	nodeid = node['node_id'] if 'node_id' in node else None
	if nodeid is None: nodeid = node['network']['mac'].replace(':','') if 'network' in node and 'mac' in node['network'] else None
	if nodeid is None:
		bot.say('Mist, ich habe gerade den Zettel verlegt auf dem die Node-ID von \'{0}\' steht, bitte frag später noch einmal.'.format(node['hostname'] if 'hostname' in node else target_name))
		return

	# query BATCAVE for node's neighbours (result is a list of MAC addresses)
	cave_result = ffpb_get_batcave_nodefield(nodeid, 'neighbours')

	# query BATCAVE for neighbour's names
	d = '&'.join([ str(n) for n in cave_result ])
	req = urllib2.urlopen('http://[fdca:ffee:ff12:a255::253]:8888/idmac2name', d)

	# filter out duplicate names
	neighbours = set()
	for n in req:
		ident,name = n.strip().split('=')
		neighbours.add(name)
	neighbours = [ x for x in neighbours ]

	# respond to the user
	if len(neighbours) == 0:
		bot.say(u'{0} hat keinen Mesh-Partner *schnüff*'.format(node['hostname']))
	elif len(neighbours) == 1:
		bot.say(u'{0} mesht mit \'{1}\''.format(node['hostname'], neighbours[0]))
	else:
		bot.say('{0} mesht mit \'{1}\' und \'{2}\''.format(node['hostname'], '\', \''.join(neighbours[:-1]), neighbours[-1]))

@willie.module.commands('exec-on-peer')
def ffpb_remoteexec(bot, trigger):
	"""Remote execution on the given node"""

	bot_params = trigger.group(2).split(' ',1)
	if len(bot_params) != 2:
		bot.say('Wenn du nicht sagst wo mach ich remote execution bei dir!')
		bot.say('Tipp: !exec-on-peer <peer> <cmd>')
		return

	target_name = bot_params[0]
	target_cmd = bot_params[1]

	# identify requested node or bail out
	node = ffpb_findnode_from_botparam(bot, target_name, ensure_recent_alfreddata=False)
	if node is None: return

	# check ACL
	if not playitsafe(bot, trigger, via_channel=True, node=node):
		return

	# use the node's first non-linklocal address
	target = [x for x in node["network"]["addresses"] if not x.lower().startswith("fe80:")][0]
	target_alias = node["hostname"]

	# assemble SSH command
	cmd = 'ssh -6 -l root ' + target + ' -- "' + target_cmd + '"'
	print("REMOTE EXEC = " + cmd)
	try:
		# call SSH
		result = subprocess.check_output(['ssh', '-6n', '-l', 'root', '-o', 'BatchMode=yes', '-o','StrictHostKeyChecking=no', target, target_cmd], stderr=subprocess.STDOUT, shell=False)

		# fetch results and sent at most 8 of them as response
		lines = str(result).splitlines()

		if len(lines) == 0:
			bot.say('exec-on-peer(' + target_alias + '): No output')
			return

		msg = 'exec-on-peer(' + target_alias + '): ' + str(len(lines)) + ' Zeilen'
		if len(lines) > 8:
			msg += ' (zeige max. 8)'

		bot.say(msg + ':')

		for line in lines[0:8]:
			bot.say(line)

	except subprocess.CalledProcessError as e:
		bot.say('Fehler '+str(e.returncode)+' bei exec-on-peer('+target_alias+'): ' + e.output)