status-bot/modules/ffpb.py

# -*- coding: utf-8 -*-
from __future__ import print_function
import willie

import datetime
import difflib
from email.utils import mktime_tz
import git
import netaddr
import json
import urllib2
import re
import os
import random
import shelve
import subprocess
import time

import dns.resolver,dns.reversename
import socket
import SocketServer
import threading

msgserver = None
peers_repo = None
monitored_nodes = None
highscores = None

alfred_method = None
alfred_data = None
alfred_update = datetime.datetime(1970,1,1,23,42)

ffpb_resolver = dns.resolver.Resolver ()
ffpb_resolver.nameservers = ['10.132.254.53']

class MsgHandler(SocketServer.BaseRequestHandler):
	def handle(self):
		data = self.request.recv(2048).strip()
		sender = self._resolve_name (self.client_address[0])

		bot = self.server.bot
		if bot is None:
			print("ERROR: No bot in handle() :-(")
			return

		target = bot.config.core.owner
		if bot.config.has_section('ffpb'):
			is_public = data.lstrip().lower().startswith("public:")

			if is_public and not (bot.config.ffpb.msg_target_public is None):
				data = data[7:].lstrip()
				target = bot.config.ffpb.msg_target_public
			elif not (bot.config.ffpb.msg_target is None):
				target = bot.config.ffpb.msg_target

		bot.msg(target, "[{0}] {1}".format(sender, str(data)))

	def _resolve_name (self, ip):
		if ip.startswith ("127."):
			return "localhost"

		try:
			addr = dns.reversename.from_address (ip)
			return re.sub ("(.infra)?.ffpb.", "", str (ffpb_resolver.query (addr, "PTR")[0]))
		except dns.resolver.NXDOMAIN:
			return ip

class ThreadingTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
	pass

def setup(bot):
	global msgserver, peers_repo, alfred_method, highscores, monitored_nodes

	bot.memory['ffpb_in_setup'] = True

	highscores = shelve.open('highscoredata', writeback=True)
	if not 'nodes' in highscores:
		highscores['nodes'] = 0
		highscores['nodes_ts'] = time.time()
	if not 'clients' in highscores:
		highscores['clients'] = 0
		highscores['clients_ts'] = time.time()

	monitored_nodes = shelve.open('nodes.monitored', writeback=True)

	seen_nodes = shelve.open('nodes.seen', writeback=True)
	bot.memory['seen_nodes'] = seen_nodes

	if not bot.config.has_section('ffpb'):
		bot.memory['ffpb_in_setup'] = False
		return

	if not bot.config.ffpb.peers_directory is None:
		peers_repo = git.Repo(bot.config.ffpb.peers_directory)
		assert peers_repo.bare == False

	if int(bot.config.ffpb.msg_enable) == 1:
		host = "localhost"
		port = 2342
		if not bot.config.ffpb.msg_host is None: host = bot.config.ffpb.msg_host
		if not bot.config.ffpb.msg_port is None: port = int(bot.config.ffpb.msg_port)

		msgserver = ThreadingTCPServer((host,port), MsgHandler)
		msgserver.bot = bot
		ip, port = msgserver.server_address
		print("Messaging server listening on {}:{}".format(ip,port))

		msgserver_thread = threading.Thread(target=msgserver.serve_forever)
		msgserver_thread.daemon = True
		msgserver_thread.start()

	alfred_method = bot.config.ffpb.alfred_method
	ffpb_updatealfred(bot)

	bot.memory['ffpb_in_setup'] = False

def shutdown(bot):
	global msgserver, highscores, monitored_nodes

	if not highscores is None:
		highscores.sync()
		highscores.close()
		highscores = None

	if not monitored_nodes is None:
		monitored_nodes.sync()
		monitored_nodes.close()
		monitored_nodes = None

	if 'seen_nodes' in bot.memory and bot.memory['seen_nodes'] != None:
		bot.memory['seen_nodes'].close()
		bot.memory['seen_nodes'] = None
		del(bot.memory['seen_nodes'])

	if not msgserver is None:
		msgserver.shutdown()
		print("Closed messaging server.")
		msgserver = None

@willie.module.commands("help")
@willie.module.commands("hilfe")
@willie.module.commands("man")
def ffpb_help(bot, trigger):
	functions = {
		"!ping <knoten>": "Prüfe ob der Knoten erreichbar ist.",
		"!status": "Aktuellen Status des Netzwerks (insb. Anzahl Knoten und Clients) ausgegeben.",
		"!info <knoten>": "Allgemeine Information zu dem Knoten anzeigen.",
		"!link <knoten>": "MAC-Adresse und Link zur Status-Seite des Knotens anzeigen.",
		"!exec-on-peer <knoten> <kommando>": "Befehl auf dem Knoten ausführen (nur möglich bei eigenen Knoten oder als Admin, in beiden Fällen auch nur wenn der SSH-Key des Bots hinterlegt wurde)",
	}
	
	param = trigger.group(2)
	if param is None:
		bot.say("Funktionen: " + str.join(", ", sorted(functions.keys())))
		return

	if param.startswith("!"): param = param[1:]

	for fun in functions.keys():
		if fun.startswith("!" + param + " "):
			bot.say("Hilfe zu '" + fun + "': " + functions[fun])
			return

	bot.say("Allgemeine Hilfe gibt's mit !help - ohne Parameter.")

def ffpb_findnode(name):
	if name is None or len(name) == 0:
		return None

	name = str(name).strip()

	names = {}

	# try to match MAC
	m = re.search("^([0-9a-fA-F][0-9a-fA-F]:){5}[0-9a-fA-F][0-9a-fA-F]$", name)
	if (not m is None):
		mac = m.group(0).lower()
		if mac in alfred_data:
			return alfred_data[mac]

		# try to find alias MAC
		for nodeid in alfred_data:
			node = alfred_data[nodeid]
			if "network" in node:
				if "mac" in node["network"] and node["network"]["mac"].lower() == mac:
					return node
				if "mesh_interfaces" in node["network"]:
					for mim in node["network"]["mesh_interfaces"]:
						if mim.lower() == mac:
							return node

		return {
			'hostname': '?-' + mac.replace(':','').lower(),
			'network': { 'addresses': [ mac2ipv6(mac, 'fdca:ffee:ff12:132:') ], 'mac': mac, },
			'hardware': { 'model': 'derived-from-mac' },
		}

	# look through the ALFRED peers
	for nodeid in alfred_data:
		node = alfred_data[nodeid]
		if 'hostname' in node:
			h = node['hostname']
			if h.lower() == name.lower():
				return node
			else:
				names[h] = nodeid

	# still not found -> try peers_repo
	if not peers_repo is None:
		peer_name = None
		peer_mac = None
		peer_file = None

		for b in peers_repo.heads.master.commit.tree.blobs:
			if b.name.lower() == name.lower():
				peer_name = b.name
				peer_file = b.abspath
				break

		if (not peer_file is None) and os.path.exists(peer_file):
			peerfile = open(peer_file, "r")
			for line in peerfile:
				if line.startswith("# MAC:"):
					peer_mac = line[6:].strip()
			peerfile.close()

		if not (peer_mac is None):
			return {
				'hostname': peer_name,
				'network': { 'addresses': [ mac2ipv6(peer_mac, 'fdca:ffee:ff12:132:') ], 'mac': peer_mac },
				'hardware': { 'model': 'derived-from-vpnkeys' },
			}

	# do a similar name lookup in the ALFRED data
	possibilities = difflib.get_close_matches(name, [ x for x in names ], cutoff=0.8)
	print('findnode: Fuzzy matching \'{0}\' got {1} entries: {2}'.format(name, len(possibilities), ', '.join(possibilities)))
	if len(possibilities) == 1:
		# if we got exactly one candidate that might be it
		return alfred_data[names[possibilities[0]]]

	return None

def ffpb_findnode_from_botparam(bot, name, ensure_recent_alfreddata = True):
	if (name is None or len(name) == 0):
		if not bot is None: bot.reply("Grün.")
		return None

	if ensure_recent_alfreddata and alfred_data is None:
		if not bot is None: bot.say("Informationen sind ausverkauft, kommen erst morgen wieder rein.")
		return None

	if ensure_recent_alfreddata and ffpb_alfred_data_outdated():
		if not bot is None: bot.say("Ich habe gerade keine aktuellen Informationen, daher sage ich mal lieber nichts zu '" + name + "'.")
		return None

	node = ffpb_findnode(name)
	if node is None:
		if not bot is None: bot.say("Kein Plan wer oder was mit '" + name + "' gemeint ist :(")
		
	return node

def mac2ipv6(mac, prefix=None):
	result = str(netaddr.EUI(mac).ipv6_link_local())
	if (not prefix is None) and (result.startswith("fe80::")):
		result = prefix + result[6:]
	return result

@willie.module.interval(30)
def ffpb_updatealfred(bot):
	"""Aktualisiere ALFRED-Daten"""
	global alfred_data, alfred_update

	if alfred_method is None or alfred_method == "None":
		return

	updated = None
	if alfred_method == "exec":
		rawdata = subprocess.check_output(['alfred-json', '-z', '-r', '158'])
		updated = datetime.datetime.now()
	elif alfred_method.startswith("http"):
		try:
			rawdata = urllib2.urlopen(alfred_method)
		except:
			print("Failed to download ALFRED data.")
			return
		updated = datetime.datetime.fromtimestamp(mktime_tz(rawdata.info().getdate_tz("Last-Modified")))
	else:
		print("Unknown ALFRED data method '", alfred_method, "', cannot load new data.", sep="")
		alfred_data = None
		return

	try:
		alfred_data = json.load(rawdata)
		#print("Fetched new ALFRED data:", len(alfred_data), "entries")
		alfred_update = updated

	except ValueError as e:
		print("Failed to parse ALFRED data: " + str(e))
		return

	seen_nodes = bot.memory['seen_nodes'] if 'seen_nodes' in bot.memory else None
	if not seen_nodes is None:
		new = []
		for nodeid in alfred_data:
			nodeid = str(nodeid)
			if not nodeid in seen_nodes:
				seen_nodes[nodeid] = updated
				new.append((nodeid,alfred_data[nodeid]['hostname']))
				print('First time seen: ' + str(nodeid))
		if len(new) > 0 and not bot.memory['ffpb_in_setup']:
			action_msg = None
			if len(new) == 1:
				action_msg = random.choice((
					'bemerkt den neuen Knoten {0}',
					'entdeckt {0}',
					'reibt sich die Augen und erblickt einen verpackungsfrischen Knoten {0}',
					'hat {0} noch nie gesehen',
				)).format('\'' + str(new[0][1]) + '\'')
			else:
				action_msg = random.choice((
					'bemerkt die neuen Knoten {0} und {1}',
					'hat {0} und {1} entdeckt',
					'bewundert {0} sowie {1}',
					'freut sich, dass {0} und {1} nun auch online sind',
				)).format('\'' + '\', \''.join([ str(x[1]) for x in new[0:-1] ]) + '\'', '\'' + str(new[-1][1]) + '\'')
			action_target = bot.config.ffpb.msg_target
			bot.msg(action_target, '\x01ACTION %s\x01' % action_msg)


def ffpb_alfred_data_outdated():
	timeout = datetime.datetime.now() - datetime.timedelta(minutes=5)
	is_outdated = timeout > alfred_update
	#print("ALFRED outdated? {0} (timeout={1} vs. lastupdate={2})".format(is_outdated, timeout, alfred_update))
	return is_outdated

@willie.module.commands('debug-alfred')
def ffpb_debug_alfred(bot, trigger):
	if alfred_data is None:
		bot.say("Keine ALFRED-Daten vorhanden.")
	else:
		bot.say("ALFRED Daten: count={0} lastupdate={1}".format(len(alfred_data), alfred_update))

@willie.module.commands('alfred-data')
def ffpb_peerdata(bot, trigger):
	if (not trigger.admin):
		bot.say('I wont leak (possibly) sensitive data to you.')
		return

	if (not trigger.is_privmsg) and (not trigger.nick in bot.ops[trigger.sender]):
		bot.say('Kein Keks? Keine Daten.')
		return

	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name)
	if node is None: return

	for key in node:
		if key in [ 'hostname' ]: continue
		bot.say("{0}.{1} = {2}".format(node['hostname'], key, str(node[key])))

@willie.module.commands('info')
def ffpb_peerinfo(bot, trigger):
	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name)
	if node is None: return

	info_mac = node["network"]["mac"]
	info_name = node["hostname"]

	info_hw = ""
	if "hardware" in node:
		if "model" in node["hardware"]:
			model = node["hardware"]["model"]
			info_hw = " model='" + model + "'"

	info_fw = ""
	info_update = ""
	if "software" in node:
		if "firmware" in node["software"]:
			fwinfo = str(node["software"]["firmware"]["release"]) if "release" in node["software"]["firmware"] else "unknown"
			info_fw = " firmware=" + fwinfo

		if "autoupdater" in node["software"]:
			autoupdater = node["software"]["autoupdater"]["branch"] if node["software"]["autoupdater"]["enabled"] else "off"
			info_update = " (autoupdater="+autoupdater+")"

	info_uptime = ""
	u = -1
	if "statistics" in node and "uptime" in node["statistics"]:
		u = int(float(node["statistics"]["uptime"]))
	elif 'uptime' in node:
		u = int(float(node['uptime']))

	if u > 0:
		d, r1 = divmod(u, 86400)
		h, r2 = divmod(r1, 3600)
		m, s = divmod(r2, 60)
		if d > 0:
			info_uptime = ' up {0}d {1}h'.format(d,h)
		elif h > 0:
			info_uptime = ' up {0}h {1}m'.format(h,m)
		else:
			info_uptime = ' up {0}m'.format(m)

	bot.say('[{1}]{2}{3}{4}{5}'.format(info_mac, info_name, info_hw, info_fw, info_update, info_uptime))

@willie.module.commands('uptime')
def ffpb_peeruptime(bot, trigger):
	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name)
	if node is None: return

	info_name = node["hostname"]
	info_uptime = ''
	u_raw = None
	if 'statistics' in node and 'uptime' in node['statistics']:
		u_raw = node['statistics']['uptime']
	elif 'uptime' in node:
		u_raw = node['uptime']

	if not u_raw is None:
		u = int(float(u_raw))
		d, r1 = divmod(u, 86400)
		h, r2 = divmod(r1, 3600)
		m, s = divmod(r2, 60)
		if d > 0:
			info_uptime += '{0}d '.format(d)
		if h > 0:
			info_uptime += '{0}h '.format(h)
		info_uptime += '{0}m'.format(m)
		info_uptime += ' # raw: \'{0}\''.format(u_raw)
	else:
		info_uptime += '?'

	bot.say('uptime(\'{0}\') = {1}'.format(info_name, info_uptime))

@willie.module.commands('link')
def ffpb_peerlink(bot, trigger):
	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name)
	if node is None: return

	info_mac = node["network"]["mac"]
	info_name = node["hostname"]
	info_v6 = mac2ipv6(info_mac, 'fdca:ffee:ff12:132:')
	bot.say('[{1}] mac {0} -> http://[{2}]/'.format(info_mac, info_name, info_v6))

@willie.module.interval(60)
def ffpb_updatepeers(bot):
	"""Aktualisiere die Knotenliste und melde das Diff"""
	if peers_repo is None:
		print('WARNING: peers_repo is None')
		return

	old_head = peers_repo.head.commit
	peers_repo.remotes.origin.pull()
	new_head = peers_repo.head.commit

	if new_head != old_head:
		print('git pull: from ' + str(old_head) + ' to ' + str(new_head))

		added = []
		changed = []
		renamed = []
		deleted = []
		for f in old_head.diff(new_head):
			if f.new_file:
				added.append(f.b_blob.name)

			elif f.deleted_file:
				deleted.append(f.a_blob.name)

			elif f.renamed:
				renamed.append([f.rename_from, f.rename_to])

			else:
				changed.append(f.a_blob.name)

		response = "Knoten-Update (VPN +{0} %{1} -{2}): ".format(len(added), len(renamed)+len(changed), len(deleted))
		for f in added:
			response += " +'{}'".format(f)
		for f in changed:
			response += " %'{}'".format(f)
		for f in renamed:
			response += " '{}'->'{}'".format(f[0],f[1])
		for f in deleted:
			response += " -'{}'".format(f)

		bot.msg(bot.config.ffpb.msg_target, response)

def ffpb_fetch_stats(bot, url, memoryid):
	response = urllib2.urlopen(url)

	data = json.load(response)
	nodes_active = 0
	nodes_total = 0
	clients_count = 0

	for node in data['nodes']:
		if node['flags']['gateway'] or node['flags']['client']:
			continue

		nodes_total += 1
		if node['flags']['online']:
			nodes_active += 1

		if 'legacy' in node['flags'] and node['flags']['legacy']:
			clients_count -= 1

	for link in data['links']:
		if link['type'] == 'client':
			clients_count += 1

	if not memoryid in bot.memory:
		bot.memory[memoryid] = { }
	stats = bot.memory[memoryid]
	stats["fetchtime"] = time.time()
	stats["nodes_active"] = nodes_active
	stats["nodes_total"] = nodes_total
	stats["clients"] = clients_count

	return (nodes_active, nodes_total, clients_count)

@willie.module.interval(15)
def ffpb_get_stats(bot):
	(nodes_active, nodes_total, clients_count) = ffpb_fetch_stats(bot, 'http://map.paderborn.freifunk.net/nodes.json', 'ffpb_stats')
	
	highscore_changed = False
	if nodes_active > highscores['nodes']:
		highscores['nodes'] = nodes_active
		highscores['nodes_ts'] = time.time()
		highscore_changed = True

	if clients_count > highscores['clients']:
		highscores['clients'] = clients_count
		highscores['clients_ts'] = time.time()
		highscore_changed = True

	if highscore_changed:
		print('HIGHSCORE changed: {0} nodes ({1}), {2} clients ({3})'.format(highscores['nodes'], highscores['nodes_ts'], highscores['clients'], highscores['clients_ts']))
		if not (bot.config.ffpb.msg_target is None):
			action_msg = 'notiert sich den neuen Highscore: {0} Knoten ({1}), {2} Clients ({3})'.format(highscores['nodes'], pretty_date(int(highscores['nodes_ts'])), highscores['clients'], pretty_date(int(highscores['clients_ts'])))
			action_target = bot.config.ffpb.msg_target
			if (not bot.config.ffpb.msg_target_public is None):
				action_target = bot.config.ffpb.msg_target_public
			bot.msg(action_target, '\x01ACTION %s\x01' % action_msg)

@willie.module.commands('status')
def ffpb_status(bot, trigger):
	"""Status des FFPB-Netzes: Anzahl (aktiver) Knoten + Clients"""
	stats = bot.memory['ffpb_stats'] if 'ffpb_stats' in bot.memory else None
	if stats is None:
		bot.say('Uff, kein Plan wo der Zettel ist. Fragst du später nochmal?')
		return

	bot.say('Es sind {0} Knoten und ca. {1} Clients online.'.format(stats["nodes_active"], stats["clients"]))

def pretty_date(time=False):
    """
    Get a datetime object or a int() Epoch timestamp and return a
    pretty string like 'an hour ago', 'Yesterday', '3 months ago',
    'just now', etc
    """
    from datetime import datetime
    now = datetime.now()
    compare = None
    if type(time) is int:
        compare = datetime.fromtimestamp(time)
    elif type(time) is float:
        compare = datetime.fromtimestamp(int(time))
    elif isinstance(time,datetime):
        compare = time
    elif not time:
        compare = now
    diff = now - compare
    second_diff = diff.seconds
    day_diff = diff.days

    if day_diff < 0:
        return ''

    if day_diff == 0:
        if second_diff < 10:
            return "gerade eben"
        if second_diff < 60:
            return "vor " + str(second_diff) + " Sekunden"
        if second_diff < 120:
            return "vor einer Minute"
        if second_diff < 3600:
            return "vor " + str(second_diff / 60) + " Minuten"
        if second_diff < 7200:
            return "vor einer Stunde"
        if second_diff < 86400:
            return "vor " + str(second_diff / 3600) + " Stunden"
    if day_diff == 1:
        return "gestern"
    if day_diff < 7:
        return "vor " + str(day_diff) + " Tagen"
    return "am " + compare.strftime('%d.%m.%Y um %H:%M Uhr') 

@willie.module.commands('highscore')
def ffpb_highscore(bot, trigger):
	bot.say('Highscore: {0} Knoten ({1}), {2} Clients ({3})'.format(
		highscores['nodes'], pretty_date(int(highscores['nodes_ts'])),
		highscores['clients'], pretty_date(int(highscores['clients_ts']))))

@willie.module.commands('rollout-status')
def ffpb_rolloutstatus(bot, trigger):
	result = { }
	for branch in [ 'stable', 'testing' ]:
		result[branch] = None
	skipped = 0

	if (not (trigger.admin and trigger.is_privmsg)) and (not trigger.nick in bot.ops[trigger.sender]):
		bot.say('Geh zur dunklen Seite, die haben Kekse - ohne Keks kein Rollout-Status.')
		return

	expected_release = trigger.group(2)
	if expected_release is None or len(expected_release) == 0:
		bot.say('Von welcher Firmware denn?')
		return

	for nodeid in alfred_data:
		item = alfred_data[nodeid]
		if (not 'software' in item) or (not 'firmware' in item['software']) or (not 'autoupdater' in item['software']):
			skipped+=1
			continue

		release = item['software']['firmware']['release']
		branch = item['software']['autoupdater']['branch']
		enabled = item['software']['autoupdater']['enabled']
		if not branch in result or result[branch] is None:
			result[branch] = { 'auto_count': 0, 'auto_not': 0, 'manual_count': 0, 'manual_not': 0, 'total': 0 }

		result[branch]['total'] += 1
		match = 'count' if release == expected_release else 'not'
		mode = 'auto' if enabled else 'manual'
		result[branch][mode+'_'+match] += 1

	output = "Rollout von '{0}':".format(expected_release)
	for branch in result:
		auto_count = result[branch]['auto_count']
		auto_total = auto_count + result[branch]['auto_not']
		manual_count = result[branch]['manual_count']
		manual_total = manual_count + result[branch]['manual_not']
		bot.say("Rollout von '{0}': {1} = {2}/{3} per Auto-Update, {4}/{5} manuell".format(expected_release, branch, auto_count, auto_total, manual_count, manual_total))
	if skipped > 0:
		bot.say("Rollout von '{0}': {1} Knoten unklar".format(expected_release, skipped))

@willie.module.commands('ping')
def ffpb_ping(bot, trigger=None, target_name=None):
	"""Ping FFPB-Knoten"""
	if target_name is None: target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name, ensure_recent_alfreddata=False)
	if node is None: return None

	target = [x for x in node["network"]["addresses"] if not x.lower().startswith("fe80:")][0]
	target_alias = node["hostname"]

	print("pinging '{0}' at {1} ...".format(target_name, target))
	result = os.system('ping6 -c 2 -W 1 ' + target + ' >/dev/null')
	if result == 0:
		print("ping to '{0}' succeeded".format(target_name))
		if not bot is None: bot.say('Knoten "' + target_alias + '" antwortet \o/')
		return True
	elif result == 1 or result == 256:
		print("ping to '{0}' failed".format(target_name))
		if not bot is None: bot.say('Keine Antwort von "' + target_alias + '" :-(')
		return False
	else:
		print("ping to '{0}' broken: result='{1}'".format(target_name, result))
		if not bot is None: bot.say('Uh oh, irgendwas ist kaputt. Chef, ping result = ' + str(result) + ' - darf ich das essen?')
		return None

@willie.module.interval(3*60)
def ffpb_monitor_ping(bot):
	notify_target = bot.config.core.owner
	if (not bot.config.ffpb.msg_target is None):
		notify_target = bot.config.ffpb.msg_target

	for node in monitored_nodes:
		mon = monitored_nodes[node]
		added = mon['added']
		last_status = mon['status']
		last_check = mon['last_check']
		last_success = mon['last_success']

		current_status = ffpb_ping(bot=None, target_name=node)
		if current_status is None: current_status = False
		mon['status'] = current_status
		mon['last_check'] = time.time()
		if current_status == True: mon['last_success'] = time.time()
		print("Monitoring ({0}) {1} (last: {2} at {3})".format(node, current_status, last_status, time.strftime('%Y-%m-%d %H:%M', time.localtime(last_check))))
		if last_status != current_status and (last_status or current_status):
			if last_check is None:
				# erster Check, keine Ausgabe
				continue

			if current_status == True:
				bot.msg(notify_target, 'Monitoring: Knoten \'{0}\' pingt wieder (zuletzt {1})'.format(node, pretty_date(last_success) if not last_success is None else '[nie]'))
			else:
				bot.msg(notify_target, 'Monitoring: Knoten \'{0}\' DOWN'.format(node))

@willie.module.commands('monitor')
def ffpb_monitor(bot, trigger):
	if not trigger.admin:
		bot.say('Ich ping hier nicht für jeden durch die Weltgeschichte.')
		return

	if trigger.group(2) is None or len(trigger.group(2)) == 0:
		bot.say('Das Monitoring sagt du hast doofe Ohren.')
		return

	cmd = trigger.group(3)
	node = trigger.group(4)
	if not node is None: node = str(node)

	if cmd == "add":
		if node in monitored_nodes:
			bot.say('Knoten \'{0}\' wird bereits gemonitored.'.format(node))
			return

		monitored_nodes[node] = {
			'added': trigger.sender, 
			'status': None,
			'last_check': None,
			'last_success': None,
		}

		bot.say('Knoten \'{0}\' wird jetzt ganz genau beobachtet.'.format(node))
		return

	if cmd == "del":
		if not node in monitored_nodes:
			bot.say('Knoten \'{0}\' war gar nicht im Monitoring?!?'.format(node))
			return
		del monitored_nodes[node]
		bot.say('Okidoki, \'{0}\' lasse ich jetzt links liegen.'.format(node))
		return

	if cmd == "info":
		if node in monitored_nodes:
			info = monitored_nodes[node]
			bot.say('Knoten \'{0}\' wurde zuletzt {1} gepingt (Ergebnis: {2})'.format(node, pretty_date(info['last_check']) if not info['last_check'] is None else "^W noch nie", info['status']))
		else:
			bot.say('Knoten \'{0}\' ist nicht im Monitoring.'.format(node))
		return

	if cmd == "list":
		nodes = ""
		for node in monitored_nodes:
			nodes = nodes + " " + node
		bot.say('Monitoring aktiv für:' + nodes)
		return

	if cmd == "help":
		bot.say('Entweder "!monitor list" oder "!monitor {add|del|info} <node>"')
		return

	bot.say('Mit "' + str(cmd) + '" kann ich nix anfangen, probier doch mal "!monitor help".')

@willie.module.commands('exec-on-peer')
def ffpb_remoteexec(bot, trigger):
	"""Remote Execution fuer FFPB_Knoten"""
	bot_params = trigger.group(2).split(' ',1)
	if len(bot_params) != 2:
		bot.say('Wenn du nicht sagst wo mach ich remote execution bei dir!')
		bot.say('Tipp: !exec-on-peer <peer> <cmd>')
		return

	target_name = bot_params[0]
	target_cmd = bot_params[1]

	if not trigger.admin:
		bot.say('I can haz sudo?')
		return

	if trigger.is_privmsg:
		bot.say('Bitte per Channel.')
		return

	if not trigger.nick in bot.ops[trigger.sender]:
		bot.say('Geh weg.')
		return

	node = ffpb_findnode_from_botparam(bot, target_name, ensure_recent_alfreddata=False)
	if node is None: return

	target = [x for x in node["network"]["addresses"] if not x.lower().startswith("fe80:")][0]
	target_alias = node["hostname"]

	cmd = 'ssh -6 -l root ' + target + ' -- "' + target_cmd + '"'
	print("REMOTE EXEC = " + cmd)
	try:
		result = subprocess.check_output(['ssh', '-6n', '-l', 'root', '-o', 'BatchMode=yes', '-o','StrictHostKeyChecking=no', target, target_cmd], stderr=subprocess.STDOUT, shell=False)
		lines = str(result).splitlines()

		if len(lines) == 0:
			bot.say('exec-on-peer(' + target_alias + '): No output')
			return

		msg = 'exec-on-peer(' + target_alias + '): ' + str(len(lines)) + ' Zeilen'
		if len(lines) > 8:
			msg += ' (zeige max. 8)'

		bot.say(msg + ':')

		for line in lines[0:8]:
			bot.say(line)

	except subprocess.CalledProcessError as e:
		bot.say('Fehler '+str(e.returncode)+' bei exec-on-peer('+target_alias+'): ' + e.output)