status-bot/modules/ffpb.py

# -*- coding: utf-8 -*-
from __future__ import print_function
import willie

import datetime
import difflib
from email.utils import mktime_tz
import git
import netaddr
import json
import urllib2
import re
import os
import shelve
import subprocess
import time

import dns.resolver,dns.reversename
import socket
import SocketServer
import threading

msgserver = None
peers_repo = None

alfred_method = None

ffpb_resolver = dns.resolver.Resolver ()
ffpb_resolver.nameservers = ['10.132.254.53']

class MsgHandler(SocketServer.BaseRequestHandler):
	"""Reads line from TCP stream and forwards it to configured IRC channels."""

	def handle(self):
		data = self.request.recv(2048).strip() 
		sender = self._resolve_name (self.client_address[0])

		bot = self.server.bot
		if bot is None:
			print("ERROR: No bot in handle( ) :-(")
			return

		target = bot.config.core.owner
		if bot.config.has_section('ffpb'):
			is_public = data.lstrip().lower().startswith("public:")

			if is_public and not (bot.config.ffpb.msg_target_public is None):
				data = data[7:].lstrip()
				target = bot.config.ffpb.msg_target_public
			elif not (bot.config.ffpb.msg_target is None):
				target = bot.config.ffpb.msg_target

		bot.msg(target, "[{0}] {1}".format(sender, str(data)))

	def _resolve_name (self, ip):
		"""Resolves the host name of the given IP address
		and strips away the suffix (.infra)?.ffpb"""

		if ip.startswith ("127."):
			return "localhost"

		try:
			addr = dns.reversename.from_address (ip)
			return re.sub ("(.infra)?.ffpb.", "", str (ffpb_resolver.query (addr, "PTR")[0]))
		except dns.resolver.NXDOMAIN:
			return ip

class ThreadingTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
	pass

def setup(bot):
	global msgserver, peers_repo, alfred_method

	bot.memory['ffpb_in_setup'] = True

	# open highscores file (backed to filesystem)
	if 'highscores' in bot.memory and not bot.memory['highscores'] is None:
		bot.memory['highscores'].close()
	highscores = shelve.open('highscoredata', writeback=True)
	if not 'nodes' in highscores:
		highscores['nodes'] = 0
		highscores['nodes_ts'] = time.time()
	if not 'clients' in highscores:
		highscores['clients'] = 0
		highscores['clients_ts'] = time.time()
	bot.memory['highscores'] = highscores

	seen_nodes = shelve.open('nodes.seen', writeback=True)
	bot.memory['seen_nodes'] = seen_nodes

	# no need to configure anything else if the ffpb config section is missing
	if not bot.config.has_section('ffpb'):
		bot.memory['ffpb_in_setup'] = False
		return

	# open the git repository containing the peers files
	if not bot.config.ffpb.peers_directory is None:
		peers_repo = git.Repo(bot.config.ffpb.peers_directory)
		assert peers_repo.bare == False

	# if configured start the messaging server
	if int(bot.config.ffpb.msg_enable) == 1:
		host = "localhost"
		port = 2342
		if not bot.config.ffpb.msg_host is None: host = bot.config.ffpb.msg_host
		if not bot.config.ffpb.msg_port is None: port = int(bot.config.ffpb.msg_port)

		msgserver = ThreadingTCPServer((host,port), MsgHandler)
		msgserver.bot = bot
		ip, port = msgserver.server_address
		print("Messaging server listening on {}:{}".format(ip,port))

		msgserver_thread = threading.Thread(target=msgserver.serve_forever)
		msgserver_thread.daemon = True
		msgserver_thread.start()

	# initially fetch ALFRED data
	alfred_method = bot.config.ffpb.alfred_method
	ffpb_updatealfred(bot)

	bot.memory['ffpb_in_setup'] = False

def shutdown(bot):
	global msgserver

	# store highscores
	if 'highscores' in bot.memory and not bot.memory['highscores'] is None:
		bot.memory['highscores'].sync()
		bot.memory['highscores'].close()
		del(bot.memory['highscores'])

	# store seen_nodes
	if 'seen_nodes' in bot.memory and bot.memory['seen_nodes'] != None:
		bot.memory['seen_nodes'].close()
		bot.memory['seen_nodes'] = None
		del(bot.memory['seen_nodes'])

	# shut down messaging server
	if not msgserver is None:
		msgserver.shutdown() 
		print("Closed messaging server.")
		msgserver = None

@willie.module.commands("help")
@willie.module.commands("hilfe")
@willie.module.commands("man")
def ffpb_help(bot, trigger):
	"""Meldet häufig benutzte Funktionen."""

	functions = {
		"!ping <knoten>": "Prüfe ob der Knoten erreichbar ist.",
		"!status": "Aktuellen Status des Netzwerks (insb. Anzahl Knoten und Clients) ausgegeben.",
		"!info <knoten>": "Allgemeine Information zu dem Knoten anzeigen.",
		"!link <knoten>": "MAC-Adresse und Link zur Status-Seite des Knotens anzeigen.",
		"!exec-on-peer <knoten> <kommando>": "Befehl auf dem Knoten ausführen (nur möglich bei eigenen Knoten oder als Admin, in beiden Fällen auch nur wenn der SSH-Key des Bots hinterlegt wurde)",
	}
	
	param = trigger.group(2)
	if param is None:
		bot.say("Funktionen: " + str.join(", ", sorted(functions.keys())))
		return

	if param.startswith("!"): param = param[1:]

	for fun in functions.keys():
		if fun.startswith("!" + param + " "):
			bot.say("Hilfe zu '" + fun + "': " + functions[fun])
			return

	bot.say("Allgemeine Hilfe gib t's mit !help - ohne Parameter.")

def ffpb_findnode(bot, name):
	"""helper: try to identify the node the user meant by the given name"""

	# no name, no node
	if name is None or len(name) == 0:
		return None

	name = str(name).strip()
	names = {}

	alfred_data = bot.memory['alfred_data'] if 'alfred_data' in bot.memory else None

	if not alfred_data is None:
		# try to match MAC in ALFRED data
		m = re.search("^([0-9a-fA-F][0-9a-fA-F]:){5}[0-9a-fA-F][0-9a-fA-F]$", name)
		if (not m is None):
			mac = m.group(0).lower()
			if mac in alfred_data:
				return alfred_data[mac]

			# try to find alias MAC
			for nodeid in alfred_data:
				node = alfred_data[nodeid]
				if "network" in node:
					if "mac" in node["network"] and node["network"]["mac"].lower() == mac:
						return node
					if "mesh_interfaces" in node["network"]:
						for mim in node["network"]["mesh_interfaces"]:
							if mim.lower() == mac:
 								return node

			return {
				'hostname': '?-' + mac.replace(':','').lower(),
				'network': { 'addresses': [ mac2ipv6(mac, 'fdca:ffee:ff12:132:') ], 'mac': mac, },
 				'hardware': { 'model': 'derived-from-mac' },
			}

		# look through the ALFRED peers
		for nodeid in alfred_data:
			node = alfred_data[nodeid]
			if 'hostname' in node:
				h = node['hostname']
				if h.lower() == name.lower():
					return node
				else:
 					names[h] = nodeid

	# try peers_repo
	if not peers_repo is None:
		peer_name = None
		peer_mac = None
		peer_file = None

		for b in peers_repo.heads.master.commit.tree.blobs:
			if b.name.lower() == name.lower():
				peer_name = b.name
				peer_file = b.abspath
				break

		if (not peer_file is None) and os.path.exists(peer_file):
			peerfile = open(peer_file, "r")
			for line in peerfile:
				if line.startswith("# MAC:"):
					peer_mac = line[6:].strip()
			peerfile.close()

		if not (peer_mac is None):
			return {
				'hostname': peer_name,
				'network': { 'addresses': [ mac2ipv6(peer_mac, 'fdca:ffee:ff12:132:') ], 'mac': peer_mac },
				'hardware': { 'model': 'derived-from-vpnkeys' },
			}

	# do a similar name lookup in the ALFRED data
	if not alfred_data is None:
		possibilities = difflib.get_close_matches(name, [ x for x in names ], cutoff=0.8)
		print('findnode: Fuzzy matching \'{0}\' got {1} entries: {2}'.format(name, len(possibilities), ', '.join(possibilities)))
		if len(possibilities) == 1:
			# if we got exactly one candidate that might be it
			return alfred_data[names[possibilities[0]]]

	# none of the above was able to identify the requested node
	return None

def ffpb_get_alfreddata(bot, ensure_recent=True):
	"""helper: return current ALFRED data (or None, 
	if the data is outdated and ensure_recent is set)"""

	if not 'alfred_data' in bot.memory or bot.memory['alfred_data'] is None:
		return None

	if ensure_recent:
		# get timestamp of last ALFRED update (set by ffpb_updatealfred())
		alfred_update = bot.memory['alfred_update'] if 'alfred_update' in bot.memory else None
		if alfred_update is None: return None

		# data must not be older than 5 minutes
		timeout = datetime.datetime.now() - datetime.timedelta(minutes=5)
		is_outdated = timeout > alfred_update
		#print("ALFRED outdated? {0} (timeout={1} vs. lastupdate={2})".format(is_outdated, timeout, alfred_update))
		if is_outdated:
			return None

	return bot.memory['alfred_data']

def ffpb_findnode_from_botparam(bot, name, ensure_recent_alfreddata = True):
	"""helper: call ffpb_findnode() and give common answers via bot if nothing has been found"""
	if (name is None or len(name) == 0):
		bot.reply("Grün.")
		return None

	alfred_data = ffpb_get_alfreddata(bot, ensure_recent_alfreddata)
	if alfred_data is None and ensure_recent_alfreddata:
		bot.say("Ich habe gerade keine (aktuellen) Informationen, daher sage ich mal lieber nichts zu '" + name + "'.")
		return None 

	node = ffpb_findnode(bot, name)
	if node is None:
		bot.say("Kein Plan wer oder was mit '" + name + "' gemeint ist :(")
		
	return node

def mac2ipv6(mac, prefix=None):
	"""Calculate IPv6 address from given MAC,
	optionally replacing the fe80:: prefix with a given one."""
	result = str(netaddr.EUI(mac).ipv6_link_local())
	if (not prefix is None) and (result.startswith("fe80::")):
		result = prefix + result[6:]
	return result

@willie.module.interval(30)
def ffpb_updatealfred(bot):
	"""Aktualisiere ALFRED-Daten"""

	if alfred_method is None or alfred_method == "None":
		return

	alfred_data = None
	updated = None
	if alfred_method == "exec":
		rawdata = subprocess.check_output(['alfred-json', '-z', '-r', '158'])
		updated = datetime.datetime.now()
	elif alfred_method.startswith("http"):
		try:
			rawdata = urllib2.urlopen(alfred_method)
		except:
			print("Failed to download ALFRED data.")
			return
		updated = datetime.datetime.fromtimestamp(mktime_tz(rawdata.info().getdate_tz("Last-Modified")))
	else:
		print("Unknown ALFRED data method '", alfred_method, "', cannot load new data.", sep="")
		bot.memory['alfred_data'] = None
		return

	try:
		alfred_data = json.load(rawdata)
		#print("Fetched new ALFRED data:", len(alfred_data), "entries")

	except ValueError as e:
		print("Failed to parse ALFRED data: " + str(e))
		return

	bot.memory['alfred_data'] = alfred_data
	bot.memory['alfred_update'] = updated

	seen_nodes = bot.memory['seen_nodes'] if 'seen_nodes' in bot.memory else None
	if not seen_nodes is None:
		new = []
		for nodeid in alfred_data:
			nodeid = str(nodeid)
			if not nodeid in seen_nodes:
				seen_nodes[nodeid] = updated
				new.append((nodeid,alfred_data[nodeid]['hostname']))
				print('First time seen: ' + str(nodeid))
		if len(new) > 0 and not bot.memory['ffpb_in_setup']:
			action_msg = 'bemerkt '
			if len(new) == 1:
				action_msg += 'den neuen Knoten \'' + str(new[0][1]) + '\''
			else:
				action_msg += 'die neuen Knoten \'' + '\', \''.join([ str(x[1]) for x in new[0:-1] ]) + '\' und \'' + str(new[-1][1]) + '\''
			action_target = bot.config.ffpb.msg_target
			bot.msg(action_target, '\x01ACTION %s\x01' % action_msg)

@willie.module.commands('debug-alfred')
def ffpb_debug_alfred(bot, trigger):
	"""Zeige Stand der Alfred-Daten an."""

	alfred_data = ffpb_get_alfreddata(bot)

	if alfred_data is None:
		bot.say("Keine ALFRED-Daten vorhanden.")
	else:
		bot.say("ALFRED Daten: count={0} lastupdate={1}".format(len(alfred_data), bot.memory['alfred_update'] if 'alfred_memory' in bot.memory else '?'))

@willie.module.commands('alfred-data')
def ffpb_peerdata(bot, trigger):
	"""Zeige Daten zum angegebenen Node an."""

	# user must be a bot admin
	if (not trigger.admin):
		bot.say('I wont leak (possibly) sensitive data to you.')
		return

	# query must be a PM or as OP in the channel
	if (not trigger.is_privmsg) and (not trigger.nick in bot.ops[trigger.sender]):
		bot.say('Kein Keks? Keine Daten.')
		return

	# identify node or bail out
	target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name)
	if node is None: return

	# reply each key in the node's data
	for key in node:
		if key in [ 'hostname' ]: continue
		bot.say("{0}.{1} = {2}".format(node['hostname'], key, str(node[key])))

@willie.module.interval(60)
def ffpb_updatepeers(bot):
	"""Aktualisiere die Knotenliste und melde das Diff"""

	if peers_repo is None:
		print('WARNING: peers_repo is None')
		return

	old_head = peers_repo.head.commit
	peers_repo.remotes.origin.pull()
	new_head = peers_repo.head.commit

	if new_head != old_head:
		print('git pull: from ' + str(old_head) + ' to ' + str(new_head))

		added = []
		changed = []
		renamed = []
		deleted = []
		for f in old_head.diff(new_head):
			if f.new_file:
				added.append(f.b_blob.name)

			elif f.deleted_file:
				deleted.append(f.a_blob.name)

			elif f.renamed:
				renamed.append([f.rename_from, f.rename_to])

			else:
				changed.append(f.a_blob.name)

		response = "Knoten-Update (VPN +{0} %{1} -{2}): ".format(len(added), len(renamed)+len(changed), len(deleted))
		for f in added:
			response += " +'{}'".format(f)
		for f in changed:
			response += " %'{}'".format(f)
		for f in renamed:
			response += " '{}'->'{}'".format(f[0],f[1])
		for f in deleted:
			response += " -'{}'".format(f)

		bot.msg(bot.config.ffpb.msg_target, response)

def ffpb_fetch_stats(bot, url, memoryid):
	"""Fetch a ffmap-style nodes.json from the given URL and
	store it in the bot's memory."""

	response = urllib2.urlopen(url)

	data = json.load(response)
	nodes_active = 0
	nodes_total = 0
	clients_count = 0

	for node in data['nodes']:
		if node['flags']['gateway'] or node['flags']['client']:
			continue

		nodes_total += 1
		if node['flags']['online']:
			nodes_active += 1

		if 'legacy' in node['flags'] and node['flags']['legacy']:
			clients_count -= 1

	for link in data['links']:
		if link['type'] == 'client':
			clients_count += 1

	if not memoryid in bot.memory:
		bot.memory[memoryid] = { }
	stats = bot.memory[memoryid]
	stats["fetchtime"] = time.time()
	stats["nodes_active"] = nodes_active
	stats["nodes_total"] = nodes_total
	stats["clients"] = clients_count

	return (nodes_active, nodes_total, clients_count)

@willie.module.interval(15)
def ffpb_get_stats(bot):
	"""Hole aktuelle Statistik-Daten, falls sich der Highscore ändert melde dies."""

	highscores = bot.memory['highscores'] if 'highscores' in bot.memory else None
	if highscores is None:
		print('HIGHSCORE not in bot memory')
		return

	(nodes_active, nodes_total, clients_count) = ffpb_fetch_stats(bot, 'http://map.paderborn.freifunk.net/nodes.json', 'ffpb_stats')
	
	highscore_changed = False
	if nodes_active > highscores['nodes']:
		highscores['nodes'] = nodes_active
		highscores['nodes_ts'] = time.time()
		highscore_changed = True

	if clients_count > highscores['clients']:
		highscores['clients'] = clients_count
		highscores['clients_ts'] = time.time()
		highscore_changed = True

	if highscore_changed:
		print('HIGHSCORE changed: {0} nodes ({1}), {2} clients ({3})'.format(highscores['nodes'], highscores['nodes_ts'], highscores['clients'], highscores['clients_ts']))
		if not (bot.config.ffpb.msg_target is None):
			action_msg = 'notiert sich den neuen Highscore: {0} Knoten ({1}), {2} Clients ({3})'.format(highscores['nodes'], pretty_date(int(highscores['nodes_ts'])), highscores['clients'], pretty_date(int(highscores['clients_ts'])))
			action_target = bot.config.ffpb.msg_target
			if (not bot.config.ffpb.msg_target_public is None):
				action_target = bot.config.ffpb.msg_target_public
			bot.msg(action_target, '\x01ACTION %s\x01' % action_msg)

def pretty_date(time=False):
    """
    Get a datetime object or a int() Epoch timestamp and return a
    pretty string like 'an hour ago', 'Yesterday', '3 months ago',
    'just now', etc
    """
    from datetime import datetime
    now = datetime.now()
    compare = None
    if type(time) is int:
        compare = datetime.fromtimestamp(time)
    elif type(time) is float:
        compare = datetime.fromtimestamp(int(time))
    elif isinstance(time,datetime):
        compare = time
    elif not time:
        compare = now
    diff = now - compare
    second_diff = diff.seconds
    day_diff = diff.days

    if day_diff < 0:
        return ''

    if day_diff == 0:
        if second_diff < 10:
            return "gerade eben"
        if second_diff < 60:
            return "vor " + str(second_diff) + " Sekunden"
        if second_diff < 120:
            return "vor einer Minute"
        if second_diff < 3600:
            return "vor " + str(second_diff / 60) + " Minuten"
        if second_diff < 7200:
            return "vor einer Stunde"
        if second_diff < 86400:
            return "vor " + str(second_diff / 3600) + " Stunden"
    if day_diff == 1:
        return "gestern"
    if day_diff < 7:
        return "vor " + str(day_diff) + " Tagen"
    return "am " + compare.strftime('%d.%m.%Y um %H:%M Uhr') 

@willie.module.commands('ping')
def ffpb_ping(bot, trigger=None, target_name=None):
	"""Ping an Knoten"""

	if target_name is None: target_name = trigger.group(2)
	node = ffpb_findnode_from_botparam(bot, target_name, ensure_recent_alfreddata=False)
	if node is None: return None

	target = [x for x in node["network"]["addresses"] if not x.lower().startswith("fe80:")][0]
	target_alias = node["hostname"]

	print("pinging '{0}' at {1} ...".format(target_name, target))
	result = os.system('ping6 -c 2 -W 1 ' + target + ' >/dev/null')
	if result == 0:
		print("ping to '{0}' succeeded".format(target_name))
		if not bot is None: bot.say('Knoten "' + target_alias + '" antwortet \o/')
		return True
	elif result == 1 or result == 256:
		print("ping to '{0}' failed".format(target_name))
		if not bot is None: bot.say('Keine Antwort von "' + target_alias + '" :-(')
		return False
	else:
		print("ping to '{0}' broken: result='{1}'".format(target_name, result))
		if not bot is None: bot.say('Uh oh, irgendwas ist kaputt. Chef, ping result = ' + str(result) + ' - darf ich das essen?')
		return None

@willie.module.commands('exec-on-peer')
def ffpb_remoteexec(bot, trigger):
	"""Remote Execution für Knoten (mit SSH-Key des Bots)"""

	bot_params = trigger.group(2).split(' ',1)
	if len(bot_params) != 2:
		bot.say('Wenn du nicht sagst wo mach ich remote execution bei dir!')
		bot.say('Tipp: !exec-on-peer <peer> <cmd>')
		return

	target_name = bot_params[0]
	target_cmd = bot_params[1]

	# remote execution may only be trigger by bot admins
	if not trigger.admin:
		bot.say('I can haz sudo?')
		return

	# make sure remote execution is done in public
	if trigger.is_privmsg:
		bot.say('Bitte per Channel.')
		return

	# double-safety: user must be op in the channel, too (hoping for NickServ authentication)
	if not trigger.nick in bot.ops[trigger.sender]:
		bot.say('Geh weg.')
		return

	# identify requested node or bail out
	node = ffpb_findnode_from_botparam(bot, target_name, ensure_recent_alfreddata=False)
	if node is None: return

	# use the node's first non-linklocal address
	target = [x for x in node["network"]["addresses"] if not x.lower().startswith("fe80:")][0]
	target_alias = node["hostname"]

	# assemble SSH command
	cmd = 'ssh -6 -l root ' + target + ' -- "' + target_cmd + '"'
	print("REMOTE EXEC = " + cmd)
	try:
		# call SSH
		result = subprocess.check_output(['ssh', '-6n', '-l', 'root', '-o', 'BatchMode=yes', '-o','StrictHostKeyChecking=no', target, target_cmd], stderr=subprocess.STDOUT, shell=False)

		# fetch results and send at most 8 of them as response
		lines = str(result).splitlines()

		if len(lines) == 0:
			bot.say('exec-on-peer(' + target_alias + '): No output')
			return

		msg = 'exec-on-peer(' + target_alias + '): ' + str(len(lines)) + ' Zeilen'
		if len(lines) > 8:
			msg += ' (zeige max. 8)'

		bot.say(msg + ':')

		for line in lines[0:8]:
			bot.say(line)

	except subprocess.CalledProcessError as e:
		bot.say('Fehler '+str(e.returncode)+' bei exec-on-peer('+target_alias+'): ' + e.output)