move data merge logic from batcave.py into BaseStorage

batcave.py

@@ -116,39 +116,9 @@ while True:
             d.push(newdata)
 
         logger.debug('Step 3/3: Merging current data ...')
-        temp = dict_merge(storage.data, {})
-        for x in temp:
-            if not x in newdata:
-                continue
-            temp[x]['aliases'] = []
-            temp[x]['clients'] = []
-            temp[x]['neighbours'] = []
-            if not '__RAW__' in temp[x]:
-                temp[x]['__RAW__'] = {}
-            if '__RAW__' in newdata[x]:
-                for key in newdata[x]['__RAW__']:
-                    if key in temp[x]['__RAW__']:
-                        del temp[x]['__RAW__'][key]
-
-        storage.data = dict_merge(temp, newdata)
-        # sanitize each item's data
-        for itemid in storage.data:
-            if itemid.startswith('__'):
-                continue
-            item = storage.data[itemid]
-
-            # remove node's MACs from clients list
-            clients = [x for x in item['clients']] if 'clients' in item else []
-            if 'mac' in item and item['mac'] in clients:
-                clients.remove(item['mac'])
-            if 'macs' in item:
-                for x in item['macs']:
-                    if x in clients:
-                        clients.remove(x)
-            storage.data[itemid]['clientcount'] = len(clients)
-        logger.debug('I have data for ' + str(len(storage.data)) + ' nodes.')
-
-        storage.save()
+        storage.merge_new_data(newdata)
+
+        logger.debug('I have data for %d nodes.', len(storage.data))
 
     except Exception as err:
         logger.error(str(err))

ffstatus/basestorage.py

@@ -69,6 +69,53 @@ class BaseStorage(object):
         """
         pass
 
+    def merge_new_data(self, newdata):
+        """Updates data in the storage by merging the new data."""
+
+        if newdata is None or not isinstance(newdata, dict):
+            raise ValueError("Expected a dict as new data.")
+
+        # start merge on a copy of the current data
+        current = ffstatus.dict_merge(self.data, {})
+        for item_id in current:
+            if not item_id in newdata:
+                continue
+
+            current[item_id]['aliases'] = []
+            current[item_id]['clients'] = []
+            current[item_id]['neighbours'] = []
+
+            if not '__RAW__' in current[item_id]:
+                current[item_id]['__RAW__'] = {}
+
+            if '__RAW__' in newdata[item_id]:
+                for key in newdata[item_id]['__RAW__']:
+                    if key in current[item_id]['__RAW__']:
+                        del current[item_id]['__RAW__'][key]
+
+        # merge the dictionaries
+        updated = ffstatus.dict_merge(current, newdata)
+
+        # sanitize each item's data
+        for itemid in updated:
+            if itemid.startswith('__'):
+                continue
+            item = updated[itemid]
+
+            # remove node's MACs from clients list
+            clients = [x for x in item.get('clients', [])]
+            if 'mac' in item and item['mac'] in clients:
+                clients.remove(item['mac'])
+            for mac in item.get('macs', []):
+                if mac in clients:
+                    clients.remove(mac)
+
+            # set clientcount
+            updated[itemid]['clientcount'] = len(clients)
+
+        # set the new data
+        self.__data = updated
+
     def get_nodes(self, sortby=None, include_raw_data=False):
         """Gets a list of all known nodes."""