import copy ''' Aggregators are used with the group function ''' def list_aggregator(agg, item): if agg ==None:return [item] agg.append(item) return agg def count_aggregator(agg, item): if agg == None: return 1 return agg+1 ''' group will take a list of dictionaries and build a dictionary out of it, where the keys are the values specified by the keys input. ''' def group(items, keys, aggregator=list_aggregator): map = {} for item in items: current = map if callable(keys): keys2 = keys(item) if not isinstance(keys2, list): keys2 = [keys2] else: keys2 = [item[key] for key in keys] while len(keys2)>0: key = keys2.pop(0) if len(keys2) == 0: if key in current: current[key] = aggregator(current[key], item) else: current[key] = aggregator(None, item) else: if key not in current: current[key] = {} current = current[key] pass pass return map ''' to_list is used with a dictionary created by the group function to flatten the results into a list of dictionaries ''' def to_list(cols, group): results = [] if len(cols) == 1: name = cols[0] if isinstance(group, dict): for key in group: val = group[key] if isinstance(val, list): for item in val: item = copy.deepcopy(item) item[name] = key results.append(item) pass pass else: record = {} record[name] = key results.append(record) pass pass pass else: item = {} item[name] = group results.append(item) pass return results name = cols[0] ncols = cols[1:] for key in group: group2 = group[key] results2 = to_list(ncols, group2) for item in results2: item[name] = key results.append(item) pass pass return results group.to_list = to_list ''' join joins two lists of dictionaries into a single list. similar to a sql join. ''' def join(left_items,right_items, left, right, left_map=None, right_map=None): results = [] def map(item):return item if left_map == None: left_map = map if right_map == None: right_map = map grp = group(left_items, left) for item in right_items: keys = right(item) if not isinstance(keys, list): keys = [keys] current = grp while len(keys)>0: key = keys.pop(0) if current != None and key in current: current = current[key] else: current = None pass if current != None: right_item = right_map(item) for item in current: left_item = left_map(item) record = {} results.append(record) for key in left_item: record[key] = left_item[key] for key in right_item: record[key] = right_item[key] pass pass pass return results def join_on_date(left_items, right_items): if len(left_items) == 0 or len(right_items) == 0: return [] lfirst = left_items[0] rfirst = right_items[0] leftkey = 'date' for key in lfirst: if key.lower() == 'date': leftkey = key pass rightkey = 'date' for key in rfirst: if key.lower() == 'date': rightkey = key pass def left(item): nonlocal leftkey return item[leftkey] def right(item): nonlocal rightkey return item[rightkey] return join(left_items=left_items, right_items=right_items, left=left, right=right) ''' sample script ''' if __name__ == "__main__": items = [{"name1":'scott', "name2":'jones', "age":12},{"name1":'scott', "name2":'jones', "age":12},{"name1":'scott', "name2":'jones', "age":12}, {"name1":'scott2', "name2":'jones', "age":22},{"name1":'scott3', "name2":'jones', "age":24},{"name1":'scott4', "name2":'jones', "age":30}, {"name1":'scott', "name2":'jones', "age":30},{"name1":'scott', "name2":'smith', "age":30},{"name1":'scott', "name2":'smith', "age":40}] grp = group(items, lambda x:x['name1'], count_aggregator) grp2 = group(items, lambda x:[x['name1'],x['name2']]) nlist = group.to_list(['name1','count'],grp) join2 = join(items,items, lambda x:x['name1'], lambda x:x['name1']) pass