Generator Aggregators
Overview
Generator aggregators are functions that compute some aggregate value over a collection of items
(typically numbers) that are returned by a generator (or some iterator). The aggregators are
designed to be defined separately from the code that iterates over the generator.
Usage
The generators library defines a function called calculator. Once a generator or iterator has be defined, you
can register a function that computes some aggregation of the generated items by calling the calculator function
and passing in the generator and the function. The returned function is a function, that when called, computes the
calculated aggregation and returns the result.
Note that calling the returned aggregator function will iterate through (and exhaust) the generator if it hasnt already
been iterated through. Also, registering the aggregator by calling calculator should pass a generator that has not been
iterated through.
from lib.generators import calculator, max, min
#define a generator
list1 = (x for x in [1,2,3,4,5])
testmax = calculator(max, list1)
testmin = calculator(min, list1)
mx1 = testmax()
mn1 = testmin()
List of Aggregators
The following are aggregators currently defined in the generators script.
- min
- max
- average
- sum
- count
Definig Aggregators
The following
def average(items):
total=0
count = 0
for item in items:
total += item
count += 1
yield item
if count == 0: return 0
return total/count
def length(items):
count = 0
for item in items:
count+=1
yield item
return count
def max(items):
val = None
for item in items:
if val == None: val=item
elif item>val:val=item
yield item
return val
Mapping and Filtering
The generators script provides functions for mapping and filtering the generator
prior to calculating an aggregator.
from lib.generators import calculator, min, max, filter, map, average
list1 = (x for x in [1,2,3,4,5])
def ten_times(x):
return 10*x
testfilter = calculator(filter(average, lambda x:x>2), list1)
testmap = calculator(map(average, ten_times), list1)
fil1 = testfilter()
map1 = testmap()
Sample Scripts
Full Script
import math
import itertools
def init_calculator():
generators = []
def func(method, generator):
nonlocal generators
matched = [x for x in generators if x['generator'] == generator]
if len(matched) == 0:
nmethod, ngen = gen(generators, method, generator)
generators.append({
"generator":generator,
"new":ngen,
"method":method
})
return nmethod
else:
newgen = matched[-1]['new']
nmethod, ngen = gen(generators, method, newgen)
generators.append({
"generator":generator,
"new": ngen,
"method":method
})
return nmethod
pass
return func
calculator = init_calculator();
def average(items):
total=0
count = 0
for item in items:
total += item
count += 1
yield item
if count == 0: return 0
return total/count
def length(items):
count = 0
for item in items:
count+=1
yield item
return count
def max(items):
val = None
for item in items:
if val == None: val=item
elif item>val:val=item
yield item
return val
def min(items):
val = None
for item in items:
if val == None: val=item
elif item0:
key = keys2.pop(0)
if len(keys2) == 0:
if key in current:
current[key] = aggregator(current[key], item)
else:
current[key] = aggregator(None, item)
else:
if key not in current:
current[key] = {}
current = current[key]
pass
yield item
pass
return map
pass
return result
def groupsum(aggregated, value):
if aggregated == None: aggregated = 0
aggregated += value['age']
return aggregated
def grouplist(aggregated, value):
if aggregated == None: aggregated = []
aggregated.append(value)
return aggregated
def grouptolist(names, groups, list=None):
results = []
pass
group.list = grouplist
group.sum = groupsum
'''
creating two generators from a single generator
import itertools
def count_stream():
yield 1
yield 2
yield 3
# Create the generator instance
gen = count_stream()
# Split it into two independent streams
copy_a, copy_b = itertools.tee(gen, 2)
print(next(copy_a)) # Output: 1
print(next(copy_b)) # Output: 1
print(next(copy_a)) # Output: 2
'''
if __name__ == "__main__":
list1 = (x for x in [1,2,3,4,5])
def testmap2(x):
return 10*x
pass
testmax = calculator(max, list1)
testmin = calculator(min, list1)
testfilter = calculator(filter(average, lambda x:x>2), list1)
testfilter2 = calculator(filter(average, lambda x:x>0), list1)
testmap = calculator(map(average, testmap2), list1)
testav = calculator(average, list1)
testav2 = calculator(mult_average(2), list1)
testcount = calculator(length, list1)
mx1 = testmax()
mn1 = testmin()
fil1 = testfilter()
fil2 = testfilter2()
map1 = testmap()
av1 = testav2()
av = testav()
ct = testcount()
items = [{"name1":'dan', "name2":'leucky', "age":12},{"name1":'dan', "name2":'leucky', "age":12},{"name1":'dan', "name2":'leucky', "age":12},
{"name1":'dan2', "name2":'leucky', "age":12},{"name1":'dan3', "name2":'leucky', "age":12},{"name1":'dan4', "name2":'leucky', "age":12},
{"name1":'dan', "name2":'leucky', "age":12},{"name1":'dan', "name2":'leucky', "age":12},{"name1":'dan', "name2":'leucky', "age":12},]
def aggregate(aggregated, value):
if aggregated == None: aggregated = 0
aggregated += value['age']
return aggregated
items = (x for x in items)
group1 = calculator(group(['name1', 'name2'], group.list), items)
group2 = calculator(group(['name1', 'name2'], aggregate), items)
group3 = calculator(group(lambda x: [x['name1'], x['name2']], group.list), items)
test1 = group1()
test2 = group2()
test3 = group3()
pass