Generators

Overview

Sample Generator Code

def average(items): total=0 count = 0 for item in items: total += item count += 1 yield item if count == 0: return 0 return total/count def max(items): val = None for item in items: if val == None: val=item elif item>val:val=item yield item return val
Then, using the calculator function, we can create two new functions. The generator will not be interated through until one of the new functions is called.
testmax = calculator(max, list1) testmin = calculator(min, list1) mx1 = testmax() mn1 = testmin()

Chaining Together Data Sources

Generators makes it easy to comvine multiple data sources into a what appears as a single data source. The following function takes two file paths, and merges them into a single data source using the generator mechanism.

Full Script

import math import itertools def init_calculator(): generators = [] def func(method, generator): nonlocal generators matched = [x for x in generators if x['generator'] == generator] if len(matched) == 0: nmethod, ngen = gen(generators, method, generator) generators.append({ "generator":generator, "new":ngen, "method":method }) return nmethod else: newgen = matched[-1]['new'] nmethod, ngen = gen(generators, method, newgen) generators.append({ "generator":generator, "new": ngen, "method":method }) return nmethod pass return func calculator = init_calculator(); def average(items): total=0 count = 0 for item in items: total += item count += 1 yield item if count == 0: return 0 return total/count def length(items): count = 0 for item in items: count+=1 yield item return count def max(items): val = None for item in items: if val == None: val=item elif item>val:val=item yield item return val def min(items): val = None for item in items: if val == None: val=item elif item<val:val=item yield item return val def mult_average(size): def average(items): nonlocal size total = 0 count = 0 for item in items: total+=item*size count+=1 yield item pass if count==0:return 0 return total/count return average def map(method, select): def func(items): nonlocal method, select it1, it2 = itertools.tee(items, 2) #miter = method(select(it2)) miter = method((select(x) for x in it2)) def niterator(items): for item in items: try: item2 = next(miter) pass except: pass yield item pass pass newlist = niterator(it1) for item in newlist: yield item pass answer = None try: next(miter) pass except StopIteration as e: answer = e.value pass return answer return func def filter(method, condition): def func(items): nonlocal method, condition it1, it2 = itertools.tee(items, 2) def filtered(items): nonlocal condition for item in items: if condition(item):yield item else:pass pass pass miter = method(filtered(it2)) iteratenext = True iterateitem = None def niterator(items): nonlocal iteratenext, iterateitem for item in items: if iteratenext == True: try: iterateitem = next(miter) pass except: iterateitem = None pass pass if iterateitem == item: iteratenext = True else: iteratenext = False yield item pass pass newlist = niterator(it1) for item in newlist: yield item pass answer = None try: next(miter) except StopIteration as e: answer = e.value return answer return func def gen(*args): method = None list1 = None parms = [] gn = None generators = None for arg in args: if generators is None: generators = arg elif method is None: method = arg elif list1 is None:list1 = arg else: parms.append(arg) def get_generator(populate): nonlocal method, list1, parms gen = method(list1, *parms) ended = False while not ended: try: item = next(gen) yield item pass except StopIteration as e: ended = True value = e.value populate(value) pass pass pass if(isinstance(list1, list)): list1 = (x for x in list1) answer = None def result(): nonlocal answer nonlocal gn nonlocal generators ngen = generators[-1]['new'] try: for value in ngen: pass pass except Exception as e: pass return answer def populate(value): nonlocal answer answer = value pass gn = get_generator(populate) return result, gn def group_old(keys, aggregator): def result(list): map = {} for item in list: current = map keys2 = [*keys] while len(keys2)>0: key = keys2.pop(0) if len(keys2) == 0: if callable(key): tempkey = key(item) if tempkey in current: current[tempkey] = aggregator(current[tempkey], item) else: current[tempkey] = aggregator(None, item) pass else: if item[key] in current: current[item[key]] = aggregator(current[item[key]], item) else: current[item[key]] = aggregator(None, item) pass else: if callable(key): tempkey = key(item) if tempkey not in current: current[tempkey] = {} current = current[tempkey] pass else: if item[key] not in current: current[item[key]] = {} current = current[item[key]] pass pass yield item pass return map pass return result def group(keys, aggregator): def result(list): map = {} for item in list: current = map if callable(keys): keys2 = keys(item) else: keys2 = [item[key] for key in keys] while len(keys2)>0: key = keys2.pop(0) if len(keys2) == 0: if key in current: current[key] = aggregator(current[key], item) else: current[key] = aggregator(None, item) else: if key not in current: current[key] = {} current = current[key] pass yield item pass return map pass return result def groupsum(aggregated, value): if aggregated == None: aggregated = 0 aggregated += value['age'] return aggregated def grouplist(aggregated, value): if aggregated == None: aggregated = [] aggregated.append(value) return aggregated def grouptolist(names, groups, list=None): results = [] pass group.list = grouplist group.sum = groupsum ''' creating two generators from a single generator import itertools def count_stream(): yield 1 yield 2 yield 3 # Create the generator instance gen = count_stream() # Split it into two independent streams copy_a, copy_b = itertools.tee(gen, 2) print(next(copy_a)) # Output: 1 print(next(copy_b)) # Output: 1 print(next(copy_a)) # Output: 2 ''' if __name__ == "__main__": list1 = (x for x in [1,2,3,4,5]) def testmap2(x): return 10*x pass testmax = calculator(max, list1) testmin = calculator(min, list1) testfilter = calculator(filter(average, lambda x:x>2), list1) testfilter2 = calculator(filter(average, lambda x:x>0), list1) testmap = calculator(map(average, testmap2), list1) testav = calculator(average, list1) testav2 = calculator(mult_average(2), list1) testcount = calculator(length, list1) mx1 = testmax() mn1 = testmin() fil1 = testfilter() fil2 = testfilter2() map1 = testmap() av1 = testav2() av = testav() ct = testcount() items = [{"name1":'dan', "name2":'leucky', "age":12},{"name1":'dan', "name2":'leucky', "age":12},{"name1":'dan', "name2":'leucky', "age":12}, {"name1":'dan2', "name2":'leucky', "age":12},{"name1":'dan3', "name2":'leucky', "age":12},{"name1":'dan4', "name2":'leucky', "age":12}, {"name1":'dan', "name2":'leucky', "age":12},{"name1":'dan', "name2":'leucky', "age":12},{"name1":'dan', "name2":'leucky', "age":12},] def aggregate(aggregated, value): if aggregated == None: aggregated = 0 aggregated += value['age'] return aggregated items = (x for x in items) group1 = calculator(group(['name1', 'name2'], group.list), items) group2 = calculator(group(['name1', 'name2'], aggregate), items) group3 = calculator(group(lambda x: [x['name1'], x['name2']], group.list), items) test1 = group1() test2 = group2() test3 = group3() pass