""" pe.py puzzles evolving """ import sys # allow importing from the 'code/' dir sys.path.append("../code") import os import platform import pickle import json import traceback import datetime import copy import numpy as np, itertools, copy import matplotlib.pyplot as plt from collections import defaultdict import importlib # module reloading import environments import agents # always forces a reload in case you have edited environments or agents importlib.reload(environments) importlib.reload(agents) #from environments.gridworld import GridWorld import environments.puzzle as pz from environments.puzzle import Puzzle, ConvBelt, getActionSpace, getObservationSpace from agents.q_agent import EvolvableAgent as Agent # DEAP imports import random from deap import creator, base, tools, algorithms import multiprocessing #pool = multiprocessing.Pool() #toolbox.register("map", pool.map) # Weight handling from mda import MultiDimArray def isotime(): return datetime.datetime.now().isoformat() def t2fn(timestamp): timestamp = timestamp.replace('.','_') timestamp = timestamp.replace(':','_') return timestamp class Holder(object): def __init__(self): pass class EvolveWeights(object): """ Class to apply DEAP to evolve a population consisting of a set of weights. """ def __init__(self, environ, popsize=100, maxgenerations=10000, cxpb=0.5, mtpb=0.05, wmin=-20.0, wmax=20.0, mut_center=0.0, mut_sigma=0.1, mut_indpb=0.05, tournsize=5, tournk=2, normalize_fitness=True, tag='environ' ): self.tag = tag self.starttime = isotime() self.logbase = tag + "_" + t2fn(self.starttime) self.environ = environ self.weights_len = environ.get_weights_len() self.popsize = popsize self.maxgenerations = maxgenerations self.cxpb = cxpb self.mtpb = mtpb self.wmin = wmin self.wmax = wmax self.mut_center = mut_center self.mut_sigma = mut_sigma self.mut_indpb = mut_indpb self.tournsize = tournsize self.tournk = tournk self.normalize_fitness = normalize_fitness pass def masv(self, pop): mav = [] maxs = [] for ind in pop: wts = [x for x in ind] mav.append(np.mean(np.abs(wts))) maxs.append(np.max(np.abs(wts))) allmax = np.max(maxs) mymasv = [x/allmax for x in mav] return mymasv def cxTwoPointCopy(self, ind1, ind2): """Execute a two points crossover with copy on the input individuals. The copy is required because the slicing in numpy returns a view of the data, which leads to a self overwriting in the swap operation. It prevents :: >>> import numpy as np >>> a = np.array((1,2,3,4)) >>> b = np.array((5,6,7,8)) >>> a[1:3], b[1:3] = b[1:3], a[1:3] >>> print(a) [1 6 7 4] >>> print(b) [5 6 7 8] """ size = len(ind1) cxpoint1 = random.randint(1, size) cxpoint2 = random.randint(1, size - 1) if cxpoint2 >= cxpoint1: cxpoint2 += 1 else: # Swap the two cx points cxpoint1, cxpoint2 = cxpoint2, cxpoint1 ind1[cxpoint1:cxpoint2], ind2[cxpoint1:cxpoint2] = ind2[cxpoint1:cxpoint2].copy(), ind1[cxpoint1:cxpoint2].copy() return ind1, ind2 def zero(self): return 0.0 def smallrandom(self, eps=None): """ Produce a small random number in [-eps .. eps]. A random variate in [-1 .. 1] is produced then multiplied by eps, so the final range is in [-eps .. eps]. """ if eps in [None]: eps = self.environ.alpha rv = ((2.0 * random.random()) - 1.0) * eps return rv def setup(self): creator.create("FitnessMax", base.Fitness, weights=(1.0,)) creator.create("Individual", np.ndarray, fitness=creator.FitnessMax) self.toolbox = base.Toolbox() self.pool = multiprocessing.Pool() self.toolbox.register("map", self.pool.map) #toolbox.register("attr_bool", random.randint, 0, 1) # non-numpy non-float version # self.toolbox.register("attr_float", random.random) #self.toolbox.register("attr_float", self.zero) self.toolbox.register("attr_float", self.smallrandom) self.toolbox.register("individual", tools.initRepeat, creator.Individual, self.toolbox.attr_float, n=self.weights_len) self.toolbox.register("population", tools.initRepeat, list, self.toolbox.individual) # self.toolbox.register("evaluate", self.environ.evaluate) self.toolbox.register("evaluate", self.environ.evaluate) #toolbox.register("mate", tools.cxTwoPoint) # non-numpy non-float version self.toolbox.register("mate", self.cxTwoPointCopy) #toolbox.register("mutate", tools.mutFlipBit, indpb=0.05) # non-numpy non-float version self.toolbox.register("mutate", tools.mutGaussian, mu=self.mut_center, sigma=self.mut_sigma, indpb=self.mut_indpb) self.toolbox.register("select", tools.selTournament, tournsize=self.tournsize, k=self.tournk) def normalize_fitnesses(self, fitnesses): #print("fitnesses", ["%3.2f" % x[0] for x in fitnesses]) maxfitness = np.max([x[0] for x in fitnesses]) #print("maxfitness", maxfitness) listfit = [x[0] for x in fitnesses] #print("listfit", listfit) normfit = [x/maxfitness for x in listfit] #print("normfit", normfit) fitnesses = [tuple([x]) for x in normfit] #print("normed fitnesses", ["%3.2f" % x[0] for x in fitnesses]) return fitnesses def log_it(self, generation): pool = self.pool toolbox = self.toolbox self.pool = None self.toolbox = None pklfn = f"{self.logbase}__{generation+1}-{self.maxgenerations}.pkl" pickle.dump(self, open(pklfn, "wb")) self.pool = pool self.toolbox = toolbox def loop(self): self.population = self.toolbox.population(n=self.popsize) #print(self.masv(self.population)) NGEN=self.maxgenerations for gen in range(NGEN): print("generation", gen) offspring = algorithms.varAnd(self.population, self.toolbox, cxpb=self.cxpb, mutpb=self.mtpb) # print("offspring", offspring) # constrain genome values to [0,1] for offspring_i,individual in enumerate(offspring): np.clip(np.array(offspring[offspring_i]), self.wmin, self.wmax) # print("clipped offspring", offspring) # Evaluate the individuals with an invalid fitness (not yet evaluated) # print("check fitness.valid") invalid_ind = [ind for ind in offspring if not ind.fitness.valid] # print("invalid_ind", len(invalid_ind)) #print("setting fitness") fitnesses = self.toolbox.map(self.toolbox.evaluate, invalid_ind) if self.normalize_fitness: fitnesses = self.normalize_fitnesses(fitnesses) """ #print("fitnesses", ["%3.2f" % x[0] for x in fitnesses]) maxfitness = np.max([x[0] for x in fitnesses]) #print("maxfitness", maxfitness) listfit = [x[0] for x in fitnesses] #print("listfit", listfit) normfit = [x/maxfitness for x in listfit] #print("normfit", normfit) fitnesses = [tuple([x]) for x in normfit] #print("normed fitnesses", ["%3.2f" % x[0] for x in fitnesses]) """ print("fitnesses", ["%3.2f" % x[0] for x in fitnesses]) self.fitness_dist(fitnesses) # print("update ind fitness") for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit #print("selection") #print("offspring\n", self.masv(offspring)) self.offspring = offspring self.population = self.toolbox.select(offspring, k=len(self.population)) if 0 == gen % 100: self.log_it(gen) #print("population after selection\n", self.masv(self.population)) #print("Report for generation", gen) self.report() def report(self): # post-evolution analysis fitnesses = self.toolbox.map(self.toolbox.evaluate, self.population) if self.normalize_fitness: fitnesses = self.normalize_fitnesses(fitnesses) self.fitnesses = fitnesses self.sortedFitnesses = sorted(fitnesses) self.sortedFitnesses.reverse() self.fitness_dist(fitnesses) self.bestFitness, self.worstFitness = self.sortedFitnesses[0], self.sortedFitnesses[-1] print("best/worst w", self.bestFitness, self.worstFitness) self.bestGenome = tools.selBest(self.population, k=1) # print(self.bestGenome) def ffmt(self, value, fmt="%3.2f"): return fmt % value def fitness_dist(self, fitnesses): listfit = [x[0] for x in fitnesses] pct05, pct25, pct50, pct75, pct95 = np.percentile(listfit, [0.05, 0.25, 0.5, 0.75, 0.95]) print(f"fitness dist: {self.ffmt(np.min(listfit))} {self.ffmt(pct05)} {self.ffmt(pct25)} {self.ffmt(pct50)} {self.ffmt(pct75)} {self.ffmt(pct95)} {self.ffmt(np.max(listfit))}") def driver(self): # Initialize self.setup() # Generation loop self.loop() # Report self.report() self.log_it(self.maxgenerations) print(self.masv(self.population)) pass def puzzles_exp_1(): '''full test of the conveyorbelt world >>> import copy >>> maxrewards = [1] >>> easy_features = [[0,1],[0,1],[3,1],[0,0]] >>> easy_rewards = [-1,-1,-1,1] >>> easy_tt = np.array([[0,0,2,3], [0,0,0,0], [2,0,2,3], [3,3,3,3]]) >>> p1 = Puzzle(tt=easy_tt, features=easy_features, rewards=easy_rewards) >>> p2 = copy.deepcopy(p1) >>> puzzles = (p1,p2) >>> world = ConvBelt(actionSpace = getActionSpace(puzzles), observationSpace = getObservationSpace(puzzles), maxRewards = maxrewards, randomize = False) >>> world.append(p1) >>> world.append(p2) : ''' maxrewards = [1] easy_features = [[0,1],[0,1],[3,1],[0,0]] easy_rewards = [-1,-1,-1,1] easy_tt = np.array([[0,0,2,3], [0,0,0,0], [2,0,2,3], [3,3,3,3]]) p1 = Puzzle(tt=easy_tt, features=easy_features, rewards=easy_rewards) p2 = copy.deepcopy(p1) puzzles = (p1, p2) world = ConvBelt(actionSpace = getActionSpace(puzzles), observationSpace = getObservationSpace(puzzles), maxRewards = maxrewards, agentclass=Agent, randomize = False, alpha=0.005) world.append(p1) world.append(p2) environ = Holder() environ.world = world ew = EvolveWeights(world, popsize=100, maxgenerations=1000, tournsize=75, tournk=3, normalize_fitness=False) ew.driver() if __name__ == "__main__": print("pe.py start...") puzzles_exp_1() print("pe.py done.")