Ants

This page contains resources for the Ants genetic programming sample.

Genetic Programming on Hadoop

You will need: Hortonworks Sandbox 2

You will need to create a place to put these in HDFS


hadoop fs -mkdir /Ants

cd Ants

 

The scripts below, saved with their exact CASE SENSITIVE NAMES will be necessary to run Ants on the Hortonworks Sandbox

 

GenerateBoards.py

#!/usr/bin/python
import sys
import random
import operator
import pickle

def createBoard(boardsize, boardname) :
    board = [[0 for i in xrange(boardsize)] for i in xrange(boardsize)]
    for n in xrange(boardsize) :
        for j in xrange(boardsize) :
                if random.randint(0, 10) == 7 :
                        board[n][j] = 1

    file = open(boardname, 'w')
    pickle.dump(board,file)
    file.close()

if(len(sys.argv) < 3) :
    boardsize = 100
    boardname = "board"
else :
    boardsize = int(sys.argv[1])
    boardname = sys.argv[2]

print "Creating board", boardname, " of Size:", boardsize

board = createBoard(boardsize, boardname)

GenPopMap.py

#!/usr/bin/python
import sys
import random
import operator
import pickle

def createAnt():
        seq = []
        seqLength = random.randint(MIN_GENE,MAX_GENE)
        for i in xrange(seqLength) :
                seq.append(random.randint(0, 3))
        return seq

moves = ["up", "down", "left", "right"]
MIN_GENE = 5
MAX_GENE = 25
if(len(sys.argv) < 2) :
    popsize = 10
else :
    popsize = int(sys.argv[1])
    MIN_GENE = int(sys.argv[2])
    MAX_GENE = int(sys.argv[3])

for i in xrange(popsize) :
        ant = createAnt()
        print '0\t', ant

RunMap.py

#!/usr/bin/python
import sys
import random
import operator
import pickle
import re
import __builtin__

sys.path.append('.')

def runMap(line):
    x=BOARD_SIZE/2
    y=x
    score = 0
    ant = []
    ant = line

    for i in xrange(NUMBER_STEPS) :
        move = ant[operator.mod(i, len(ant))]
        if move == 0 : #"up" :
            x = x+1
        elif move == 1 : #"down" :
            x = x-1
        elif move == 2 : #"right" :
            y = y+1
        elif move == 3 : #"left" :
            y = y-1
        if x>=0 and x<BOARD_SIZE and y>=0 and y<BOARD_SIZE and board[x][y] == 1 :
            score = score + 1
    print "%s\t%s" % (score, ant)

def createAnt():
        seq = []
        seqLength = random.randint(2,10)
        for i in xrange(seqLength) :
                seq.append(moves[random.randint(0, 3)])
        return seq

BOARD_NAME = sys.argv[1]
NUMBER_STEPS = int(sys.argv[2])
BOARD_SIZE = int(sys.argv[3])

file = open(BOARD_NAME, 'r')
board = pickle.load(file)
file.close()

for line in sys.stdin:
    line = line.strip()
    foodCount, rawAnt = line.split('\t', 1)
    ant = []
    ant.extend(map(int, rawAnt.strip('[]').split(',')))
    runMap(ant)

ReproReducer.py

#!/usr/bin/python
import sys
import array
import random
import operator
import pickle
import re
import __builtin__
from operator import itemgetter

#create ant
def createAnt():
        seq = []
        seqLength = random.randint(MIN_GENE,MAX_GENE)
        for i in xrange(seqLength) :
                seq.append(random.randint(0, 3))
        return seq

def breedAnts(ant1, ant2) :
    nant = []
    nant.extend(ant1[:len(ant1)/2])
    nant.extend(ant2[len(ant2)/2:])
    return nant

sys.path.append('.')

POPULATION_SIZE = int(sys.argv[1])
MIN_GENE = int(sys.argv[2])
MAX_GENE = int(sys.argv[3])

count = 0
ant, prevAnt = [],[]
prevAnt = createAnt()

for line in sys.stdin :
    if(count < POPULATION_SIZE/5) :
        #cross breed ants
        line = line.strip()
        foodCount, rawAnt = line.split('\t', 1)
        ant.extend(map(int, rawAnt.strip('[]').split(',')))
        print '%s\t%s' % (0, breedAnts(prevAnt, ant))
        prevAnt = ant
        ant = []
    else :
        print '%s\t%s' % (0, createAnt())
    count += 1

RunGenetic.sh

#!/bin/bash
Gen="3"
Pop="1000"
Board="100"
Steps="100"
MinGene="5"
MaxGene="25"

WorkingDir="/Ants/Runtime"
BoardName="smallBoard"

ArgCount=0
for opt in "$@"
do
    let ArgCount=ArgCount+1
done
if [ $ArgCount == "6" ]   ; then
	echo args passed
	Gen=$1
	Pop=$2
	Board=$3
	Steps=$4
	MinGene=$5
	MaxGene=$6
fi

echo running Ants with the following Parameters:
echo Generations: $Gen
echo Population: $Pop
echo Board Size: $Board
echo Steps per Ant: $Steps
echo Max Gene Size: $MaxGene
echo Min Gene Size: $MinGene

hadoop fs -mkdir $WorkingDir

python GenerateBoards.py $Board $BoardName

hadoop fs -put $BoardName $WorkingDir/$BoardName

COUNTER=0
hadoop jar /usr/lib/hadoop-mapreduce/hadoop-streaming-2*.jar -file 'GenPopMap.py' -input $WorkingDir/$BoardName -output $WorkingDir/$COUNTER -mapper "python ./GenPopMap.py $Pop $MinGene $MaxGene"

while [  $COUNTER -lt $Gen ]; do
	let NEXTCOUNTER=COUNTER+1
	hadoop jar /usr/lib/hadoop-mapreduce/hadoop-streaming-2*.jar -D mapred.output.key.comparator.class=org.apache.hadoop.mapred.lib.KeyFieldBasedComparator -D mapreduce.partition.keycomparator.options=-nr -file $BoardName -file 'RunMap.py' -file 'ReproReducer.py' -input $WorkingDir/$COUNTER/* -output $WorkingDir/$NEXTCOUNTER -mapper "python ./RunMap.py $BoardName $Steps $Board" -reducer "python ./ReproReducer.py $Pop $MinGene $MaxGene" -numReduceTasks 2
	let COUNTER=COUNTER+1
done

hadoop jar /usr/lib/hadoop-mapreduce/hadoop-streaming-2*.jar -D mapred.output.key.comparator.class=org.apache.hadoop.mapred.lib.KeyFieldBasedComparator -D mapreduce.partition.keycomparator.options=-nr -file $BoardName -file 'RunMap.py' -input $WorkingDir/$COUNTER/* -output $WorkingDir/FINAL -mapper "python ./RunMap.py $BoardName $Steps $Board" -numReduceTasks 2
Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s

%d bloggers like this: