Commit 224fdec7 authored by Balazs's avatar Balazs

- changes to the parameter handling within the Service Orchestrator

- installation of software from dedicated Cinder Volume
- restructuring / refactoring
parent 4cf91e97
......@@ -4,12 +4,12 @@ parameters:
type: string
description: Which image should be the master's os setup with?
default: "$master_image$"
default: $master_image$
type: string
description: Which image should be the slave's os setup with?
default: "$slave_image$"
default: $slave_image$
type: string
......@@ -31,30 +31,25 @@ parameters:
description: slave's name (the index will be appended at the end)
default: $slavenode$
type: string
description: name of public key of openstack user for master node
default: "$ssh_key$"
type: string
description: CIDR of subnet that cluster is going to use
default: "$subnet_cidr$"
default: $subnet_cidr$
type: string
description: subnet's gateway's IP
default: "$subnet_gw_ip$"
default: $subnet_gw_ip$
type: string
description: allocation pool's starting IP address
default: "$subnet_allocation_pool_start$"
default: $subnet_allocation_pool_start$
type: string
description: allocation pool's last IP address
default: "$subnet_allocation_pool_end$"
default: $subnet_allocation_pool_end$
type: comma_delimited_list
......@@ -62,7 +57,7 @@ parameters:
default: $subnet_dns_servers$
$users_ssh_public_key$ sshpublickey:
name: ssh_cluster_pub_key
public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDkbCR95uSaiAmspMoSPTKGSk3+5Vd6paKmdZQmdbHJd0a9HKLN87mYgc8qOURvxkjCEKMNYws2d2IQO0e7WELSfmMuQn/tUrLAiFTBfGsNxbarRO+UMvDvQwqQgRZ94wrjF/ELz4rQTe87MTuxpR79rbiGxHrHmJFxGAgS2evczwfmCKyyS15brWsNXaxDLyiBN3mWWz/O+4b3IEN/FKHYQcgy+SytGfHQBFkuOI5ETrRgeBy4ohGZzrn44TYjK4L2tLJJPT0rUffl81huVWoqcgwmkHETszDZm7OYkeLvX9g02JzeiKrqAFEvaWxjyreDkLEqKNF9oxi3SfdX72Yr mesz@CLT-MOB-T-6253-2
......@@ -83,7 +78,7 @@ resources:
type: OS::Neutron::Net
name: "hadoopNet"
name: hadoopNet
type: OS::Neutron::Subnet
......@@ -110,7 +105,7 @@ resources:
name: { get_param: master_name }
image: { get_param: master_image }
flavor: { get_param: master_flavor }
key_name: { get_resource: sshpublickey }
key_name: $master_ssh_key_entry$
- port: { get_resource: hadoop_port }
......@@ -122,16 +117,12 @@ $paramsslave$
type: OS::Neutron::FloatingIP
floating_network: "external-net"
type: OS::Neutron::Router
network: "external-net"
network: external-net
type: OS::Neutron::RouterInterface
......@@ -139,13 +130,20 @@ $slaves$
router_id: { get_resource: hadoop_router }
subnet_id: { get_resource: hadoop_subnet }
type: OS::Neutron::FloatingIPAssociation
# software_volume:
# type: OS::Cinder::Volume
# properties:
# name: software_volume
# size: 6
# image: $image_id$
type: OS::Cinder::VolumeAttachment
floatingip_id: { get_resource: hadoop_ip }
port_id: { get_resource: hadoop_port }
volume_id: $image_id$
instance_uuid: { get_resource: hadoop_master }
mountpoint: /dev/vdb
description: The IP address of the deployed master node
value: { get_attr: [ hadoop_ip, floating_ip_address ] }
\ No newline at end of file
# configuration file for service orchestrator (currently just hadoop as a
# service)
# Everything has to be within the section "cluster" because of the SO's
# implementation. All data can also be gived at instantiation. If done so, that
# data will supersede the data given in this file. On the other hand, if the
# required data is neither passed during instantiation nor in this file, a
# default value will be set by the SO which might not be according to the
# user's needs.
# NOTE: there is an entry (icclab.haas.rootfolder) which cannot be included in
# this file as it is also needed in order to find this very file. It has to be
# delivered to the SO as an attribute additionally to every deployment call!
# name of the whole cluster - at the moment, this value isn't used for anything distributedprocessing
# image files to take as base image for master and slaves; it should be a
# debian distribution with user ec2-user and passwordless sudo execution
icclab.haas.master.image: Ubuntu-Trusty-1404-7-10-2015
icclab.haas.slave.image: Ubuntu-Trusty-1404-7-10-2015
# Name of SSH public key registration in keystone; if a name is given without a
# public key, this key name will be included into the master - in this case,
# the key has to be registered on keystone already. If key name is given with a
# public key, a key name with the given public key will be inserted into
# keystone. (and the master) If neither a key name nor a public key are given,
# the public key from the file will be inserted into the
# master.
# flavor for master / slave
icclab.haas.master.flavor: m1.small
icclab.haas.slave.flavor: m1.small
# number of masters and slaves to be configured (possibly certain frameworks
# don't allow multiple masters; plus decision whether a slave node should be
# started on the master node as well
icclab.haas.master.number: 1
icclab.haas.slave.number: 2
icclab.haas.master.slaveonmaster: True
# should a floating IP be created for the cluster?
icclab.haas.master.withfloatingip: True
icclab.haas.master.attachfloatingipwithid: 034f34b7-5ff5-4109-9e8b-a9d919e7ff39
# name for the master(s) and slave(s) masternode slavenode
# network configuration for the subnet ["",""]
# id of the nova image containing the directory structure for the software to
# be installed (can be retrieved with `nova image-list` or from Horizon)
icclab.haas.master.imageid: 8d92eb3d-2347-4177-939d-4f0f8537d31a
# method how to transfer the files to the slaves of the cluster: either
# transfer them in a packed form and unpack them on the slaves or transfer them
# in an unpacked way; this option is the name of a defined bash function.
# Currently, the two optinos transferUnpackedFiles and transferFirstUnpackLater
# are possible.
icclab.haas.master.transfermethod: transferUnpackedFiles
# some information about the Linux user has to be provided - this user has to
# be present on all of the deployed machines and it needs to have passwordless
# sudo access
icclab.haas.cluster.username: ec2-user
icclab.haas.cluster.usergroup: ec2-user
icclab.haas.cluster.homedir: /home/ec2-user
# debug settings - you can decide on your own whether to deploy the created
# template on OpenStack and for debugging purposes, you can also provide a path
# for saving the template locally on the machine where the SO is run
icclab.haas.debug.donotdeploy: False
# NOTE about this bash file: this file will be used to setup the distributed
# computing cluster on OpenStack. This includes copying the actual application
# frameworks from an external cinder volume to each master/slave and writing
# the configuration files to each of them. The parameters within dollar signs
# (e.g. $homedir$) will be filled by the service orchestrator ( with
# either given values from the user, default settings either from file
# defaultSettings.cfg / assumptions within the serice orchestrator or with
# pre-defined configuration files within the /data directory of the SO bundle.
# disable IPv6 as Hadoop won't run on a system with it activated
echo "disabling IPv6" >> /home/ec2-user/deployment.log
echo "disabling IPv6" >> $homedir$/deployment.log
echo -e "\nnet.ipv6.conf.all.disable_ipv6 = 1\nnet.ipv6.conf.default.disable_ipv6 = 1\nnet.ipv6.conf.lo.disable_ipv6 = 1" >> /etc/sysctl.conf
sysctl -p
# add group and user for hadoop and setup SSH for it
#echo "adding user ec2-user:hadoop" >> /home/ec2-user/deployment.log
#addgroup hadoop
#useradd --gid hadoop --home /home/ec2-user --create-home --shell /bin/bash ec2-user
#echo "ec2-user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
#passwd --delete ec2-user
#su ec2-user -c "mkdir /home/ec2-user/.ssh"
su ec2-user -c 'echo -e "$master.id_rsa$" > /home/ec2-user/.ssh/id_rsa'
su ec2-user -c 'echo -e "$$" > /home/ec2-user/.ssh/'
#su ec2-user -c "cat /home/ec2-user/.ssh/ >> /home/ec2-user/.ssh/authorized_keys"
su ec2-user -c 'echo -e "Host *\n StrictHostKeyChecking no\n UserKnownHostsFile=/dev/null" > /home/ec2-user/.ssh/config'
chmod 0600 /home/ec2-user/.ssh/*
# download Hadoop & Java on the master and install them (including setting the environment variables)
# setup master's SSH configuration
su $username$ -c 'echo -e "$master.id_rsa$" > $homedir$/.ssh/id_rsa'
su $username$ -c 'echo -e "$$" > $homedir$/.ssh/'
su $username$ -c 'echo -e "Host *\n StrictHostKeyChecking no\n UserKnownHostsFile=/dev/null" > $homedir$/.ssh/config'
chmod 0600 $homedir$/.ssh/*
# copying Hadoop & Java on the master and install them (including setting the
# environment variables)
cd /root
echo "downloading hadoop..." >> /home/ec2-user/deployment.log
wget $hadoop_uri$
echo "downloading jdk..." >> /home/ec2-user/deployment.log
wget --no-cookies --no-check-certificate --header "Cookie:; oraclelicense=accept-securebackup-cookie" "$jdk_uri$"
echo "setting up files for deployment on slaves..." >> /home/ec2-user/deployment.log
# move packages to ec2-user's home directory (access rights)
mkdir /home/ec2-user/archives
mv /root/{hadoop-2.7.1,jdk-8u60-linux-x64}.tar.gz /home/ec2-user/archives
# disk has to be mounted by ID as the virtual disk device in the /dev folder
# can be different with every restart
mount -o ro /dev/disk/by-id/$disk_id$ /mnt/
mkdir $homedir$/archives
echo "setting up files for deployment on slaves..." >> $homedir$/deployment.log
cat - >> /root/bashrc.suffix <<'EOF'
export JAVA_HOME=/usr/lib/java/jdk
export PATH=$PATH:$JAVA_HOME/bin
......@@ -38,97 +43,128 @@ EOF
# configure Hadoop
# first of all, let's create the config files for the slaves
mkdir /home/ec2-user/hadoopconf
mv /root/bashrc.suffix /home/ec2-user/hadoopconf
mkdir $homedir$/hadoopconf
mv /root/bashrc.suffix $homedir$/hadoopconf
# creating /etc/hosts file's replacement - don't forget: slaves need to have the same name as configured with Heat Template!!!
# creating /etc/hosts file's replacement - don't forget: slaves need to have
# the same name as configured with Heat Template!!!
echo -e "\tlocalhost\n`/sbin/ifconfig eth0 | grep 'inet addr' | cut -d: -f2 | awk '{print $1}'` $masternode$" > /root/hosts.replacement
cat - >> /root/hosts.replacement <<'EOF'
mv -f /root/hosts.replacement /home/ec2-user/hadoopconf
mv -f /root/hosts.replacement $homedir$/hadoopconf
# create yarn-site.xml:
cat - > /home/ec2-user/hadoopconf/yarn-site.xml << 'EOF'
cat - > $homedir$/hadoopconf/yarn-site.xml << 'EOF'
# create core-site.xml:
cat - > /home/ec2-user/hadoopconf/core-site.xml << 'EOF'
cat - > $homedir$/hadoopconf/core-site.xml << 'EOF'
# create mapred-site.xml:
cat - >> /home/ec2-user/hadoopconf/mapred-site.xml << 'EOF'
cat - >> $homedir$/hadoopconf/mapred-site.xml << 'EOF'
# create hdfs-site.xml: (here, replication factor has to be entered!!!)
cat - >> /home/ec2-user/hadoopconf/hdfs-site.xml << 'EOF'
cat - >> $homedir$/hadoopconf/hdfs-site.xml << 'EOF'
# create
cat - >> /home/ec2-user/hadoopconf/ << 'EOF'
cat - >> $homedir$/hadoopconf/ << 'EOF'
# setup parallel ssh
apt-get install -y pssh
cat - > /home/ec2-user/hosts.lst << 'EOF'
# copy pssh/pscp to /usr/bin/pssh on master
# originally from Git repo
cp -r /mnt/pssh/pssh /usr/bin/
cat - > $homedir$/hosts.lst << 'EOF'
echo "copying hadoop and jdk to slaves" >> /home/ec2-user/deployment.log
su ec2-user -c "parallel-scp -h /home/ec2-user/hosts.lst /home/ec2-user/archives/{hadoop-2.7.1.tar.gz,jdk-8u60-linux-x64.tar.gz} /home/ec2-user"
echo "unpacking hadoop" >> /home/ec2-user/deployment.log
su ec2-user -c "parallel-ssh -t 200 -h /home/ec2-user/hosts.lst \"tar -xzf /home/ec2-user/hadoop-2.7.1.tar.gz\""
echo "unpacking jdk" >> /home/ec2-user/deployment.log
su ec2-user -c "parallel-ssh -t 200 -h /home/ec2-user/hosts.lst \"tar -xzf /home/ec2-user/jdk-8u60-linux-x64.tar.gz\""
echo "setting up both" >> /home/ec2-user/deployment.log
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo mkdir /usr/lib/hadoop\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo mv /home/ec2-user/hadoop-2.7.1 /usr/lib/hadoop\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo ln -s /usr/lib/hadoop/hadoop-2.7.1 /usr/lib/hadoop/hadoop\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo mv /usr/lib/hadoop/hadoop-2.7.1/etc/hadoop/ /etc/\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo mkdir -p /usr/lib/java\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo mv /home/ec2-user/jdk1.8.0_60/ /usr/lib/java/\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo ln -s /usr/lib/java/jdk1.8.0_60/ /usr/lib/java/jdk\""
su ec2-user -c "parallel-scp -h /home/ec2-user/hosts.lst /home/ec2-user/hadoopconf/bashrc.suffix /home/ec2-user"
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo sh -c \\\"cat /home/ec2-user/bashrc.suffix >> /etc/bash.bashrc\\\"\""
function transferFirstUnpackLater {
# copying hadoop & jdk to slaves in a compact form and unpacking them on
# the slaves
echo "copying hadoop and jdk to slaves" >> $homedir$/deployment.log
su $username$ -c "/usr/bin/pssh/bin/pscp -h $homedir$/hosts.lst /mnt/{hadoop/hadoop-2.7.1.tar.gz,java/jdk-8u60-linux-x64.tar.gz} $homedir$"
echo "unpacking hadoop" >> $homedir$/deployment.log
su $username$ -c "/usr/bin/pssh/bin/pssh -t 2000 -h $homedir$/hosts.lst \"tar -xzf $homedir$/hadoop-2.7.1.tar.gz\""
echo "unpacking jdk" >> $homedir$/deployment.log
su $username$ -c "/usr/bin/pssh/bin/pssh -t 2000 -h $homedir$/hosts.lst \"tar -xzf $homedir$/jdk-8u60-linux-x64.tar.gz\""
echo "setting up both" >> $homedir$/deployment.log
# done with copying/unpacking hadoop/jdk
function transferUnpackedFiles {
# in this scenario, hadoop/jdk will be transferred to the slaves in an
# unpacked form
echo "copying hadoop and jdk to slaves" >> $homedir$/deployment.log
su $username$ -c "/usr/bin/pssh/bin/pscp -r -h $homedir$/hosts.lst /mnt/{hadoop/hadoop-2.7.1,java/jdk1.8.0_60} $homedir$"
# done with transfer
# here, the script has to decide which function to call:
# transferFirstUnpackLater or transferUnpackedFiles
echo "transferring hadoop & jdk to the masters/slaves and unpacking them" >> $homedir$/deployment.log
# the mounted volume isn't needed anymore - for security's sake, it will be
# unmounted
umount /mnt
echo "setting up hadoop & jdk" >> $homedir$/deployment.log
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo mkdir /usr/lib/hadoop\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo mv $homedir$/hadoop-2.7.1 /usr/lib/hadoop\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo ln -s /usr/lib/hadoop/hadoop-2.7.1 /usr/lib/hadoop/hadoop\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo mv /usr/lib/hadoop/hadoop-2.7.1/etc/hadoop/ /etc/\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo mkdir -p /usr/lib/java\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo mv $homedir$/jdk1.8.0_60/ /usr/lib/java/\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo ln -s /usr/lib/java/jdk1.8.0_60/ /usr/lib/java/jdk\""
su $username$ -c "/usr/bin/pssh/bin/pscp -h $homedir$/hosts.lst $homedir$/hadoopconf/bashrc.suffix $homedir$"
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo sh -c \\\"cat $homedir$/bashrc.suffix >> /etc/bash.bashrc\\\"\""
# now, let's copy the files to the slaves
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo mkdir -p /app/hadoop/tmp\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo chown ec2-user:ec2-user /app/hadoop/tmp\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo chmod 750 /app/hadoop/tmp\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo chown -R ec2-user:ec2-user /etc/hadoop\""
# the file has to be copied into the user directory as ec2-user doesn't have permissions to write into /etc/hadoop
echo "copying config files from master to slave..." >> /home/ec2-user/deployment.log
su ec2-user -c "parallel-scp -h /home/ec2-user/hosts.lst /home/ec2-user/hadoopconf/core-site.xml /home/ec2-user"
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo mkdir -p /app/hadoop/tmp\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo chown $username$:$usergroup$ /app/hadoop/tmp\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo chmod 750 /app/hadoop/tmp\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo chown -R $username$:$usergroup$ /etc/hadoop\""
# the file has to be copied into the user directory as $username$ doesn't have
# permissions to write into /etc/hadoop
echo "copying config files from master to slave..." >> $homedir$/deployment.log
su $username$ -c "/usr/bin/pssh/bin/pscp -h $homedir$/hosts.lst $homedir$/hadoopconf/core-site.xml $homedir$"
# move file to its final location (/etc/hadoop)
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo mv -f /home/ec2-user/core-site.xml /etc/hadoop\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo mv -f $homedir$/core-site.xml /etc/hadoop\""
su ec2-user -c "parallel-scp -h /home/ec2-user/hosts.lst /home/ec2-user/hadoopconf/{{mapred,hdfs,yarn}-site.xml,} /etc/hadoop"
su $username$ -c "/usr/bin/pssh/bin/pscp -h $homedir$/hosts.lst $homedir$/hadoopconf/{{mapred,hdfs,yarn}-site.xml,} /etc/hadoop"
su ec2-user -c "parallel-scp -h /home/ec2-user/hosts.lst /home/ec2-user/hadoopconf/hosts.replacement /home/ec2-user"
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"sudo mv -f /home/ec2-user/hosts.replacement /etc/hosts\""
su $username$ -c "/usr/bin/pssh/bin/pscp -h $homedir$/hosts.lst $homedir$/hadoopconf/hosts.replacement $homedir$"
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"sudo mv -f $homedir$/hosts.replacement /etc/hosts\""
su ec2-user -c "parallel-ssh -h /home/ec2-user/hosts.lst \"ln -s /etc/hadoop /usr/lib/hadoop/hadoop-2.7.1/etc/hadoop\""
su $username$ -c "/usr/bin/pssh/bin/pssh -h $homedir$/hosts.lst \"ln -s /etc/hadoop /usr/lib/hadoop/hadoop-2.7.1/etc/hadoop\""
# set master and slave nodes
echo $masternode$ > /etc/hadoop/masters
cat - > /etc/hadoop/slaves << 'EOF'
source /etc/hadoop/
su ec2-user -c "/usr/lib/hadoop/hadoop/bin/hdfs namenode -format"
su ec2-user -c "/usr/lib/hadoop/hadoop/sbin/"
su ec2-user -c "/usr/lib/hadoop/hadoop/sbin/"
echo "hadoop cluster ready" >> /home/ec2-user/deployment.log
su $username$ -c "/usr/lib/hadoop/hadoop/bin/hdfs namenode -format"
su $username$ -c "/usr/lib/hadoop/hadoop/sbin/"
su $username$ -c "/usr/lib/hadoop/hadoop/sbin/"
echo "hadoop cluster ready" >> $homedir$/deployment.log
echo "deployment took me $duration seconds"
# in the following line, the whole regular output will be redirected to the
# file debug.log in the user's home directory and the error output to the file
# error.log within the same directory
} 2> $homedir$/error.log | tee $homedir$/debug.log
......@@ -10,7 +10,25 @@
"icclab.haas.slave.image": "",
"icclab.haas.master.sshkeyname": "",
"icclab.haas.master.publickey": "",
"icclab.haas.cluster.procframework": ""
"icclab.haas.cluster.procframework": "",
"icclab.haas.master.flavor": "",
"icclab.haas.slave.flavor": "",
"icclab.haas.master.withfloatingip": "",
"icclab.haas.master.attachfloatingipwithid": "",
"": "",
"": "",
"": "",
"": "",
"": "",
"": "",
"": "",
"icclab.haas.master.imageid": "",
"icclab.haas.master.transfermethod": "",
"icclab.haas.cluster.username": "",
"icclab.haas.cluster.usergroup": "",
"icclab.haas.cluster.homedir": "",
"icclab.haas.debug.donotdeploy": "",
"icclab.haas.debug.savetemplatetolocalpath": ""
"service_endpoint": "",
"depends_on": []
......@@ -17,21 +17,6 @@
key_name: { get_resource: sshpublickey }
- port: { get_resource: hadoop_slave_port_$slavenumber$ }
# user_data:
# str_replace:
# template: |
# #!/bin/bash
# addgroup hadoop
# useradd --gid hadoop --home /home/hduser --create-home --shell /bin/bash hduser
# # as the only user to enter this VM is hduser, he needs to be able to access root functionality without a password
# echo "hduser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
# passwd --delete hduser
# su hduser -c "mkdir /home/hduser/.ssh"
# su hduser -c 'echo -e "$$" > /home/hduser/.ssh/authorized_keys'
# chmod 0600 /home/hduser/.ssh/config
# echo $info$
# params:
# $info$: "no info today"
############### end slave $slavenumber$ ##############
......@@ -17,12 +17,10 @@ import os
from sdk.mcn import util
from import service_orchestrator
from import LOG
# TODO replace print statements with LOG
# from neutronclient import client
import ConfigParser
class SOE(service_orchestrator.Execution):
def __init__(self, token, tenant): "initializing stack")
......@@ -38,8 +36,40 @@ class SOE(service_orchestrator.Execution):"token = "+self.token)"tenant = "+self.tenant)
# the instance variables have to be initialised - for more details, see
# defaultSettings.cfg
self.masterImage = None
self.slaveImage = None
self.masterFlavor = None
self.slaveFlavor = None
self.slaveOnMaster = None
self.SSHPublicKeyName = None
self.SSHMasterPublicKey = None
self.withFloatingIP = None
self.master_name = None
self.slave_name = None
self.subnet_cidr = None
self.subnet_gw_ip = None
self.subnet_allocation_pool_start = None
self.subnet_allocation_pool_end = None
self.subnet_dns_servers = None
self.image_id = None
self.floatingIpId = None
self.slaveCount = None
self.masterBash = None
self.transferMethod = None
self.diskId = None
self.username = None
self.usergroup = None
self.homedir = None
self.saveToLocalPath = None
self.noDeployment = None
def design(self):
print( "designing stack" );"designing stack")
Do initial design steps here.
......@@ -48,96 +78,95 @@ class SOE(service_orchestrator.Execution):
def deploy(self, attributes=None):"the attributes are:")
print( "deploying stack")
masterCount = 1
slaveCount = 1
slaveOnMaster = True
clusterName = "DistributedProcessing"
masterImage = "Ubuntu-Trusty-1404-7-10-2015"
slaveImage = "Ubuntu-Trusty-1404-7-10-2015"
sshKeyName = "mesz MNMBA"
sshPublicKey = ""
processingFramework = None
masterFlavor = "m1.small"
slaveFlavor = "m1.small"
if 'icclab.haas.master.number' in attributes:
masterCount = int(attributes['icclab.haas.master.number'])
if 'icclab.haas.slave.number' in attributes:
slaveCount = int(attributes['icclab.haas.slave.number'])
if '' in attributes:
clusterName = attributes['']
if 'icclab.haas.master.image' in attributes:
masterImage = attributes['icclab.haas.master.image']
if 'icclab.haas.slave.image' in attributes:
slaveImage = attributes['icclab.haas.slave.image']
if 'icclab.haas.master.flavor' in attributes:
masterFlavor = attributes['icclab.haas.master.flavor']
if 'icclab.haas.slave.flavor' in attributes:
slaveFlavor = attributes['icclab.haas.slave.flavor']
if 'icclab.haas.master.slaveonmaster' in attributes:
slaveOnMaster = attributes['icclab.haas.master.slaveonmaster'] in ['true', '1']
if 'icclab.haas.master.sshkeyname' in attributes:
sshKeyName = attributes['icclab.haas.master.sshkeyname']
if 'icclab.haas.master.publickey' in attributes:
sshPublicKey = attributes['icclab.haas.master.publickey']
if 'icclab.haas.cluster.procframework' in attributes:
processingFramework = attributes['icclab.haas.cluster.procframework']'Calling deploy')
LOG.debug('Executing deployment logic')
# compile heat template and deploy it
# TODO this will fail! :-) Fix
from import BUNDLE_DIR
rootFolder = "data/"
f = open(rootFolder + "cluster.yaml")
clusterTemplate =
f = open(rootFolder + "slave.yaml")
slaveTemplate =
f = open(rootFolder + "")
masterBash =
f = open(rootFolder + "master.id_rsa")
master_id_rsa =
master_id_rsa = master_id_rsa.replace("\n","\\n")
f = open(rootFolder + "")
master_id_rsa_pub ="\n","")
f = open(rootFolder + "yarn-site.xml")
yarn_site_xml =
f = open(rootFolder + "core-site.xml")
core_site_xml =
f = open(rootFolder + "mapred-site.xml")
mapred_site_xml =
f = open(rootFolder + "hdfs-site.xml")
hdfs_site_xml =
f = open(rootFolder + "")
hadoop_env_sh =
# this function will return the first of following values in the order
# of occurrence:
# - given by the user during instantiation
# - set as a default value in the config file
# - an empty string
def getAttr(attrName):
if(attrName in attributes):
return attributes[attrName]
return config.get('cluster',attrName)
return ""
# the rootFolder is needed in order to load the config files
self.rootFolder = getAttr('icclab.haas.rootfolder')
if self.rootFolder=="":
# if no rootFolder has been provided, take the path it's located
# within the Docker image
self.rootFolder = "data/"
# setup parser for config file
config = ConfigParser.RawConfigParser(),'defaultSettings.cfg'))"deploying stack...")
# the needed variables for the Heat Orchestration Template will be set
# on the following lines; the function getAttr() is returning the
# appropriate value. As an empty value cannot be used for int/float
# conversion, they have to be set within a try-except block.
# masterCount = 1
# try:
# masterCount = int(getAttr('icclab.haas.master.number'))
# except:
# pass
self.slaveCount = 1
self.slaveCount = int(getAttr('icclab.haas.slave.number'))
# clusterName = getAttr('')
self.masterImage = getAttr('icclab.haas.master.image')
self.slaveImage = getAttr('icclab.haas.slave.image')
self.masterFlavor = getAttr('icclab.haas.master.flavor')
self.slaveFlavor = getAttr('icclab.haas.slave.flavor')
self.slaveOnMaster = getAttr('icclab.haas.master.slaveonmaster').lower() in ['true', '1']
self.SSHPublicKeyName = getAttr('icclab.haas.master.sshkeyname')
self.SSHMasterPublicKey = getAttr('icclab.haas.master.publickey')
self.withFloatingIP = getAttr('icclab.haas.master.withfloatingip').lower() in ['true','1']
self.master_name = getAttr('')
self.slave_name = getAttr('')
self.subnet_cidr = getAttr('')
self.subnet_gw_ip = getAttr('')
self.subnet_allocation_pool_start = getAttr('')
self.subnet_allocation_pool_end = getAttr('')
self.subnet_dns_servers = getAttr('')
self.image_id = getAttr('icclab.haas.master.imageid')
self.floatingIpId = getAttr('icclab.haas.master.attachfloatingipwithid')
self.transferMethod = getAttr('icclab.haas.master.transfermethod')
self.username = getAttr('icclab.haas.cluster.username')
self.usergroup = getAttr('icclab.haas.cluster.usergroup')
self.homedir = getAttr('icclab.haas.cluster.homedir')
self.noDeployment = getAttr('icclab.haas.debug.donotdeploy').lower() in ['true','1']
self.saveToLocalPath = getAttr('icclab.haas.debug.savetemplatetolocalpath')
self.diskId = 'virtio-'+self.image_id[0:20]
masterSSHKeyEntry = ''
def getFileContent(fileName):
f = open(os.path.join(self.rootFolder, fileName))
retVal =
return retVal
# read all the necessary files for creating the Heat template
clusterTemplate = getFileContent("cluster.yaml");
slaveTemplate = getFileContent("slave.yaml")
self.masterBash = getFileContent("")
master_id_rsa = getFileContent("master.id_rsa").replace("\n","\\n")
master_id_rsa_pub = getFileContent("").replace("\n","")
yarn_site_xml = getFileContent("yarn-site.xml")
core_site_xml = getFileContent("core-site.xml")
mapred_site_xml = getFileContent("mapred-site.xml")
hdfs_site_xml = getFileContent("hdfs-site.xml")
hadoop_env_sh = getFileContent("")
slaves = ""
hostFileContent = ""
......@@ -146,83 +175,153 @@ class SOE(service_orchestrator.Execution):
slavesFile = ""
hostsListFile = ""