This document contains Vagrant configuration files for setting up virtual machine clusters for the Cloudera, Hortonworks, and MapR Hadoop distributions. It defines scripts to install each distribution's packages and configure SSH keys and host files on the VMs. The Vagrantfiles configure multiple VMs with private networking and provisioning to install and set up the different Hadoop distributions.

1. Vagrantfile for Cloudera 5.3 Hadoop distribution
$master_script = <<SCRIPT
#!/bin/bash
# This script will execute for master node
echo `date` Setting up keys for root
# Copy ssh key from your repository to vm machine
# Below is the example
mkdir /root/.ssh
wget 192.168.2.177/testbed/keys/id_rsa -O /root/.ssh/id_rsa
wget 192.168.2.177/testbed/keys/id_rsa.pub -O /root/.ssh/id_rsa.pub
wget 192.168.2.177/testbed/keys/authorized_keys -O /root/.ssh/authorized_keys
chmod 600 -R /root/.ssh
apt-get install curl -y
REPOCM=${REPOCM:-cm5.3}
CM_REPO_HOST=${CM_REPO_HOST:-archive.cloudera.com}
CM_MAJOR_VERSION=$(echo $REPOCM| sed -e 's/cm([0-9]).*/1/')
CM_VERSION=$(echo $REPOCM | sed -e 's/cm([0-9][0-9]*)/1/')
OS_CODENAME=$(lsb_release -sc)
OS_DISTID=$(lsb_release -si | tr '[A-Z]' '[a-z]')
if [ $CM_MAJOR_VERSION -ge 4 ]; then
cat > /etc/apt/sources.list.d/cloudera-$REPOCM.list <<EOF
deb [arch=amd64]
http://$CM_REPO_HOST/cm$CM_MAJOR_VERSION/$OS_DISTID/$OS_CODENAME/amd64/
cm $OS_CODENAME-$REPOCMcontrib
deb-src
http://$CM_REPO_HOST/cm$CM_MAJOR_VERSION/$OS_DISTID/$OS_CODENAME/amd64/
cm $OS_CODENAME-$REPOCMcontrib
EOF
curl -s
http://$CM_REPO_HOST/cm$CM_MAJOR_VERSION/$OS_DISTID/$OS_CODENAME/amd64/
cm/archive.key > key
apt-key add key
rm key
fi
apt-get update
export DEBIAN_FRONTEND=noninteractive
# Download Cloudera packages
apt-get -q -y --force-yes install oracle-j2sdk1.7 cloudera-manager-server-db cloudera-
manager-server cloudera-manager-daemons
service cloudera-scm-server-db initdb
service cloudera-scm-server-db start
service cloudera-scm-server start
SCRIPT

2. $hosts_script = <<SCRIPT
# This script will execute on all nodes
echo `date` Setting up keys for root
mkdir /root/.ssh
wget 192.168.2.177/testbed/keys/id_rsa -O /root/.ssh/id_rsa
wget 192.168.2.177/testbed/keys/id_rsa.pub -O /root/.ssh/id_rsa.pub
wget 192.168.2.177/testbed/keys/authorized_keys -O /root/.ssh/authorized_keys
chmod 600 -R /root/.ssh
cat > /etc/hosts <<EOF
127.0.0.1 localhost
10.211.55.100 vm-cdh-node1.test.com
10.211.55.101 vm-cdh-node2.test.com
10.211.55.102 vm-cdh-node3.test.com
10.211.55.103 vm-cdh-node4.test.com
# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
EOF
SCRIPT
Vagrant.configure("2") do |config|
# Define base image
config.vm.box = "precise64"
config.vm.box_url = "http://files.vagrantup.com/precise64.box"
# Manage /etc/hosts on host and VMs
# config.hostmanager.enabled = false
# config.hostmanager.manage_host = true
# config.hostmanager.include_offline = true
# config.hostmanager.ignore_private_ip = false
config.vm.define :master do |master|
master.vm.provider :virtualbox do |v|
v.name = "vm-cdh-node1"
v.customize ["modifyvm", :id, "--memory", "8192"]
end
master.vm.network :private_network, ip: "10.211.55.100"

3. master.vm.network :forwarded_port, guest: 9090, host: 9091
master.vm.hostname = "vm-cdh-node1.test.com"
master.vm.provision :shell, :inline => $hosts_script
# master.vm.provision :hostmanager
master.vm.provision :shell, :inline => $master_script
end
config.vm.define :slave1 do |slave1|
slave1.vm.box = "precise64"
slave1.vm.provider :virtualbox do |v|
v.name = "vm-cdh-node2"
v.customize ["modifyvm", :id, "--memory", "8192"]
end
slave1.vm.network :private_network, ip: "10.211.55.101"
slave1.vm.hostname = "vm-cdh-node2.test.com"
slave1.vm.provision :shell, :inline => $hosts_script
# slave1.vm.provision :hostmanager
end
config.vm.define :slave2 do |slave2|
slave2.vm.box = "precise64"
slave2.vm.provider :virtualbox do |v|
v.name = "vm-cdh-node3"
v.customize ["modifyvm", :id, "--memory", "8192"]
end
slave2.vm.network :private_network, ip: "10.211.55.102"
slave2.vm.hostname = "vm-cdh-node3.test.com"
slave2.vm.provision :shell, :inline => $hosts_script
# slave2.vm.provision :hostmanager
end
config.vm.define :slave3 do |slave3|
slave3.vm.box = "precise64"
slave3.vm.provider :virtualbox do |v|
v.name = "vm-cdh-node4"
v.customize ["modifyvm", :id, "--memory", "8192"]
end
slave3.vm.network :private_network, ip: "10.211.55.103"
slave3.vm.hostname = "vm-cdh-node4.test.com"
slave3.vm.provision :shell, :inline => $hosts_script
# slave3.vm.provision :hostmanager
end
end

4. Vagrantfile for HDP 2.2 Hadoop distribution
$master_script = <<SCRIPT
#!/bin/bash
echo `date` Setting up keys for root
mkdir /root/.ssh
wget 192.168.2.177/testbed/keys/id_rsa -O /root/.ssh/id_rsa
wget 192.168.2.177/testbed/keys/id_rsa.pub -O /root/.ssh/id_rsa.pub
wget 192.168.2.177/testbed/keys/authorized_keys -O /root/.ssh/authorized_keys
chmod 600 -R /root/.ssh
#apt-get install curl -y
# Download and move the Ambari repo bits to /etc/yum.repos.d
#wget http://public-repo-
1.hortonworks.com/ambari/centos6/2.x/updates/2.0.0/ambari.repo
#mv ambari.repo /etc/yum/repos.d
cd /etc/apt/sources.list.d
sudo wget http://public-repo-
1.hortonworks.com/ambari/ubuntu12/2.x/updates/2.0.0/ambari.list
#Check repo has been configured correctly
#yum repolist
#Install Ambari
sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com B9733A7A07513CAD
sudo apt-get update
sudo apt-get -q -y install ambari-server
#Default configuration
ambari-server setup -s
#Spin up Ambari
ambari-server start
SCRIPT
$hosts_script = <<SCRIPT
sudo apt-get -q -y install wget
sudo apt-get -q -y install ntp
#sudo service ntp status
echo `date` Setting up keys for root
mkdir /root/.ssh

5. wget 192.168.2.177/testbed/keys/id_rsa -O /root/.ssh/id_rsa
wget 192.168.2.177/testbed/keys/id_rsa.pub -O /root/.ssh/id_rsa.pub
wget 192.168.2.177/testbed/keys/authorized_keys -O /root/.ssh/authorized_keys
chmod 600 -R /root/.ssh
cat > /etc/hosts <<EOF
127.0.0.1 localhost
10.211.55.105 vm-hdp-node1
10.211.55.106 vm-hdp-node2
10.211.55.107 vm-hdp-node3
10.211.55.108 vm-hdp-node4
# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
EOF
SCRIPT
Vagrant.configure("2") do |config|
# Define base image
config.vm.box = "precise64"
config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/precise/current/precise-
server-cloudimg-amd64-vagrant-disk1.box"
# Manage /etc/hosts on host and VMs
# config.hostmanager.enabled = false
# config.hostmanager.manage_host = true
# config.hostmanager.include_offline = true
# config.hostmanager.ignore_private_ip = false
config.vm.define :master do |master|
master.vm.provider :virtualbox do |v|
v.name = "vm-hdp-node1"
v.customize ["modifyvm", :id, "--memory", "32768"]
end
master.vm.network :private_network, ip: "10.211.55.105"
# master.vm.network :forwarded_port, guest: 9090, host: 9091
master.vm.hostname = "vm-hdp-node1"
master.vm.provision :shell, :inline => $hosts_script
# master.vm.provision :hostmanager
master.vm.provision :shell, :inline => $master_script
end

6. config.vm.define :slave1 do |slave1|
slave1.vm.box = "precise64"
slave1.vm.provider :virtualbox do |v|
v.name = "vm-hdp-node2"
v.customize ["modifyvm", :id, "--memory", "32768"]
end
slave1.vm.network :private_network, ip: "10.211.55.106"
slave1.vm.hostname = "vm-hdp-node2"
slave1.vm.provision :shell, :inline => $hosts_script
# slave1.vm.provision :hostmanager
end
config.vm.define :slave2 do |slave2|
slave2.vm.box = "precise64"
slave2.vm.provider :virtualbox do |v|
v.name = "vm-hdp-node3"
v.customize ["modifyvm", :id, "--memory", "32768"]
end
slave2.vm.network :private_network, ip: "10.211.55.107"
slave2.vm.hostname = "vm-hdp-node3"
slave2.vm.provision :shell, :inline => $hosts_script
# slave2.vm.provision :hostmanager
end
config.vm.define :slave3 do |slave3|
slave3.vm.box = "precise64"
slave3.vm.provider :virtualbox do |v|
v.name = "vm-hdp-node4"
v.customize ["modifyvm", :id, "--memory", "32768"]
end
slave3.vm.network :private_network, ip: "10.211.55.108"
slave3.vm.hostname = "vm-hdp-node4"
slave3.vm.provision :shell, :inline => $hosts_script
# slave3.vm.provision :hostmanager
end
end

7. Vagrantfile for MAPR Hadoop distribution
$master_script = <<SCRIPT
#!/bin/bash
#Download MAPR installer
#
mkdir -p mapr-installer
cd mapr-installer
wget http://package.mapr.com/releases/installer/mapr-setup.sh
sudo sh mapr-setup.sh -y install
SCRIPT
$hosts_script = <<SCRIPT
sudo apt-get -q -y install wget
#sudo service ntp status
echo `date` Setting up keys for root
mkdir /root/.ssh
wget 192.168.2.177/testbed/keys/id_rsa -O /root/.ssh/id_rsa
wget 192.168.2.177/testbed/keys/id_rsa.pub -O /root/.ssh/id_rsa.pub
wget 192.168.2.177/testbed/keys/authorized_keys -O /root/.ssh/authorized_keys
chmod 600 -R /root/.ssh
cat > /etc/hosts <<EOF
127.0.0.1 localhost
10.211.55.108 mapr-node1
10.211.55.109 mapr-node2
10.211.55.110 mapr-node3
10.211.55.111 mapr-node4
10.211.55.112 mapr-node5
# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
EOF
SCRIPT
Vagrant.configure("2") do |config|

8. # Define base image
config.vm.box = "precise64"
config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/precise/current/precise-
server-cloudimg-amd64-vagrant-disk1.box"
# Manage /etc/hosts on host and VMs
# config.hostmanager.enabled = false
# config.hostmanager.manage_host = true
# config.hostmanager.include_offline = true
# config.hostmanager.ignore_private_ip = false
config.vm.define :master do |master|
master.vm.provider :virtualbox do |v|
v.name = "mapr-node1"
v.customize ["modifyvm", :id, "--memory", "32768"]
unless File.exist?('datadisk1.vdi')
v.customize ['createhd', '--filename', 'datadisk1.vdi', '--size', 50 * 1024]
end
v.customize ['storageattach', :id, '--storagectl', 'SATAController', '--port', 1, '--
device', 0, '--type', 'hdd', '--medium', 'datadisk1.vdi']
end
master.vm.network :private_network, ip: "10.211.55.108"
master.vm.hostname = "mapr-node1"
master.vm.network "forwarded_port", guest: 9443, host: 9443
master.vm.network "forwarded_port", guest: 22, host: 9444
master.vm.provision :shell, :inline => $hosts_script
master.vm.provision :shell, :inline => $master_script
end
config.vm.define :slave1 do |slave1|
slave1.vm.box = "precise64"
slave1.vm.provider :virtualbox do |v|
v.name = "mapr-node2"
v.customize ["modifyvm", :id, "--memory", "32768"]
unless File.exist?('datadisk2.vdi')
v.customize ['createhd', '--filename', 'datadisk2.vdi', '--size', 50 * 1024]
end
v.customize ['storageattach', :id, '--storagectl', 'SATAController', '--port', 1, '--
device', 0, '--type', 'hdd', '--medium', 'datadisk2.vdi']
end
slave1.vm.network :private_network, ip: "10.211.55.109"
slave1.vm.hostname = "mapr-node2"
slave1.vm.provision :shell, :inline => $hosts_script
end
config.vm.define :slave2 do |slave2|
slave2.vm.box = "precise64"
slave2.vm.provider :virtualbox do |v|
v.name = "mapr-node3"
v.customize ["modifyvm", :id, "--memory", "32768"]

9. unless File.exist?('datadisk3.vdi')
v.customize ['createhd', '--filename', 'datadisk3.vdi', '--size', 50 * 1024]
end
v.customize ['storageattach', :id, '--storagectl', 'SATAController', '--port', 1, '--
device', 0, '--type', 'hdd', '--medium', 'datadisk3.vdi']
end
slave2.vm.network :private_network, ip: "10.211.55.110"
slave2.vm.hostname = "mapr-node3"
slave2.vm.provision :shell, :inline => $hosts_script
end
config.vm.define :slave3 do |slave3|
slave3.vm.box = "precise64"
slave3.vm.provider :virtualbox do |v|
v.name = "mapr-node4"
v.customize ["modifyvm", :id, "--memory", "32768"]
unless File.exist?('datadisk4.vdi')
v.customize ['createhd', '--filename', 'datadisk4.vdi', '--size', 50 * 1024]
end
v.customize ['storageattach', :id, '--storagectl', 'SATAController', '--port', 1, '--
device', 0, '--type', 'hdd', '--medium', 'datadisk4.vdi']
end
slave3.vm.network :private_network, ip: "10.211.55.111"
slave3.vm.hostname = "mapr-node4"
slave3.vm.provision :shell, :inline => $hosts_script
end
config.vm.define :slave4 do |slave4|
slave4.vm.box = "precise64"
slave4.vm.provider :virtualbox do |v|
v.name = "mapr-node5"
v.customize ["modifyvm", :id, "--memory", "32768"]
unless File.exist?('datadisk5.vdi')
v.customize ['createhd', '--filename', 'datadisk5.vdi', '--size', 50 * 1024]
end
v.customize ['storageattach', :id, '--storagectl', 'SATAController', '--port', 1, '--
device', 0, '--type', 'hdd', '--medium', 'datadisk5.vdi']
end
slave4.vm.network :private_network, ip: "10.211.55.112"
slave4.vm.hostname = "mapr-node5"
slave4.vm.provision :shell, :inline => $hosts_script
end
end