HdInsight Cluster Backup Strategy - PART 1 bootup.ps1

Create a PowerShell Script as shown below:

$servername = "xxxxxxxx-ssh.azurehdinsight.net"
$username = "admin123"
$password = "Test@45678"
$port = 22
$command2 = "hadoop fs -copyToLocal wasb://container-name@storage-name.blob.core.windows.net/backup/* /home/admin123/"
$command3 = "sudo chmod -R 777 /home/admin123/*"
$command4 = "hadoop fs -rm -R wasb://conatinerarchive@storage-name.blob.core.windows.net/archive/*"
$command5 = "hadoop fs -copyFromLocal /home/admin123/* wasb://conatinerarchive@storage-name.blob.core.windows.net/archive/ "
$command6 = "/home/admin123/scripts/executehivebootup.sh"
$command7 = "/home/admin123/HueScripts/hueurl.sh"
$command8 = "sudo /home/admin123/scripts/add_configuration.sh"
$command9 = "sudo apt-get install expect"
$command10 = "/home/admin123/scripts/SFTPrun.sh"

$commandname = @($command2,$command3,$command4,$command5)

Import-Module SSH-Sessions
New-SshSession -ComputerName $servername -Username $username -Password $password -Port $port

Write-Host "Started bootup data pull..!!!"

foreach ($c in $commandname) {
$SshResults = Invoke-SshCommand -InvokeOnAll -Command $c
}
Write-Host "Step 1 : Done With Copying..!!!"

Write-Host "Step 2 : Creating Schemas..."

$SshResults2 = Invoke-SshCommand -InvokeOnAll -Command $command6

Write-Host "Step 3 : Generating Hue Link...."

$SshResults3 = Invoke-SshCommand -InvokeOnAll -Command $command7

Write-Host "Step 4 : Completed...."

$SshResults4 = Invoke-SshCommand -InvokeOnAll -Command $command8

Write-Host "Step 5 : Completed core site changes..."

$SshResults5 = Invoke-SshCommand -InvokeOnAll -Command $command9

Write-Host "Step 6 : Installed dependent libraries..."

$SshResults5 = Invoke-SshCommand -InvokeOnAll -Command $command10

Write-Host "Step 7 : Installed  SFTP dependencies..."

Remove-SshSession -RemoveAll


***************************************
Contents of shell scripts used:
***************************************

executehivebootup.sh

oxi_array=(`find /home/admin123/HiveStartup/ -type f -follow` )
for element in "${oxi_array[@]}"
do
  hive -f "${element}"
done

Content of HiveStartup
Create a folder HiveSchema - Add a file xxxdata.hql
xxxdata.hql:
CREATE DATABASE IF NOT EXISTS xxxdata;
USE xxxdata;
CREATE EXTERNAL TABLE IF NOT EXISTS xxxdata( Bill_account string,Reading_type string, Reading_datetime Timestamp,Reading_value string ,`Interval` string )
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '|'
LOCATION 'wasb://container-name@storageacc-name.blob.core.windows.net/xxxdata/';


hueurl.sh

hostname -f> /home/admin123/HueScripts/hueurl.properties
sudo sed -i s/hn0/huelink=hn0/ /home/admin123/HueScripts/hueurl.properties

Content of hueurl.properties

huelink=hn0-linuxxxxxxxxxxxxxxxjqd33z3g.ex.internal.cloudapp.net


add_configuration.sh

chmod -R 777 /usr/hdp/2.2.*/hadoop/lib
cp /home/admin123/jar/com.mysql.jdbc_5.1.5.jar /usr/hdp/2.2.*/hadoop/lib/
cp /etc/hadoop/conf/core-site.xml /home/admin123/scripts/
awk '/<configuration>/ {print;print "<property>\n<name>fs.azure.account.key.storageaccname.blob.core.windows.net</name>\n<value>Fajj9v2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxetg==</value>\n</property>\n<property>\n<name>fs.azure.account.key.storageaccname.blob.core.windows.net</name>\n<value>vNcoxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxrUw==</value>\n</property>\n<property>\n<name>fs.azure.account.key.storageaccname.blob.core.windows.net</name>\n<value>PUWxxxxxxxxxxxxxxxxxxHq5iQ==</value>\n</property>"; next }1' /home/admin123/scripts/core-site.xml > /etc/hadoop/conf/core-site.xml


SFTPrun.sh

sftp_array=(`find /home/admin123/SFTP/sftpfirsttime/ -type f -follow` )
for element in "${sftp_array[@]}"
do
  "${element}"
done

Contents of sfttpfirsttime: ftp_test_com.sh

#!/usr/bin/expect

spawn sftp demo@test.rebex.net
expect "yes/no)?"
send "yes\n"
expect "password:"
send "password\n"
expect "sftp>"
send "exit\n"
interact