Execute a Sqoop Job using Windows Power Shell

#region - provide the following values

$subscriptionID = "XXXXXXXXXXXXXXXXXXXXXXXXXXX"

#region - variables

# Resource group variables
$resourceGroupName = "XXXXXXXXXXX"
$location = "XXXXXXXXXXX" # used by all Azure services defined in this tutorial


# HDInsight variables
$hdinsightClusterName = "XXXXXXXXXXXXXXXXX"
$defaultStorageAccountName = "XXXXXXXXXXXXXXXXXXX"
$defaultBlobContainerName = "XXXXXXXXXXXXXXXXXXXX"
$defaultStorageAccountKey=Get-AzureRmStorageAccountKey -ResourceGroupName $resourceGroupName -Name $defaultStorageAccountName | %{ $_.Key1 }


$username = "XXXXXXXXXX"
$password = " XXXXXXXXXX" | ConvertTo-SecureString -AsPlainText -Force
$httpCredential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $username, $password
#endregion

#region - Connect to Azure subscription
Write-Host "`nConnecting to your Azure subscription ..." -ForegroundColor Green
try{Get-AzureRmContext}
catch{Login-AzureRmAccount}
#endregion

#region - Create Azure resouce group
Write-Host "`nCreating an Azure resource group ..." -ForegroundColor Green
try{
    Get-AzureRmResourceGroup -Name $resourceGroupName
}
catch{
    New-AzureRmResourceGroup -Name $resourceGroupName -Location $location
}
#endregion

# Enter Table Name
$tableName_log4j = "XXXXXXXXXXXXXX"

# Connection string for Remote SQL Database.

$connectionString = "jdbc:sqlserver://sqlserver-vm1.cloudapp.net:1433;database=XXXXXXXXXXXXX;username=XXXXXXX;password=XXXXXXXX"

# Submit a Sqoop job
$sqoopDef = New-AzureRmHDInsightSqoopJobDefinition `
    -Command "import --connect $connectionString --table $tableName_log4j --num-mappers 32 --null-string '\\N' --null-non-string '\\N' --target-dir  /sqlserverdump/XXXXXXXXXXX"
$sqoopJob = Start-AzureRmHDInsightJob `
                -ClusterName $hdinsightClusterName `
                -HttpCredential $httpCredential `
                -JobDefinition $sqoopDef #-Debug -Verbose
Wait-AzureRmHDInsightJob `
    -ResourceGroupName $resourceGroupName `
    -ClusterName $hdinsightClusterName `
    -HttpCredential $httpCredential `
    -JobId $sqoopJob.JobId

Write-Host "Standard Error" -BackgroundColor Green
Get-AzureRmHDInsightJobOutput -ResourceGroupName $resourceGroupName -ClusterName $hdinsightClusterName -DefaultStorageAccountName $defaultStorageAccountName -DefaultStorageAccountKey $defaultStorageAccountKey -DefaultContainer $defaultBlobContainerName -HttpCredential $httpCredential -JobId $sqoopJob.JobId -DisplayOutputType StandardError
Write-Host "Standard Output" -BackgroundColor Green
Get-AzureRmHDInsightJobOutput -ResourceGroupName $resourceGroupName -ClusterName $hdinsightClusterName -DefaultStorageAccountName $defaultStorageAccountName -DefaultStorageAccountKey $defaultStorageAccountKey -DefaultContainer $defaultBlobContainerName -HttpCredential $httpCredential -JobId $sqoopJob.JobId -DisplayOutputType StandardOutput

#endregion