Qubole : Load Multiple tables to Qubole Hive table from a Data Store

API call to Load Multiple tables from a Qubole Data Store to Hive table. 

[rahul@local qubole]$ cat /databasescript 

#!/bin/bash

#Qubole API Key

AUTH="***********"

#Database Name

DB_NAME="***********"

#Host Name

DB_HOST="***********"

#User Name

DB_USER="***********

#Password 

DB_PASS='***********'

echo $DB_PASS

## request table import from tap;

function tableImport() {

request_body=$(cat <<EOF

{

   "command_type":"DbImportCommand",

   "mode":"1",

   "hive_serde":"orc",

   "hive_table":"<HIVE TABLE NAME>.$1",

   "dbtap_id":"$2",

   "db_table":"$1",

   "db_parallelism":"1",

   "use_customer_cluster":"1",

   "customer_cluster_label":"Qubole_Data_Import",

   "tags":[" Data"]

}

EOF

)

echo $request_body

   curl -X POST \

-H "X-AUTH-TOKEN: $AUTH" \

-H "Content-Type:application/json" \

-d "$request_body" https://api.qubole.com/api/v1.2/commands/

}

##register database with tap

request_body=$(cat <<EOF

{

  "db_name":"$DB_NAME",

  "db_host":"$DB_HOST",

  "db_user":"$DB_USER",

  "db_passwd":"$DB_PASS",

  "db_type":"sqlserver",

  "db_location":"on-premise",

  "gateway_ip": "***********",

  "gateway_port": "***********",

  "gateway_username": "***********",

  "gateway_private_key": "***********"}

EOF

)

echo $KEY

ID=$(curl -s -X POST \

-H "X-AUTH-TOKEN: $AUTH" \

-H "Content-Type:application/json" \

-d "$request_body" https://api.qubole.com/api/v1.2/db_taps/ | jq .id)

#get the tables and call import

curl -s -H "X-AUTH-TOKEN: $AUTH" \

     -H "Content-Type:application/json" \

     https://api.qubole.com/api/v1.2/db_taps/$ID/tables | jq -r .[] | while read x; do  tableImport $x $ID; done

# can't delete the tap at the end unless we continuously poll for no active jobs;

STATUS="null"

while [ "$STATUS" = "null" ]

do

STATUS=$(curl  -s -X DELETE \

 -H "X-AUTH-TOKEN: $AUTH" \

 -H "Content-Type:application/json" \

 https://api.qubole.com/api/v1.2/db_taps/$ID | jq .status)

echo -n "."

sleep 5

done