工作中linux常用命令备份

 

 

 

rm -rf conf_$DATE > /dev/null 2>&1
mkdir conf_$DATE
for t in `cat ../../table-oracle-tohbase.list | grep -v ^#`
 do
 table=`echo $t | awk -F ',' '{print $1}'`
  sh genconf.sh $table
 done

 

 

 

 

USUAGE="Usuage: bash run_all_bulkload.sh"

source ../../../conf.properties

LOG_DIR=logs_$DATE
if [ ! -d $LOG_DIR ]; then
  mkdir $LOG_DIR
fi

for t in `cat ../../table-oracle-tohbase.list | grep -v ^#`; do
	table=`echo $t | awk -F ',' '{print $1}'`
	echo "Bulkload "$table" ..."
	nohup sh run_bulkload.sh $table > $LOG_DIR/$table.log 2>&1 &
done

 

 

 

 

#! /bin/bash
USUAGE="Usuage: bash run_bulkload.sh [TABLENAME] <-test>"
PROJECT_HOME=.
CLASSPATH=$PROJECT_HOME/lib/bulkload.jar   
IS_TEST=1  #Default is false  闈炴祴璇?


if [ "$#" -lt "1"  ]; then
  echo $USUAGE
  exit 1
elif [ "$#" -eq "1" ]; then  
  TABLENAME=$1
elif [ "$#" -eq "2" ]; then
  TABLENAME=$1
  IS_TEST=0
else
  echo $USUAGE
  exit 1
fi

source ../../../conf.properties

tablename=$TABLENAME"_"$DATE
CONF_FILE=conf_$DATE/$tablename.properties
source $CONF_FILE

# When use the test mode, judge whether the test parameters have been set 
checktest=0
if [ "$IS_TEST" -eq "0" ]; then
  if [ "$test_outputDir" = "" ]; then
    echo "ERROR: You must set the test_outputDir in the conf file."
    checktest=1
  fi
  if [ "$test_tableName" = "" ]; then
    echo "ERROR: You must set the test_tableName in the conf file."
    checktest=1
  fi
fi

if [ "$checktest" -eq "1" ]; then
  exit 1
fi

if [ "$IS_TEST" -eq "0" ];then
  HFILE_DIR=$test_outputDir
  HBASE_TABLE_NAME=$test_tableName
  INDEX_TABLE_NAME=$test_indextablename
else
  HFILE_DIR=$outputDir   
  HBASE_TABLE_NAME=$tableName
  INDEX_TABLE_NAME=$indextablename  
fi

#======= Check and disable the hbase table ===============
echo "Check whether the table exists"   
if echo "list" | hbase shell | grep $tableName ; then
  echo "The table has existed, disable and drop the table"
  echo "disable '$tableName'; drop '$tableName'" | hbase shell
fi

echo "Start loading data into HBase at `date`"
start=`date +%s`


jars=`ls $PROJECT_HOME/lib`
for jar in $jars
do
    CLASSPATH="$CLASSPATH:$PROJECT_HOME/lib/$jar"   
done

CLASSPATH=/etc/hdfs1/conf:/etc/hadoop/conf:/etc/hbase/conf:$CLASSPATH  
sudo -u hdfs hdfs dfs -rm -r $HFILE_DIR   
if [ "$IS_TEST" -eq "0" ]; then
  java -Djava.library.path=/usr/lib/hadoop/lib/native -cp $CLASSPATH com.transwarp.hbase.bulkload.ImportTextFile2HBase $CONF_FILE -test
else
  java -Djava.library.path=/usr/lib/hadoop/lib/native -cp $CLASSPATH com.transwarp.hbase.bulkload.ImportTextFile2HBase $CONF_FILE   
fi

sleep 60
sudo -u hdfs hdfs dfs -chmod -R 777 $HFILE_DIR
sleep 60

if [ "$INDEX_TABLE_NAME" != "" ]; then 
   /usr/lib/hbase/bin/hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles $HFILE_DIR $INDEX_TABLE_NAME
else 
	/usr/lib/hbase/bin/hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles $HFILE_DIR $HBASE_TABLE_NAME 
fi
#echo "Delete the hfile output dir "
#sudo -u hdfs hdfs dfs -rmr $HFILE_DIR

end=`date +%s`
interval=`expr $end - $start`
echo "End loading data into HBase at `date`"
echo "Time used $interval seconds"