пропозиції роботи
konto usunięte
Temat: compress and split na 4.4GB pliki ?
Witam,zrobilem sobie skrypta ktory pakuje i kroi je na 4.4GB czesci by mozna je wypalic na DVD ale skrypt jest strasznie wolny.
Szukalem troche na google i z tego co pisza to wynika za sama tar komenda jest wolna.
Jesli ktos by mial jakies uwagi jak ulepszyc (przyspieszyc) mojego skrypta bylbym bardzo wdzieczny !!
Mysle by cos pozmieniac na gzip albo moze lzop ale chetnie sprubuje innych pomyslow.
Aha, file structure do kazdego Fileserver jest okolo 150GB
compress files, split into volumes and burn into DVD's
#!/bin/bash
#### NOTES ######################
# to join the files back into one file:
# for example to join all fileserver1 files into 1 .tar file use this:
# cat fileserver1-* > fileserver1_FULL.tar &
# to join all fileserver2 :
# cat fileserver2-* > fileserver2_FULL.tar &
# to join all fileserver3 :
# cat fileserver3-* > fileserver3_FULL.tar &
# to join all fileserver4 :
# cat fileserver4-* > fileserver4_FULL.tar &
######## ----------------------------##
##
#
# Script is divided into Sections:
# Section 1 - fileserver1
# Section 2 - fileserver2
# Section 3 - fileserver3
# Section 4 - fileserver4
#
###########################################
## ARCHIVE FILES AND SPLIT FOR DVD'S
rm -rf name.tmp
#### VARAIBLES & FUNCTIONS
LOG='compressionandburn.log'
FIL1='/fileserver1/'
FIL2='/fileserver2/'
FIL3='/fileserver3/'
FIL4='/fileserver4/'
TODAY=`date | awk '{print $1 $2 $3 $4}'`
FILE() {
echo "last24hrsForFIL1-"
}
TOD() {
date | awk '{print $1 $2 $3 $4}'
}
TOD2() {
date | awk '{print $1 $2 $3 }'
}
echo changedFilesFIL1-$(TOD).txt > name.tmp
CHANGEDFILES=`cat name.tmp`
TARNAME() {
cat name.tmp | awk -F. '{print $1}'
}
#### ACTIONS:
# this is testing and timing for search is different for test purposes
echo "START COMPRESSION AND SPLIT" >> $LOG
echo " " >> $LOG
echo " -------------------------------------------------------" >> $LOG
echo " " >> $LOG
echo " " >> $LOG
echo "---------Begin Section 1 -------" >> $LOG
echo "`date`" >> $LOG
echo " " >> $LOG
## -- Section 1 - FIL1 -- ##
echo "Begin search for changed files for FIL1" >> $LOG
echo "`date`" >> $LOG
#sudo find $FIL1 -type f -mtime -1 >> changedFilesFIL1.txt
# use below command for first time to create DVD's of full backup
sudo find $FIL1 -type f >> changedFilesFIL1.txt
## check if the file changedFilesFIL1.txt is empty ##
FIL1T="changedFilesFIL1.txt"
echo " " >> $LOG
echo "checking if $FIL1T is empty" >> $LOG
if [[ -s $FIL1T ]] ; then
echo " file $FIL1T is ok.. script goes on.. " >> $LOG
#
echo " " >> $LOG
echo "Search completed, now start to creating the tar file.." >> $LOG
for FILE in $(cat changedFilesFIL1.txt) ; do sudo tar -rvf changedFilesFIL1.tar $FILE >> archiving_FIL1-$TODAY.log 2>&1 ;done
echo "`date`" >> $LOG
echo " " >> $LOG
echo "big archive created for FIL1" >> $LOG
echo " " >> $LOG
echo "`ls -lh changedFilesFIL1.tar`" >>$LOG
echo " " >> $LOG
echo "start splitting for FIL1" >> $LOG
split --line-bytes=4400m changedFilesFIL1.tar FIL1-$(TOD)_
echo "`date`" >> $LOG
echo "split completed for FIL1" >> $LOG
echo " " >> $LOG
echo "`ls -lh FIL1-*`" >>$LOG
else
echo " Error!! File $FIL1T is empty !!" >> $LOG | mail -s " !! Error: File $FIL1T is EMPTY !!!!!" admin@domain.com
echo " Error: no tar will be created and compressed !! " >> $LOG
fi
mv changedFilesFIL1.txt changedFilesFIL1-$(TOD2).txt
rm -rf changedFilesFIL1.tar
echo " ------------ End of Section 1 -------" >> $LOG
## -- end of Section 1 -- ##
echo " " >>$LOG
echo " " >>$LOG
echo " " >>$LOG
echo " " >>$LOG
## -- Section 2 - FIL2 -- ##
echo " ------------ Begin of Section 2 ------" >> $LOG
echo "Begin to search changed files for FIL2" >> $LOG
echo "`date`" >> $LOG
echo " " >> $LOG
#sudo find $FIL2 -type f -mtime -1 >> changedFilesFIL2.txt
# use below command for first time to create DVD's of full backup
sudo find $FIL2 -type f >> changedFilesFIL2.txt
## check if the file changedFilesFIL2.txt is empty ##
FIL2T="changedFilesFIL2.txt"
echo "checking if $FIL2T is empty" >> $LOG
echo " " >> $LOG
if [[ -s $FIL2T ]] ; then
echo " file $FIL2T is ok.. script goes on.. " >> $LOG
#
echo "Search completed, now start to creating the tar file.." >> $LOG
echo " " >> $LOG
for FILE in $(cat changedFilesFIL2.txt) ; do sudo tar -rvf changedFilesFIL2.tar $FILE >> archiving_FIL2-$TODAY.log 2>&1 ;done
echo "`date`" >> $LOG
echo " " >> $LOG
echo "big archive created for FIL2" >> $LOG
echo " " >> $LOG
echo "`ls -lh changedFilesFIL2.tar`" >>$LOG
echo " " >> $LOG
echo "start splitting for FIL2" >> $LOG
split --line-bytes=4400m changedFilesFIL2.tar FIL2-$(TOD)_
echo "`date`" >> $LOG
echo " " >> $LOG
echo "splt completed for FIL2" >> $LOG
echo " " >> $LOG
echo "`ls -lh FIL2-*`" >>$LOG
else
echo " Error!! File $FIL2T is empty !!" >> $LOG | mail -s "!!! Error: File $FIL2T is EMPTY !!!!!" admin@domain.com
echo " Error: no tar will be created and compressed !! " >> $LOG
fi
mv changedFilesFIL2.txt changedFilesFIL2-$(TOD2).txt
rm -rf changedFilesFIL2.tar
echo " " >> $LOG
echo " ------------ End of Section 2 --------" >> $LOG
## -- end of Section 2 -- ##
echo " " >>$LOG
echo " " >>$LOG
echo " " >>$LOG
echo " " >>$LOG
## -- Section 3 - FIL3 -- ## (last 7 days)
echo " ---------- Begin of Section 3 --------" >> $LOG
echo "Begin to search changed files for FIL3" >> $LOG
echo " " >> $LOG
echo "`date`" >> $LOG
#sudo find $FIL3 -type f -mtime -7 >> changedFilesFIL3.txt
# use below command for first time to create DVD's of full backup
sudo find $FIL3 -type f >> changedFilesFIL3.txt
## check if the file changedFilesFIL3.txt is empty ##
FIL3T="changedFilesFIL3.txt"
echo "checking if $FIL3T is empty" >> $LOG
echo " " >> $LOG
if [[ -s $FIL3T ]] ; then
echo " file $FIL3T is ok.. script goes on.. " >> $LOG
#
echo "Search completed, now start to creating the tar file.." >> $LOG
for FILE in $(cat changedFilesFIL3.txt) ; do sudo tar -rvf changedFilesFIL3.tar $FILE >> archiving_FIL3-$TODAY.log 2>&1 ;done
echo "`date`" >> $LOG
echo " " >> $LOG
echo "big archive created for FIL3" >> $LOG
echo " " >> $LOG
echo "`ls -lh changedFilesFIL3.tar`" >>$LOG
echo " " >> $LOG
echo "start splitting for FIL3" >> $LOG
split --line-bytes=4400m changedFilesFIL3.tar FIL3-$(TOD)_
echo "`date`" >> $LOG
echo " " >> $LOG
echo "split complete for FIL3" >> $LOG
echo " " >> $LOG
echo "`ls -lh FIL3-*`" >>$LOG
else
echo " Error!! File $FIL3T is empty !!" >> $LOG | mail -s "Error: File $FIL3T is EMPTY !!!!!" admin@domain.com
echo " Error: no tar will be created and compressed !! " >> $LOG
fi
mv changedFilesFIL3.txt changedFilesFIL3-$(TOD2).txt
rm -rf changedFilesFIL3.tar
echo " " >> $LOG
echo " ---------- End of Section 3 --------" >> $LOG
## -- end of Section 3 -- ##
echo " " >>$LOG
echo " " >>$LOG
echo " " >>$LOG
echo " " >>$LOG
## -- Section 4 - FIL4 -- ## (last 706 days for testing)
echo " ------------ Begin of Section 4 ----------" >> $LOG
echo "Begin to search changed files for FIL4" >> $LOG
echo " " >> $LOG
echo "`date`" >> $LOG
#sudo find $FIL4 -type f -mtime -708 >> changedFilesFIL4.txt
##sudo find $FIL4 -type f -mtime -7 >> changedFilesFIL4.txt
# use below command for first time to create DVD's of full backup
sudo find $FIL4 -type f >> changedFilesFIL4.txt
## check if the file changedFilesFIL4.txt is empty ##
FIL4T="changedFilesFIL4.txt"
echo " " >> $LOG
echo "checking if $FIL4T is empty" >> $LOG
if [[ -s $FIL4T ]] ; then
echo " file $FIL4T is ok.. script goes on.. " >> $LOG
#
echo " " >> $LOG
echo "Search completed, now start to creating the tar file.." >> $LOG
for FILE in $(cat changedFilesFIL4.txt) ; do sudo tar -rvf changedFilesFIL4.tar $FILE >> archiving_FIL4-$TODAY.log 2>&1 ;done
echo "`date`" >> $LOG
echo " " >> $LOG
echo "big archive created for FIL4" >> $LOG
echo " " >> $LOG
echo "`ls -lh changedFilesFIL4.tar`" >>$LOG
echo " " >> $LOG
echo "start splitting for FIL4" >> $LOG
echo " " >> $LOG
split --line-bytes=4400m changedFilesFIL4.tar FIL4-$(TOD)_ 2>>$LOG
echo "`date`" >> $LOG
echo "spltting complete for FIL4" >> $LOG
echo " " >> $LOG
echo "`ls -lh FIL4-*`" >>$LOG
echo " " >> $LOG
else
echo " Error!! File $FIL4T is empty !!" >> $LOG | mail -s "Error: File $FIL4T is EMPTY !!!!!" admin@domain.com
echo " Error: no tar will be created and compressed !! " >> $LOG
fi
mv changedFilesFIL4.txt changedFilesFIL4-$(TOD).txt
rm -rf changedFilesFIL4.tar
## -- end of Section 4 -- ##
echo "`date`" >> $LOG
echo " " >> $LOG
echo "------------- End of Section 4 -------" >> $LOG
echo " " >> $LOG
echo " " >> $LOG
echo " -------------------------------------------------------" >> $LOG
echo "COMPRESSION AND SPLITTING COMPLETED " >> $LOG
cat $LOG | mail -s " TEST Compression and splitting completed" admin@domain.com
# rename log file
mv $LOG compressionandburn-$(TOD).log
## Cleanup ##
# remove the single tar files
rm -rf changedFiles*.tar
## End of the Script
###############################################
Z gory dziekuje !Ernest Gulik edytował(a) ten post dnia 20.02.10 o godzinie 03:21
