Difference between revisions of "Bash"

From Blue-IT.org Wiki

(→‎Search and replace text in a document with sed)
(→‎ia32-libs)
 
(22 intermediate revisions by the same user not shown)
Line 5: Line 5:
 
  export LC_MESSAGES="en_US.UTF-8"
 
  export LC_MESSAGES="en_US.UTF-8"
 
  export LC_TYPE="en_US.UTF-8"
 
  export LC_TYPE="en_US.UTF-8"
−
  export LANGUAGE="en_US.UTF-8
+
  export LANGUAGE="en_US.UTF-8"
 +
 
 +
 
 +
== Man ==
 +
Everyone knows how to do
 +
man find
 +
 
 +
but did you know that you can printout any manpage in postscript?
 +
 
 +
man -t  find > man_find.ps
 +
 
 +
A simple script does this, changes it to pdf and opens the file in evince ;-) Cool and much much better for reading!
 +
 
 +
#!/bin/sh
 +
#
 +
# man2pdf
 +
# www.apos.de 2005
 +
#
 +
# Print a man page as pdf and save it to the users
 +
# desktop ( ~/tmp/. )#
 +
VER=0.3
 +
VIEWER="/usr/bin/evince"
 +
TEMP="/tmp"
 +
STORE="/tmp"
 +
 +
#
 +
if test $# -lt 1
 +
then
 +
cat <<EOF
 +
USE:
 +
man2pdf
 +
EOF
 +
 +
else
 +
 +
if man ${1} > /dev/null
 +
then
 +
  man -t ${1} > /${TEMP}/man_${1}.ps
 +
ps2pdf14 /${TEMP}/man_${1}.ps /${STORE}/${1}_manpage.pdf
 +
rm /${TEMP}/man_${1}.ps
 +
 +
exec ${VIEWER} /${STORE}/${1}_manpage.pdf &
 +
else
 +
echo "Sorry, there is no manpage for ${1} - this program."
 +
fi
 +
 +
fi
 +
 
 +
== Terminal or GUI ==
 +
You can determine, if a script is running in a terminal (TTY) or within a gui:
 +
 
 +
# check, if we are running in a terminal
 +
 +
if tset -q | grep linux
 +
then
 +
echo "Running in terminal session ... continue the script "
 +
        GUI=0
 +
 +
else
 +
# Open a GUI dialog
 +
if zenity --question --title "GUI Dialog" \
 +
--text "Would you like to continue?"
 +
        then
 +
            # set some vars
 +
            GUI=1
 +
        else
 +
            exit 1
 +
        fi
 +
 +
fi
  
  
Line 131: Line 200:
  
 
* Source: http://stackoverflow.com/questions/7161821/how-to-grep-a-continuous-stream
 
* Source: http://stackoverflow.com/questions/7161821/how-to-grep-a-continuous-stream
 +
 +
=== remove  empty lines ===
 +
cat some_text |  egrep -v '^#|^$'
 +
 +
=== remove carriage return (newline ===
 +
echo "something" | tr -d '\n'
  
 
== Graphical Tools ==
 
== Graphical Tools ==
Line 255: Line 330:
 
=== Search and replace text in a document with sed===
 
=== Search and replace text in a document with sed===
  
 +
<pre>
 
  # Search and replace text in a document globally or first
 
  # Search and replace text in a document globally or first
 +
# apos@gmx.de - 2008
 
  #
 
  #
 
  # param1: search string
 
  # param1: search string
Line 261: Line 338:
 
  # param3: the document URL
 
  # param3: the document URL
 
  search_replace_global(){
 
  search_replace_global(){
−
 
+
 
search="$(printf "%s\n" "${1}" | sed 's/[][\.*^$/]/\\&/g')"
 
search="$(printf "%s\n" "${1}" | sed 's/[][\.*^$/]/\\&/g')"
 
replace="$(printf "%s\n" "${2}" | sed 's/[][\.*^$/]/\\&/g')"
 
replace="$(printf "%s\n" "${2}" | sed 's/[][\.*^$/]/\\&/g')"
Line 267: Line 344:
 
    
 
    
 
  }
 
  }
−
 
+
 
 
  search_replace_first(){
 
  search_replace_first(){
 
    
 
    
Line 275: Line 352:
 
    
 
    
 
  }
 
  }
 +
</pre>
 +
 +
=== Cut text out of a file ===
 +
<pre>
 +
# Cut text out of a file.
 +
# Searches for the first (!) - and only the first - occurence of the Expression
 +
# "FIRST_TEXT_FRAGMENT" and then, from that line on to the first occurence of
 +
# "LAST_TEXT_FRAGMENT". It then cuts the lines between these two -
 +
# including the found lines (!) - out with sed.
 +
#
 +
# param1: first text fragment
 +
# param2: second text fragment
 +
# param3: input file - the file to inspect
 +
cut_text(){
 +
 +
THEDOC="${3}"
 +
#FIRST_TEXT_FRAGMENT="$(printf "%s\n" "${1}" | sed 's/[][\.*^$/]/\\&/g')"
 +
FIRST_TEXT_FRAGMENT="${1}"
 +
#LAST_TEXT_FRAGMENT="$(printf "%s\n" "${2}" | sed 's/[][\.*^$/]/\\&/g')"
 +
LAST_TEXT_FRAGMENT="${2}"
 +
 +
BEGIN_CUT=$(cat ${THEDOC} | fgrep -n --max-count=1 "${FIRST_TEXT_FRAGMENT}" | cut -d':' -f1)
 +
BEGIN_CUT=$(expr ${BEGIN_CUT})
 +
echo ${BEGIN_CUT}
 +
 +
END_CUT=$(cat ${THEDOC} | sed -e "1,${BEGIN_CUT}d" | fgrep -n --max-count=1 "${LAST_TEXT_FRAGMENT}" | cut -d':' -f1)
 +
echo ${END_CUT}
 +
END_CUT=$(expr ${END_CUT} + ${BEGIN_CUT})
 +
echo ${END_CUT}
 +
 +
sed -i "${BEGIN_CUT},${END_CUT}d" "${THEDOC}"
 +
 +
}
 +
</pre>
 +
 +
=== Read a file by line and manipulate content ===
 +
<pre>
 +
# get the linecount of the file
 +
lines=$(wc -l myFile | awk '{print $1}')
 +
 +
# process the file line by line
 +
for l in $line
 +
  do head -n $l Blink.ino | sed -e 's/delay/XXXXX/g';
 +
done
 +
</pre>
 +
 +
== Copy director tree structure without content ==
 +
cd /path/to/top
 +
find . -type d -depth | cpio -pvdma /path/to
 +
 +
 +
== Renaming files with bad content ==
 +
 +
find . -depth -name "*.*" -execdir rename 's/:/_/g;s/:/_/g;s/</_/g;s/>/_/g;' "{}" \;
 +
 +
* https://resource.dopus.com/viewtopic.php?t=7318#p35395
 +
 +
also see
 +
 +
sudo apt-get install detox
 +
detox --help
 +
 +
== Finding duplicate files ==
 +
 +
apt-get install fdupes
 +
 +
fdupes -r -d -S .
 +
 +
-> interactively delete files.
 +
 +
== Find and delete .cache ==
 +
 +
This cleans up the .cache directory and deletes all files that are older than 14 days  AND if the files are greater than 30MB:
 +
 +
sudo vim /etc/crontab
 +
 +
@daily    find /home/YOUR_USERNAME/.cache/ -atime +14 -size 30M -delete
 +
 +
See: http://askubuntu.com/questions/176480/limit-the-size-of-a-directory-by-deleting-old-files
 +
 +
== Kill all processes with a certain name ==
 +
 +
vim ~/bin/killall_
 +
 +
#!/bin/bash
 +
PROGRAM="${1}"
 +
 +
if [ ${PROGRAM} = "" ]
 +
then
 +
echo "Programname required."
 +
exit 1
 +
fi
 +
 +
KILL="$(ps x | grep -v grep | grep -v killall | grep -v $(basename $0) | grep "${PROGRAM}" | sed 's/^\ //' | cut -d' ' -f1 | sed 's/ //g')"
 +
 +
if [ "${KILL}" != "" ]
 +
then
 +
for killme in "${KILL}"
 +
do
 +
echo "Killing <9> process number $killme"
 +
kill -9 $killme
 +
done
 +
else
 +
echo "Nothing to kill ..."
 +
fi
 +
 +
== ia32-libs ==
 +
 +
Older Ubuntu versions:
 +
sudo apt-get install ia32-libs
 +
 +
Newer (> 14.04 ) Ubuntu versions:
 +
sudo apt-get install lib32z1 lib32ncurses5 lib32bz2-1.0
 +
  
 +
== Troubleshooting ==
 +
I am refferring to this excellent guide:
 +
* http://mywiki.wooledge.org/BashPitfalls?highlight=never#for_i_in_.24.28ls_.2A.mp3.29
  
 
[[Category:Network]]
 
[[Category:Network]]
 
[[Category:Bash]]
 
[[Category:Bash]]

Latest revision as of 18:21, 31 May 2017

Environment

Put this in front of any script to avoid problems when running the script in other languages! The en_US locale should be avaiabel on all systems: 

export LC_MESSAGES="en_US.UTF-8"
export LC_TYPE="en_US.UTF-8"
export LANGUAGE="en_US.UTF-8"


Man

Everyone knows how to do 

man find

but did you know that you can printout any manpage in postscript? 

man -t  find > man_find.ps

A simple script does this, changes it to pdf and opens the file in evince ;-) Cool and much much better for reading! 

#!/bin/sh
#
# man2pdf
# www.apos.de 2005
#
# Print a man page as pdf and save it to the users
# desktop ( ~/tmp/. )#
VER=0.3
VIEWER="/usr/bin/evince"
TEMP="/tmp"
STORE="/tmp"

#
if test $# -lt 1
then
cat <<EOF
USE:
man2pdf 
EOF

else

	if man ${1} > /dev/null
	then
 		man -t ${1} > /${TEMP}/man_${1}.ps
		ps2pdf14 /${TEMP}/man_${1}.ps /${STORE}/${1}_manpage.pdf
		rm /${TEMP}/man_${1}.ps

		exec ${VIEWER} /${STORE}/${1}_manpage.pdf &
	else
		echo "Sorry, there is no manpage for ${1} - this program."
	fi

fi

Terminal or GUI

You can determine, if a script is running in a terminal (TTY) or within a gui: 

# check, if we are running in a terminal

if tset -q | grep linux
then
	echo "Running in terminal session ... continue the script "
       GUI=0

else
	# Open a GUI dialog
	if zenity --question --title "GUI Dialog" \
		--text "Would you like to continue?"
       then
            # set some vars
            GUI=1
       else
            exit 1
       fi

fi


Sending mails

mailx - former known as nail

Tested in

  • Ubuntu 12.04
  • Ubuntu 14.04

Tested with:

  • gmx
  • google

Inspired by

Install packages

sudo apt-get install ca-certificates heirloom-mailx msmtp

Edit configuration files

One for msmtp: 

vim ~/.msmtprc

# config options: http://msmtp.sourceforge.net/doc/msmtp.html#A-user-configuration-file
defaults
logfile /tmp/msmtp.log

# settings for gmx
account gmx
auth on
host mail.gmx.de
port 587
user username@gmx.de
from username@gmx.de
password PASSHERE
tls on
tls_trust_file /etc/ssl/certs/ca-certificates.crt

# settings for google
account gmx
auth on
host smtp.gmail.com
port 587
user username@gmail.com
from username@gmail.com
password PASSHERE
tls on
tls_trust_file /etc/ssl/certs/ca-certificates.crt
  
# set default account to
account default: gmx

One for mailx: 

vim ~/.mailrc

# set smtp for mailx

# (default)
set from="username@gmx.de"
set sendmail="/usr/bin/msmtp"
set message-sendmail-extra-arguments="-a gmx"

account gmx {
set from="username@gmx.de (Your name) send from command line"
set sendmail="/usr/bin/msmtp"
set message-sendmail-extra-arguments="-a gmx"
}

account google {
set from="username@gmail.com (Your Name) send from command line"
set sendmail="/usr/bin/msmtp"
set message-sendmail-extra-arguments="-a google"
}

Send the mail

mailx -A gmx -s "gmx test" username@gmx.de < /tmp/test_email.txt
echo "mail text" | mailx -A gmx -s "gmx test" username@gmx.de
cat /tmp/test_email.txt | mailx -A gmx -s "gmx test" username@gmx.de

Sending an attachment 

echo "mail text" | mailx -A gmx -a attachment.file -s "gmx test" username@gmx.de

Sending dmesg message (or other) via pipe 

dmesg | mailx -A gmx -s "$HOSTNAME dmesg" username@gmx.de

Explanation: 

-A xxx    : the name of the isp account that must match in ~/.msmtprc AND .mailrc.
            If not specified, the as default defined account will be used.
-s "Text" : the subject
-a Datei  : Attachment

Singleton

The singleton pattern is a very handy one. You can realise it in bash like this: 

if ps x | grep -v grep | grep -v $$ | grep $0 | grep -v subl | grep -v vi
then
        echo "$0 already running. Exiting"
        exit 1
else

################################################
# PUT YOUR CODE HERE
# vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv

./run_me

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
################################################ 
 
fi

Grep

Grep a steam with tail, head

Sometimes you need not to filter a standard out only once, but with tail 

tail -f -n 300 /var/log/syslog | stdbuf -o0 grep PATTERN

remove empty lines

cat some_text |  egrep -v '^#|^$'

remove carriage return (newline

echo "something" | tr -d '\n'

Graphical Tools

Sometimes you need to get out of yor script, give a message to your users - visually.

notify-send

See Ubuntu_Desktop#Notify_OSD 

#!/bin/bash
 
notify_start() {

    aplay Critical_Error.wav
   
    notify-send   "Backup is is mounted." \
                -i /usr/share/icons/gnome/48x48/actions/document-open-recent.png \
                "Read and write support for you is working."
}

notify_end() {

    notify-send   "Backup Drive is not ready!" \
                -i /usr/share/icons/gnome/48x48/actions/stock-delete.png \
                "Resolve the problem and save your work"
}


while(true); do

	if touch /backup/testfile
	then
		rm /backup/testfile
		notify_start
	else
		notify_end
	fi

sleep 10

done

Monitoring

Sometimes you like to run a monitor in the background. Here an example for checking if a drive had an drive error. 

#!/bin/bash

if ps x | grep -v grep | grep -v $$ | grep $0 | grep -v subl | grep -v vi
then
        echo "$0 already running. Exiting"
        exit 1
else

################################################
# PUT YOUR CODE HERE
# vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv

 notify_start() {

     aplay /local/share/sounds/KDE_Critical_Error.wav
    
     notify-send   "Backup is is mounted." \
                 -i /usr/share/icons/gnome/48x48/actions/document-open-recent.png \
                 "Read and write support for you is working."
 }
 
 notify_end() {
 
     aplay	/local/share/sounds/KDE_Critical_Error.wav

     notify-send   "Backup Drive is not ready!" \
                 -i /usr/share/icons/gnome/48x48/actions/stock-delete.png \
                 "Resolve the problem and save your work"
 }

while(true); do

	message_head="SSD WRITE TEST:"
	message_time="date +%F_%Hh:%Ms:%Nms"
	error="ERROR:"
	
	logger "${message_head} $(${message_time}) starting sync"

	if sync
	then
		logger "${message_head} $(${message_time}) sync finished successfully."
	else
		logger "${message_head} $(${message_time}) ${error} while do sync."
	fi

	sleep 2

 	
	if touch ~/testfile
 	then
		logger "${message_head} $(${message_time}) touched testfile in home successfully."

 		if rm ~/testfile 
		then 
			logger "${message_head} $(${message_time}) removed testfile in home successfully."
		else
			logger "${message_head} $(${message_time}) ${error} while removing tesfile in home."
		fi

 	else
		logger "${message_head} $(${message_time}) ${error} while touching tesfile in home."
 		
		notify_end
 	fi
 
sleep 28
 

done

# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
################################################ 
 
fi


sed

Search and replace text in a document with sed

 # Search and replace text in a document globally or first
 # apos@gmx.de - 2008
 #
 # param1: search string
 # param2: replace string
 # param3: the document URL
 search_replace_global(){
 
	search="$(printf "%s\n" "${1}" | sed 's/[][\.*^$/]/\\&/g')"
	replace="$(printf "%s\n" "${2}" | sed 's/[][\.*^$/]/\\&/g')"
	sed -i "s/${search}/${replace}/g" "${3}"
  
 }
   
 search_replace_first(){
   
	search="$(printf "%s\n" "${1}" | sed 's/[][\.*^$/]/\\&/g')"
	replace="$(printf "%s\n" "${2}" | sed 's/[][\.*^$/]/\\&/g')"
	sed -i "s/${search}/${replace}/" "${3}"
  
 }

Cut text out of a file

# Cut text out of a file.
# Searches for the first (!) - and only the first - occurence of the Expression
# "FIRST_TEXT_FRAGMENT" and then, from that line on to the first occurence of 
# "LAST_TEXT_FRAGMENT". It then cuts the lines between these two - 
# including the found lines (!) - out with sed. 
#
# param1: first text fragment
# param2: second text fragment
# param3: input file - the file to inspect
cut_text(){

	THEDOC="${3}"
	#FIRST_TEXT_FRAGMENT="$(printf "%s\n" "${1}" | sed 's/[][\.*^$/]/\\&/g')"
	FIRST_TEXT_FRAGMENT="${1}"
	#LAST_TEXT_FRAGMENT="$(printf "%s\n" "${2}" | sed 's/[][\.*^$/]/\\&/g')"
	LAST_TEXT_FRAGMENT="${2}"

	BEGIN_CUT=$(cat ${THEDOC} | fgrep -n --max-count=1 "${FIRST_TEXT_FRAGMENT}" | cut -d':' -f1)
	BEGIN_CUT=$(expr ${BEGIN_CUT})
	echo ${BEGIN_CUT}

	END_CUT=$(cat ${THEDOC} | sed -e "1,${BEGIN_CUT}d" | fgrep -n --max-count=1 "${LAST_TEXT_FRAGMENT}" | cut -d':' -f1)
	echo ${END_CUT}
	END_CUT=$(expr ${END_CUT} + ${BEGIN_CUT})
	echo ${END_CUT}

	sed -i "${BEGIN_CUT},${END_CUT}d" "${THEDOC}"

}

Read a file by line and manipulate content

# get the linecount of the file
lines=$(wc -l myFile | awk '{print $1}')

# process the file line by line
for l in $line
   do head -n $l Blink.ino | sed -e 's/delay/XXXXX/g'; 
done

Copy director tree structure without content

cd /path/to/top
find . -type d -depth | cpio -pvdma /path/to


Renaming files with bad content

find . -depth -name "*.*" -execdir rename 's/:/_/g;s/:/_/g;s/</_/g;s/>/_/g;' "{}" \;

also see 

sudo apt-get install detox
detox --help

Finding duplicate files

apt-get install fdupes

fdupes -r -d -S .

-> interactively delete files.

Find and delete .cache

This cleans up the .cache directory and deletes all files that are older than 14 days AND if the files are greater than 30MB: 

sudo vim /etc/crontab

@daily    find /home/YOUR_USERNAME/.cache/ -atime +14 -size 30M -delete

See: http://askubuntu.com/questions/176480/limit-the-size-of-a-directory-by-deleting-old-files

Kill all processes with a certain name

vim ~/bin/killall_ 

#!/bin/bash
PROGRAM="${1}"

if [ ${PROGRAM} = "" ] 
then
	echo "Programname required."
	exit 1
fi

KILL="$(ps x | grep -v grep | grep -v killall | grep -v $(basename $0) | grep "${PROGRAM}" | sed 's/^\ //' | cut -d' ' -f1 | sed 's/ //g')"

if [ "${KILL}" != "" ]
then
	for killme in "${KILL}"
	do
		echo "Killing <9> process number $killme"
		kill -9 $killme
	done
else
	echo "Nothing to kill ..."
fi

ia32-libs

Older Ubuntu versions: 

sudo apt-get install ia32-libs

Newer (> 14.04 ) Ubuntu versions: 

sudo apt-get install lib32z1 lib32ncurses5 lib32bz2-1.0


Troubleshooting

I am refferring to this excellent guide:

Retrieved from "https://wiki.blue-it.org/index.php?title=Bash&oldid=6399"