Jump to content

Linux Bash Shell Scripting

From Torben's Wiki

Bash commands

rm -rf dir   # do not show warning if not existing
mkdir -p dir # do not print warning if already existing AND make needed parents

Output

View File 

less File

Print last line of file 

tail -1 File

Print last line, column 2 of file (cut expects tab separating) 

tail -1 File | cut -f2

Follow changes in (log)file 

tail -f File

Print file contents to std output 

cat File

Filter contents of file to std output 

grep "ERROR" File

Filter contents of file to std output (case-insensitive) 

grep -i "error" File

Filter contents of file to std output, +2 lines before +1 following lines each 

grep -B2 -A1 "ERROR" File

Print file contents to std output, starting at key word 

grep -A9999999 "2016-04-19T10:38" File

Save program output (script/grep/cat...) into file 

sh script.sh > log.txt      # std output, (errors to terminal)
sh script.sh 1> log.txt 2> log-errors.txt  # separate files
sh script.sh &> log.txt     # std and errors both to 1 file

Send output to file and terminal 

touch log.txt & tail -f log.txt & sh script.sh > log.txt

Piping 

sh script.sh | less

Filtering of output 

sh script.sh | grep tmenke

Search file contents in dir recursively

From [1] 

grep -rn '/path/to/somewhere/' -e 'pattern'

Input

Enter a value

echo "Enter to continue, CTRL+C to cancel"
read ok

echo -n "Was? "
read var
echo "$var"

Dialog Input

dialog --yesno "Cleanup Old Backups First" 0 0
CleanUpBackups=$?
# yes -> 0 # no -> 1
if [ $CleanUpBackups -eq 0 ] # spaces needed!!!
  then
  perl /save/_save/backupPlatteAufraemen/loeschen2.pl
fi


Loops

Zip all subdirs 

for D in `ls -d */`
do
    # remove tailing /
    D=${D%/}
    echo === $D ====
    cd $D
    zip -rq9 ~/sicher/html/$D.zip .
    # tar cfz ~/sicher/html/$D.tgz .
    cd ..
done

Check if variable in list 

if DukeTypem2D/)$ ; then
    echo skipping $D

Good overview of parameters for if [2]
If 

if [ -d "backup.0" ]; then
  echo "gibts"
else 
  echo "gibts nicht"
fi

one-liner: 

 if [ -d "backup.0" ] ; then sudo mv backup.0 backup.1 ; fi

Check if process is running 

if [ ! "$(pidof workrave)" ] ; then
  nohup workrave
fi

While
Normal loop 

i=1
while [ $i -le 4 ] do
(( i++ ))
done

Endless loop 

while true; do
    # ...
    echo "Enter to continue, CTRL+C to cancel"
    read user_input
done

For loop 

for FILE in *.txt; 
  do 
  mv "$FILE" `echo $FILE | tr ' ' '_'`
  done

Check if files matching *.xml exist, store Number as string and print it 

NUM=$(find $TRANSPORT/*.xml -type f 2> /dev/null |wc -l)
echo $NUM
if [ $NUM != "0" ] ; then
echo "$NUM files found, copying" 
else 
echo "No files found, exiting" 
exit 0
fi

Loop over contents / columns of file

file contains tab separated columns $1 -> column1 etc. 

while read line ; do
    set $line
	myCommit=$1
	myDate=$2
    echo "$myCommit ; $myDate"
done <"history-change.tsv"

Exit upon Error

set -e
touch /tmp/not_found.txt
echo will not reach here

Parameters

from [3] 

OPTIONS=$(getopt -o f:t -l file:,titlepage -- "$@")
if [ $? -ne 0 ]; then
  echo "getopt error"
  exit 1
fi
eval set -- $OPTIONS
while true; do
  case "$1" in
    -f|--file) FILE="$2" ; shift ;;
    -t|--titlepage) TITLEPAGE=1 ;;
    --)        shift ; break ;;
    *)         echo "unknown option: $1" ; exit 1 ;;
  esac
  shift
done
if [ $# -ne 0 ]; then
  echo "unknown option(s): $@"
  exit 1
fi

if [ ! $FILE ]; then
  echo "no file given as parameter"
  exit 1
fi
if [ ! -f $FILE ]; then
  echo "file '$FILE' not found"
  exit 1
fi
# echo "titlepage: $TITLEPAGE"
# echo "file: $FILE"
# exit 1

Combine find and some other command e.g chmod

# chmod for dirs
find -type d -exec chmod g+xs {} \;

# make scripts excecuteable
find . -name "*.sh" -exec chmod 744 {} \;

# remove .zip files older 30 days
find ./R*/logs/*.zip -mtime +30 -exec rm {} \;

# Copy empty Directory Structure
mkdir /Zielverzeichnis 
cd Quellverzeichnis
find . -type d -depth | cpio -pvdma /Zielverzeichnis/

sed

If you want to replace the text 'asdf' by 'qwert' in all '*.txt' files in a directory at once, use the following command: 

sed -i -e 's/asdf/qwert/g' *.txt

add a line to file 

sed -i '8i\\\input{layout/hp-format}\n\\input{layout/hp-markup}\n' $target_file

delete/"grep out" lines 

sed -i '/\\\input{layout\/hp-contents}/d' $target_file

insert file into file at marker 

sed -i -e '/<style/r ebook/epub.css' $target_file

# -z changes the delimiter to null characters (\0) to allow for multiline matching
sed -i -z 's/<\/header>.*< p >HARRY POTTER<\/p>/<\/header>\n< p >HARRY POTTER<\/p>/' $target_file

get filename dirname etc

cd to parent dir 

script_dir=$(dirname $0)
cd $script_dir/..

dirname file
basename file

myPath=data/de-districts/de-district_timeseries-02000.tsv
myDir=$(dirname -- "$myPath")
myFileName=$(basename -- "$myPath")
myFileExt="${myFileName##*.}"
myFileName="${myFileName%.*}"

filename without extension 

FILEOLD="${FILE%.pdf}-alt.pdf"

for file in *
do
  if [ -f $file ] ; then
    name=${file%\.*} # name without extension
    echo ${name}
  fi ;
done

name=`basename /path/filename.ext .ext` # -> filename
for i in *.jpeg; do mv "$i" "`basename "$i" .jpeg`.jpg"; done

Path of current working dir 

dir=$PWD

Name of current directory without path 

dirname=${PWD##*/}

Location of this script 

dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

Get Date as String

DATE=`date +"%y-%m-%d_%H-%M"`
echo $DATE

Count Files in Folder

find . | wc -l

Check free inodes on partition 

df -i

Tar

get rid of message "removing leading /": use option P

Pause

read -p "Press any key to start backup..."

fstab mount discs harddrive partitions

read UUID 

sudo blkid
sudo blkid /dev/sda3

UUID=D4AE21F4AE21D032 /media/win ntfs auto,user,uid=torben,gid=torben,fmask=0177,dmask=0077,ro 0 0

UUID=f103902f-bbd6-49af-b983-b1dddedd361a /media/linux ext3 auto,user,rw 0 0

UUID=EA06-C29C /media/files vfat auto,user,utf8,uid=torben,gid=torben,fmask=0177,dmask=0077,rw 0 0

Network Tests

ping 10.10.10.10 -c 1
netcat / ncat / nc 10.10.10.10 8080
telnet 10.10.10.10 8080

netcat

Netcat is pretty cool, as it can be used to send messages between two machines, or even set files, see [4] 

# host1: start to listen on a port 
netcat -l <PORT>
# host2: connect to host1
netcat <IP-host1> <PORT>
#  now messages can be send in both ways

Transfer file 

# host1: start to listen on a port 
netcat -l <PORT> > received_file
# host2: send a file
netcat <IP-host1> <PORT> < original_file

wget

wget --no-check-certificate https://192.168.0.123:8080
wget --no-check-certificate --user=myUser --password=myPasswd https://10.10.10.10/url.xml
# DynDNS
wget -d -v "http://192.168.0.123:8080/nic/update?hostname=myhost.10.10.1.100&myip=10.10.10.12&localip=10.10.10.12" --user="u123456" --password="p123456" -O output.xml
wget --ca-certificate=MyCert.pem --user=myUser --password=myPwd https://myUrl:80/file.xml

curl

curl --trace /dev/stdout \
 --cacert MyCert.pem \
 https://myUrl:80/file.xml \
 -u myUser:myPwd \
 --header "Content-Type: text/xml;charset=UTF-8" \
 --header "SOAPAction: http://MySoapActionURL" \
 --header "Accept-Encoding: gzip, deflate" \
 --data '<MySoapData>'

Access Strava API and retrieve JSON format 

curl -X GET "https://www.strava.com/api/v3/athlete?access_token=1234567890" -H  "accept: application/json"

Examples

Search for a word in some files (here *.txt) 

if [ -z $1 ]; then
  echo -n "Was? "
  read was
else 
  was=$1
  echo "$was"
fi
grep --color=auto -i $was *.txt

Crontab

# every 5 minutes
*/5     *       *       *       *       script1.sh
# every 5 minutes + 1
1-56/5  *       *       *       *       script2.sh

systemctl Services

# create service
sudo vim /etc/systemd/system/mqtt_influx.service

[Unit]
Description=My MQTT to InfluxDB Service

[Service]
ExecStart=/usr/bin/env python3.9 /home/pi/influx-collectors/mqtt.py
WorkingDirectory=/home/pi/influx-collectors
Restart=always
User=pi
Group=pi
Nice=10

[Install]
WantedBy=multi-user.target

# activate and start service
sudo systemctl daemon-reload
sudo systemctl enable mqtt_influx.service
sudo systemctl start mqtt_influx.service
sudo systemctl restart mqtt_influx.service
sudo systemctl stop mqtt_influx.service
sudo systemctl status mqtt_influx.service
# access log (-f = follow)
sudo journalctl -u mqtt_influx.service -f

Clean Harddisk by filling with Zeros

Caution: double check which device you are cleaning... 

sudo dd if=/dev/zero of=/dev/sdd1 bs=1M

Screen

# liste anzeigen
screen -list
# neuen starten
screen -R name
# screen in Hintergrund
STRG+a d -> detact screen

Screen Start Skript in new Screen 

screen -A -m -d -S myScrName ./start.sh

kill Screen Session 

STRG+d

kill Background Screen Session without entering 

screen -X -S myScrName quit

PID store and kill

Set the name of the process 

bash -c "exec -a <MyName> <Command>"

kill the process by name 

pkill -f MyName

get and store PID 

/home/user/bin/script.sh & echo $! > script.pid

kill process using PID 

kill $(cat script.pid)

Check if running as root/sudo

if $EUID -ne 0 ; then
  echo "E: run via sudo!!!" 2>&1
  exit 1
fi

check for username 

if [ `whoami` != 'torben' ]
  then
    echo "You must be torben to do this."
    exit
fi

Add Group and User and Set Password

groupadd testuser
useradd testuser -g testuser
echo -e "testpass\ntestpass" | (passwd --stdin testuser)

Monitoring open files of a process

## check system limit
# ulimit -n 
DATE=`date +"%y-%m-%d_%H-%M"`
ID=`pgrep -U username -f ProcessName`
FILES=`ls -1 /proc/$ID/fd/ | wc -l` 
echo "$DATE  $FILES  ">> /home/user/logfile.log
## check process limit
# cat /proc/$ID/limits

Backup Script for folder using tar and filters

#!/bin/bash
initialdir=$PWD
DATE=`date +"%Y-%m-%d_%H-%M"`

# cd to "path of this file /.."
dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd $dir/..
# name of current folder = name to use for backup
name=${PWD##*/}
backupSource=$PWD

cd $backupSource/../backup/
zipDir=$(pwd)
zipTarget=$zipDir/$name-$DATE.tgz

cd $backupSource
tar cfvz $zipTarget ./ --exclude='logs/*' --exclude='import/*'

cd $initialdir

Zip dir including hidden files

zip -r9q myfile.zip . -x '.git/*'

Start Stop Script for Java Application

#!/bin/bash
# Start and Stop Linux Service for Java application
# uses java paramerter -Dname=$PROGRAM
#      pgrep -f $PROGRAM 
#      pkill -f $PROGRAM
# with $PROGRAM = Name of directory above location of this script: 
# /home/torben/Tool/bin/startscript.sh -> Tool
# STDOUT and STDERR are written into ./logs/$PROGRAM-STDOUT.log

# cd to "path of this file /.." to get dirname to use as service name
dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd $dir/..
dirname=${PWD##*/}
# Attention, name has to be unique on the system -> added -Service !!!
PROGRAM=$dirname-Service

# ensure that this script can only be started as user torben and not accidently as root
username=torben 

jar="./MyJavaApplication.jar"
log_conf="./config/log4j2.xml"
app_params="-conf ./config/config.xml"

vm_params=" \
-Dname=$PROGRAM \
-Dlog4j.configurationFile=$log_conf \
-DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector \
"
server_params=" \
-server -Xms512m -Xmx2048m \
-XX:+UseParallelGC \
-XX:+AggressiveOpts \
-XX:+UseFastAccessorMethods \
-XX:-OmitStackTraceInFastThrow \
-XX:+HeapDumpOnOutOfMemoryError \
-XX:HeapDumpPath=./logs/rss-heap-$(date +%Y%m%d_%H%M%S).hprof \
"
# for remote jmx access (disabled by default)
jmx_params=" \
-Djava.rmi.server.hostname=172.19.13.200 \
-Dcom.sun.management.jmxremote.port=1100 \
-Dcom.sun.management.jmxremote.local.only=false \
-Dcom.sun.management.jmxremote.authenticate=false \
-Dcom.sun.management.jmxremote.ssl=false \
"
JAVAPROGRAM="\
$vm_params \
-jar $jar $app_params \
$server_params \
"

# for remote jmx access add
# $jmx_params 

# for problems with IPv6 add (i.e. if Startup takes more than 10 minute until first log entry)
#-Djava.net.preferIPv4Stack=true
#-Djava.net.preferIPv6Addresses=false

# for Java debugging:
# -Xverbose:codegen 


# do not modify below here #

# Check user
if [ `whoami` != $username ]
  then
    echo "ERROR: Only user $username is allowed to modify this service"
    exit
fi

# Functions
check() {
   pgrep -U $username -f $PROGRAM >/dev/null
}
kill() {
   pkill -U $username -f $PROGRAM
}
stop() {
   check
   if [ $? -eq 0 ]; then
      kill >/dev/null
      # Make sure it is stopped before returning
      until [ $? -ne 0 ]; do
         sleep 1
         check
      done
      check
      if [ $? -eq 0 ]; then
         echo "ERROR stopping application $PROGRAM"
         exit 1
      else
         echo "Application $PROGRAM stopped"
         #rm -f /var/lock/subsys/[init script name] <--- Perhaps, this lock file has to be removed, too.
      fi
   else
      echo "Application $PROGRAM is not running"
   fi
}
start() {
   check
   if [ $? -eq 0 ]; then
      echo "Application $PROGRAM is already running"
      exit 0
   fi
   export LC_ALL="de_DE@euro";
   export LANG=german;
   nohup java $JAVAPROGRAM 0<&- &>./logs/$PROGRAM-STDOUT.log &
   echo "Application $PROGRAM started"
}
restart() {
   stop
   start
}
status() {
   check
   if [ $? -eq 0 ]; then
      echo "Application $PROGRAM is running"
   else
      echo "Application $PROGRAM is stopped"
	  exit 1
   fi
}
#Actions
case "$1" in
start)
   start
;;
stop)
   stop
;;
restart)
   restart
;;
status)
   status
;;
*)
   echo "Usage: $0 {start|stop|restart|status}"
   exit 1
esac
exit 0
Retrieved from "https://entorb.net//wiki/index.php?title=Linux_Bash_Shell_Scripting&oldid=5386"