rsync volume to volume with speed in consideration

rsync -axHAWXS --numeric-ids --info=progress2 /source/ /destination/

What the options do:
-a : all files, with permissions, etc..
-v : verbose, mention files
-x : stay on one file system
-H : preserve hard links (not included with -a)
-A : preserve ACLs/permissions (not included with -a)
-X : preserve extended attributes (not included with -a)
-W : avoid calculating deltas/diffs of the files (for speed)
-S : to consider sparse/tiny files

–numerical-ids : avoids the overhead of mapping UID/GID values (more speed)
–info=progress2 : avoid even more overhead by showing overall progress

Block countries, networks and VoIP blacklists combining Netfilter’s iptables, ipset and voipbl.org

Make ipset a service,keeping block lists alive and surviving power cycles:

vi /lib/systemd/system/ipset.service

[Unit]
Description=IPSET VoIP Blacklist
Before=netfilter-persistent.service
ConditionFileNotEmpty=/etc/ipset/voipbl

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/sbin/ipset -exist -file /etc/ipset/voipbl restore
ExecStop=/sbin/ipset -file /etc/ipset/voipbl save

[Install]
WantedBy=multi-user.target
Alias=ipset.service

systemctl daemon-reload
systemctl enable ipset

vi /usr/local/bin/voipbl.sh

#!/bin/bash

exec 5> >(logger -t $0)
BASH_XTRACEFD="5"
PS4='$LINENO: '
set -x

SHELL=/bin/bash
PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin
MAILTO=alerts@email.com

get_voiprbl='http://voipbl.org/update/'
get_arinonly='http://voipbl.org/update/?wn[]=arin'

if [ -e '/etc/ipset/voipbl.txt' ]
then
  echo "/etc/ipset/voipbl.txt file exists."
  echo "Checking timestamp and size..."
  olddt=`stat -c '%y' /etc/ipset/voipbl.txt`
  oldsize=`ls -lh /etc/ipset/voipbl.txt | cut -d" " -f5`
else
  echo "/etc/ipset/voipbl.txt not found."
  echo "Touching file for first run..."
  touch /etc/ipset/voipbl.txt
  olddt=`stat -c '%y' /etc/ipset/voipbl.txt`
  oldsize=`ls -lh /etc/ipset/voipbl.txt | cut -d" " -f5`
fi

if [ -e '/etc/ipset/arinonly.txt' ]
then
  echo "/etc/ipset/arinonly.txt file exists."
  echo "Checking timestamp and size..."
  olddtarin=`stat -c '%y' /etc/ipset/arinonly.txt`
  oldsizearin=`ls -lh /etc/ipset/arinonly.txt | cut -d" " -f5`
else
  echo "/etc/ipset/arinonly.txt not found."
  echo "Touching file for first run..."
  touch '/etc/ipset/arinonly.txt'
  olddtarin=`stat -c '%y' /etc/ipset/arinonly.txt`
  oldsizearin=`ls -lh /etc/ipset/arinonly.txt | cut -d" " -f5`
fi

echo "Downloading VoIPBL GLOBAL IP network shuns."
wget -qO - $get_voiprbl -O /etc/ipset/voipbl.txt
echo "Downloading US/CA ARIN networks only lists."
wget -qO - $get_arinonly -O /etc/ipset/arinonly.txt
echo ""
newdt=`stat -c '%y' /etc/ipset/voipbl.txt`
newdtarin=`stat -c '%y' /etc/ipset/arinonly.txt`
newsize=`ls -lh /etc/ipset/voipbl.txt | cut -d" " -f5`
newsizearin=`ls -lh /etc/ipset/arinonly.txt | cut -d" " -f5`
echo "voipbl.txt file differentials:"
echo "old: $olddt SIZE: $oldsize"
echo "new: $newdt SIZE: $newsize"
echo ""
echo "arinonly.txt file differentials:"
echo "old: $olddtarin SIZE: $oldsizearin"
echo "new: $newdtarin SIZE: $newsizearin"
echo ""
echo "Creating hash lists in memory..."
ipset create -exist tmp_voipbl hash:net
ipset create -exist tmp_arin hash:net
ipset create -exist voipbl hash:net
ipset create -exist arinonly hash:net
echo ""

if [ -e '/etc/ipset/tmp_voipbl' ]; then
  echo "/etc/ipset/tmp_voipbl file exists."
  echo "Preparing hash lists for swaping..."
else
  echo "/etc/ipset/tmp_voipbl not found."
  echo "Touching file for first run..."
  touch '/etc/ipset/tmp_voipbl'
fi

cp "/dev/null" "/etc/ipset/tmp_voipbl"
cp "/dev/null" "/etc/ipset/tmp_arin"
echo ""
echo "Parsing new downloads..."

for voipblist in `tail -n +2 /etc/ipset/voipbl.txt`; do
  echo add tmp_voipbl $voipblist >> /etc/ipset/tmp_voipbl
done

for arin in `tail -n +2 /etc/ipset/arinonly.txt`; do
  echo add tmp_arin $arin >> /etc/ipset/tmp_arin
done

# swap the temp ipsets for the live ones
ipset flush tmp_voipbl
ipset flush tmp_arin
ipset restore < /etc/ipset/tmp_voipbl
ipset restore < /etc/ipset/tmp_arin
ipset flush voipbl
ipset flush arinonly
ipset swap tmp_voipbl voipbl
ipset swap tmp_arin arinonly
echo ""
ipset save tmp_voipbl -f /etc/ipset/tmp_voipbl
ipset save tmp_arin -f /etc/ipset/tmp_arin
ipset destroy tmp_voipbl
ipset destroy tmp_arin
ipset save voipbl -f /etc/ipset/voipbl
ipset save arinonly -f /etc/ipset/arinonly
echo "List inventory in RAM and in use by Netfilter:"
echo ""
echo "`ipset list -t`"

# log the file modification time for use in minimizing lag in cron schedule
moredt=`date`;
logger -p cron.notice "IPSet: voipbl updated as of: $moredt" ;

#Check if rules in iptables
if ! $(/sbin/iptables -w --check INPUT -m set --match-set voipbl src -j LOGNDROP > /dev/null 2>&1); then
  /sbin/iptables -I INPUT 1 -m set --match-set voipbl src -j LOGNDROP
fi

if ! $(/sbin/iptables -w --check INPUT -m set --match-set arinonly src -j DROP > /dev/null 2>&1); then
  /sbin/iptables -I INPUT 1 -m set --match-set arinonly src -j DROP
fi

echo "";
echo "Netfilter IPSet rules updated and reloaded into RAM";
echo "Successful completion...";
echo "Bye Bye...";

crontab -e

0 1 * * * /usr/local/bin/voipbl.sh

Via email:

/etc/ipset/voipbl.txt file exists.
Checking timestamp and size...
/etc/ipset/arinonly.txt file exists.
Checking timestamp and size...
Downloading VoIPBL GLOBAL IP network shuns.
Downloading US/CA ARIN networks only lists.

voipbl.txt file differentials:
old: 2018-11-25 11:00:21.502886736 -0600 SIZE: 868K
new: 2018-11-26 01:00:19.904743514 -0600 SIZE: 879K

arinonly.txt file differentials:
old: 2018-11-25 11:00:30.438886415 -0600 SIZE: 631K
new: 2018-11-26 01:00:21.100743556 -0600 SIZE: 651K

Creating hash lists in memory...

/etc/ipset/tmp_voipbl file exists.
Preparing hash lists for swaping...

Parsing new downloads...

List inventory in RAM and in use by Netfilter:

Name: voipbl
Type: hash:net
Revision: 5
Header: family inet hashsize 32768 maxelem 65536
Size in memory: 1394616
References: 1

Name: arinonly
Type: hash:net
Revision: 5
Header: family inet hashsize 16384 maxelem 65536
Size in memory: 792280
References: 1

Netfilter IPSet rules updated and reloaded into RAM
Successful completion...

OpenWRT R11E-LTE-US

AT E0 V1 

AT+CFUN=1,1 

AT E0 V1 

AT+GMI 

AT+GMM 

AT+GMR 

AT+CGSN 

AT+CFUN=4 

AT+CMEE=1 

AT+CREG=2 

AT+CGREG=2 

AT+CEREG=2 

AT+CGEREP=2,0 

AT+CMGF=0 

AT+CNMI=1,1,0,1,0 

AT+CFUN=1 

AT+CPIN? 

AT+CPMS="SM","SM","SM" 

AT+CFUN? 

AT+CNUM 

AT+CIMI 

AT+CPIN? 

AT+COPS=0 

AT+CGDCONT=1,"IP","vzwinternet" 

AT$QCPDPP=1,0 

AT+CEREG=2 

AT+COPS? 

AT+CFUN? 

AT+CSQ 

AT+COPS? 

AT+CFUN? 

QAT$QCRSRP? 

QAT$QCRSRQ? 

at$ecmcall=1 

at$ecmcall? 

AT+CGCONTRDP=1

WordPress/Linux Bash CLI Notes Engine Facility

I created this process to improve my record keeping while working on a Linux bash shell. It allows for one to take notes while on the command line as well as publish to WordPress for tidy and organized record keeping.

Features:

  • Posts logs to WordPress from any Linux hosts.
  • Log all CLI commands to syslog globally.
  • Record literal notes to syslog.
  • Privately post logs to WordPress via CLI.
  • Posting of Subject from CLI to WP posts.
  • Auto insert [hostname] into WordPress post subject.
  • Auto remove undesired data from WP posts.
  • Auto insert [shortcode] into WordPress posts.

Roadmap:

  • Auto redact sensitive data on CLI WP post actions.
  • Record file changes to syslog (fork tripwire).
  • Post Categories to WordPress posts from CLI.
  • Post Tags to WordPress posts from CLI.

Requirements:

  • A self-hosted WordPress Installation.
  • The free Twenty Fifteen Theme.
  • FOSS Postie Plugin by Wayne Allen.
  • FOSS SyntaxHilighter Evolved by Alex Mills.
  • A pop3 mailbox (Postie to pull & DELETE emails).
  • Logger via the util-linux package.
  • Mailx via the mail-utils package.
  • Rsyslog via rsyslog package.
  • A bit of troysio microcode.

Steps:

  • Install WordPress.
  • Activate Twenty Fifteen Theme.
  • Download/Activate SyntaxHilighter Evolved in plugins.
  • Download/Activate Postie within plugins area.
  • Configure Postie to fetch and posts PRIVATELY text only.
  • Install mail-utils and util-linux packages on all Linux hosts.
  • Create your CLI logging syslog service.
  • Drop the following troysio microcode into e.g. /usr/local/bin/yourio and make it executable.
  • #!/bin/sh
    # Troy Perkins
    # https://troys.io
    # Note: Make sure to remove all "\"es from Shortcode lines before executing. I had to place them in order for this code to be published here due to surrounding bash shortcode in this post.
    
    #Host
    host=`hostname`;
    
    #Pop3 mailbox;
    mailb="your@email.com";
    
    #Content
    file='/var/log/commands.log';
    #file='/var/log/test.log';
    
    #Shortcode
    shrtstart='\[\bash]';
    shrtend='\[\/\bash]';
    
    { echo $shrtstart; cat $file | cut -d':' --complement -s -f1,2,3,4 | sed 's/\ [^\ ]*$//'; echo $shrtend; } | uniq | mailx -s "[$host] $1" $mailb
    

Usage:
To take manual notes type a # sign then your note then hit enter:

root@crm:# # Here is an example of a manual note on the command line.

To send notes to WordPress:

root@crm:# troysio "Here is my subject"

Example Output:

vi /usr/bin/troysio
vi /usr/bin/troysio-redact
troysio "testing new cut"
cat /var/log/commands.log | cut -d':' --complement -s -f1,2,3,4
vi /usr/bin/troysio
vi /usr/bin/troysio-redact
troysio "testing new cut"
troysio uniq
vi /usr/bin/troysio
troysio uniq
troysio | uniq
vi /usr/bin/troysio
vi /usr/bin/troysio-redact
troysio "testing uniq"
vi /usr/bin/troysio
troysio "testing uniq"
cat -A /var/log/commands.log
cp /usr/bin/troysio /usr/bin/troysio-test
vi /usr/bin/troysio-test
troysio-test | cat -A
troysio-test | sub(/\r$/,"") | cat -A
troysio-test | cat -A
# Here is an example of a manual note on the command line.
# Hope you enjoy your note taking helper.
# Cheers.
troysio "here is my subject"
# WordPress post made with above command with title [crm] < host and "here is my subject" on the title of post.
# Joy

To view the actual post, which I’ve changed from Private to Public just for you go here: https://troys.io/crm-here-is-my-subject/

Relocate your WP website

Move your WordPress website anywhere in 10 minutes via an SSH shell with access to your website files directory and database.

Current URL: example.com/wordpress
Where to: example.com

Or, complete URL change, it doesn’t matter using this method.

Export your WordPress database.

mysqldump -u loginID -p example.com_wp > example.com_wp.sql

Now bust out a little sed.

cat example.com_wp.sql | s/example.com\/wordpress/example.com/g > new.sql

Now import your db sed edits.

mysql -u loginID -p example.com_wp < new.sql

Now move your directory files.

cd /var/www/example.com/wordpress/
tar cvf all.tar *
mv all.tar ../
cd ../
tar xvf all.tar
rm ./wordpress
rm all.tar

Done. Visit example.com

Send CLI commands to syslog

Create a CLI command syslog service

echo "local6.*    /var/log/commands.log" >> /etc/rsyslog.d/bash.conf
systemctl restart rsyslog.service

Set global .bashrc shells to log

vi /etc/bash.bashrc

# log all bash shell sessions globally
export PROMPT_COMMAND='RETRN_VAL=$?;logger -p local6.debug "$(whoami) [$$]: $(history 1 | sed "s/^[ ]*[0-9]\+[ ]*//" ) [$RETRN_VAL]"'

To test new configuration:

  • Either log out and log back into your shell.
  • Or source bashrc with the command
    source ~/.bashrc
  • Or start a new shell with the command
    bash

Type a few commands… then

tail /var/log/commands.log

Flash OpenWRT to RouterBoard

# No dhcpd or tftpd, no problem. Load OpenWRT firmware on Mikrotik RouterBoards the easy way using dnsmasq. Open two bash shell windows. In the 1st window execute the following and create your loader file:

# FIRST BASH SHELL WINDOW

mkdir -p /var/lib/tftpboot
vi /var/lib/tftpboot/loader.sh

# Create your loader.sh file

#/bin/bash
ifconfig enp6s0 192.168.1.10 up
dnsmasq -i enp6s0 --dhcp-range=192.168.1.100,192.168.1.200 \
--dhcp-boot=openwrt-18.06.1-<your supported routerboard-initramfs>-kernel.bin \
--enable-tftp --tftp-root=/var/lib/tftpboot/ -d -u root -p0 -K --log-dhcp --bootp-dynamic

# Make it executable.

chmod 755 /var/lib/tftpboot/loader.sh

# Ensure both your desired initram AND upgrade image files (2 files) have been moved to the above tftpboot directory. If you forget the corresponding upgrade file, on reboot you’ll find RouterOS is back. Once you’re interfacing with OpenWRT, make sure to manually execute the upgrade in order to purge the RouterOS. It will ensure your OpenWRT kernel sticks and survives power cycles.

# Unplug your ethernet cable that connects the Linux host you are currently on from your current internet router. Then connect this loose end into the WAN/Internet port on your MikroTik Routerboard.

# In your second bash window, execute the following commands: First we determine what the main interface name is that’s using your ethernet connection (in my case its enp6s0). Then we create a virtual interface assigning it an IP address that resides on the 192.168.88.0/24 network.

# SECOND BASH SHELL WINDOW

ifconfig | grep Ethernet | cut -d" " -f1
#enp6s0  <-- e.g. output. You may have more than one listed, its usually the top.
ifconfig enp6s0:0 192.168.88.2 up

# Now telnet to 192.168.88.1 and login (u:admin p:) assuming this is a new Routerboard. Once logged in execute the following two commands (do not close this window.)

/system routerboard settings set boot-device=try-ethernet-once-then-nand
/system routerboard settings set boot-protocol=dhcp

# FIRST BASH SHELL WINDOW

cd /var/lib/tftpboot/
./loader.sh

# Now quickly Unplug the power source from your Routerboard and plug it back in. You should soon see in your FIRST BASH WINDOW dhcp and a tftp directives and the sending of your OpenWRT image to the Routerboard. Give it some time to complete. A sign of completion is you will see OpenWRT dhcp messages in your FIRST BASH WINDOW.

# Once the first flashing is complete, unplug your ethernet cable from the Routerboard WAN port and plug it into its first LAN port. Depending on which Routerboard you have it may vary. You should now be able use a browser and log into OpenWRT via ip address 192.168.1.1. NOTE! you’re NOT FINISHED! Log into OpenWRT and navigate to System> Backup / Flash Firmware. In the Flash new firmware image section click choose file and load the OpenWRT sysupgrade image you stored in /var/lib/tftpboot then click Flash Image. Now your new OpenWRT router will survive reboots.

# Lastly decide to delete, or keep but disable and lock loader.sh until its needed again.

chmod 000 /var/lib/tftpboot/loader.sh
chattr +i /var/lib/tftpboot/loader.sh

# Visual confirmation loader.sh is locked down using lsattr.

# Sample results:
root@host:/var/lib/tftpboot# lsattr
-------------e-- ./openwrt-18.06.1-ramips-mt7621-mikrotik_rbm33g-initramfs-kernel.bin
-------------e-- ./openwrt-18.06.1-ar71xx-mikrotik-rb-nor-flash-16M-squashfs-sysupgrade.bin
-------------e-- ./openwrt-18.06.1-ar71xx-mikrotik-rb-nor-flash-16M-initramfs-kernel.bin
----i--------e-- ./loader.sh
-------------e-- ./openwrt-18.06.1-ramips-mt7621-mikrotik_rbm33g-squashfs-sysupgrade.bin