Menu

Show posts

This section allows you to view all posts made by this member. Note that you can only see posts made in areas you currently have access to.

Show posts Menu

Messages - Itow

#1
Hello there  :)

yesterday i got following error:

[1580222648] unbound-checkconf[27733:0] error: cannot parse name queda212..duckdns.org
[1580222648] unbound-checkconf[27733:0] error: bad zone name queda212..duckdns.org static
[1580222648] unbound-checkconf[27733:0] fatal error: failed local-zone, local-data configuration


So i update the script to replace more then one dot with a single dot.


#!/usr/local/bin/bash

oldtmp="$(find /tmp -type f -name 'tmp.*' | wc -l)"
        if [ $oldtmp -gt 0 ]; then
        echo Found $oldtmp old tmp-files.
        echo Delete the old Files
        find /tmp -type f -name 'tmp.*' -exec rm -f {} \;
        fi

#Erstelle Temp Datein
tmp1="$(mktemp)"
tmp2="$(mktemp)"
tmp3="$(mktemp)"
tmp4="$(mktemp)"
tmp5="$(mktemp)"
tmp6="$(mktemp)"
tmp7="$(mktemp)"
file="/var/unbound/adblocklist.conf"
filebackup="/var/unbound/adblocklist.bck"
whitelist="/root/whitelist"
blacklist="/root/blacklist"
actionfile="/usr/local/opnsense/service/conf/actions.d/actions_AdBlock.conf"

# Backup der alten Blocklist
if [ -f $file ]; then
        mv $file $filebackup
else
        touch $filebackup
        echo '# Empty File' > $filebackup
fi

# Pruefe Custom White and Blacklist
if ! [ -f $whitelist ]; then
        touch $whitelist
        echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nicht loeschen!' > $whitelist
fi

if ! [ -f $blacklist ]; then
        touch $blacklist
        echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nicht loeschen!' > $blacklist
fi

# Pruefe und Erstelle Actionfile fuer Cron
if ! [ -f $actionfile ]; then
        touch $actionfile
        printf "[reload]\ncommand:/root/adblockscript\nparameter:\ntype:script\nmessage:Update AdBlocklist and load them\ndescription:Update AdBlocklist and load them" >> $actionfile
        service configd restart
        echo 'Now you can configure the Cron Job via Web Interface'
fi
# Download Blocklist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/ultimate/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/xtreme/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/regional/formats/domains.txt; \
        cat $blacklist; \
} > $tmp1

# Download Whitelist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/unblock/master/basic/formats/domains.txt; \
        cat $whitelist; \
} > $tmp4

# Saeubere die Blocklisten

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp1
cat $tmp1 | sed -r 's/\.+/\./' | tr -d '\r' >> $tmp2
sed -i '' -e 's/ *$//' $tmp2 && sort -uf $tmp2 |tee |uniq -i > $tmp3
sed -i '' -e '/^$/d' $tmp3
rm $tmp1 $tmp2

# Saeubere die Whitelist

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp4
cat $tmp4 | sed -r 's/\.+/\./' | tr -d '\r' >> $tmp5
sed -i '' -e 's/ *$//' $tmp5 && sort -uf $tmp5 |tee |uniq -i > $tmp6
sed -i '' -e '/^$/d' $tmp6
rm $tmp4 $tmp5

# Fuege Blocklist und Whitelist zusammen

comm -23 $tmp3 $tmp6 > $tmp7
rm $tmp3 $tmp6
sed -i '' -e '/^$/d' $tmp7
awk '$0="local-zone: \""$0"\" static"' $tmp7 > $file
rm $tmp7

# Check Unbound Config

if ! unbound-checkconf /var/unbound/unbound.conf; then
        rm $file
        echo Nutze alte Blockliste
        mv $filebackup $file
        exit 1
else
#       echo Config ist Okey
        if ! grep -cq "include: $file" /var/unbound/unbound.conf; then
        echo Please add "include: $file" to your custom Unbound settings.
        fi
        rm $filebackup
fi
if [ "$1" == info ]; then
        domains=$(awk '!/^#/ && !/^$/{c++}END{print c}' $file | awk '{ len=length($0); res=""; for (i=0;i<=len;i++) { res=substr($0,len-i+1,1) res; if (i > 0 && i < len && i % 3 == 0) { res = "," res } }; print res }')
        echo "Anzahl zu Blockender Domains = $domains"
fi
# Aendere Benutzer und Gruppe

chown unbound:unbound $file

# Starte Unbound neu
pluginctl dns
exit 0


Thanks Itow

--Edit 30.01.20 
fix Typos and few Issues add new checks
#2
Quote from: eprom on January 23, 2020, 04:51:41 PM
if you do not add server: to the start of the first line of .conf file unbound will not start.

I only could start Unbound with that:
example:

server:local-zone: "0--ass-cinema-newsp.da.ru" static
local-zone: "0--bondage.dk" static
local-zone: "0--fightingshaving.da.ru" static
local-zone: "0--foodwarez.da.ru" static


Cheers and thanks for your work and scripts, Working like a charm.

Hi,

usually it is not nethethery to add anything to the created Blocklist.

Please check if you have add include: /var/unbound/adblocklist.conf in your Unbound configuration under Custom options in the Web interface. And try remove any addition to the custom options to check if Unbound will function normally without it.

-
I have added a check if the crucial line exist in the unbound config.
Also added checks if Custom Whitelist and Blacklist exist.


#!/usr/local/bin/bash

#Erstelle Temp Datein

tmp1="$(mktemp)"
tmp2="$(mktemp)"
tmp3="$(mktemp)"
tmp4="$(mktemp)"
tmp5="$(mktemp)"
tmp6="$(mktemp)"
tmp7="$(mktemp)"
file="/var/unbound/adblocklist.conf"
filebackup="/var/unbound/adblocklist.bck"
whitelist="/root/whitelist"
blacklist="/root/blacklist"

# Backup der alten Blocklist
if [ -f $file ]; then
        mv $file $filebackup
fi

if ! [ -f $whitelist ]; then
        touch $whitelist
        echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nichr loeschen!' > $whitelist
fi

if ! [ -f $blacklist ]; then
        touch $blacklist
        echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nichr loeschen!' > $blacklist
fi

# Pruefe Custom White and Blacklist

# Download Blocklist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/ultimate/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/xtreme/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/regional/formats/domains.txt; \
        cat $blacklist; \
} > $tmp1

# Download Whitelist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/unblock/master/basic/formats/domains.txt; \
        cat $whitelist; \
} > $tmp4

# Saeubere die Blocklisten

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp1
cat $tmp1 | tr -d '\r' >> $tmp2
sed -i '' -e 's/ *$//' $tmp2 && sort -uf $tmp2 |tee |uniq -i > $tmp3
sed -i '' -e '/^$/d' $tmp3
rm $tmp1 $tmp2

# Saeubere die Whitelist

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp4
cat $tmp4 | tr -d '\r' >> $tmp5
sed -i '' -e 's/ *$//' $tmp5 && sort -uf $tmp5 |tee |uniq -i > $tmp6
sed -i '' -e '/^$/d' $tmp6
rm $tmp4 $tmp5

# Fuege Blocklist und Whitelist zusammen

comm -23 $tmp3 $tmp6 > $tmp7
rm $tmp3 $tmp6
sed -i '' -e '/^$/d' $tmp7
awk '$0="local-zone: \""$0"\" static"' $tmp7 > $file

# Check Unbound Config

if ! unbound-checkconf /var/unbound/unbound.conf; then
        rm $file
        echo Nutze alte Blockliste
        mv $filebackup $file
        exit 1
else
#       echo Config ist Okey
        if ! grep -cq "include: $file" /var/unbound/unbound.conf; then
        echo Please add "include: $file" to your custom Unbound settings.
        rm $filebackup
        fi
fi
if [ "$1" == info ]; then
        domains=$(awk '!/^#/ && !/^$/{c++}END{print c}' $file | awk '{ len=length($0); res=""; for (i=0;i<=len;i++) { res=substr($0,len-i+1,1) res; if (i > 0 && i < len && i % 3 == 0) { res = "," res } }; print res }')
        echo "Anzahl zu Blockender Domains = $domains"
fi
# Aendere Benutzer und Gruppe

chown unbound:unbound $file


# Starte Unbound neu
pluginctl dns


Thanks Itow
#3
Quote from: mihak on January 18, 2020, 09:15:04 PM
Sinnce https://energized.pro started to publish Unbound-readable blocklists, we should simplify our tutorial to something like this:

1. add include: /var/unbound/ad-blacklist.conf into Custom options of Unbound
2. Create Ad-blacklist-refresh.sh in /var/unbound with:
curl https://raw.githubusercontent.com/EnergizedProtection/block/master/blu/formats/unbound.conf -o /var/unbound/ad-blacklist.conf

3. insert a regular execution of this script into crontab

There are multiple levels of compiled lists by energized.pro team - just pick the right strength, let Unbound use it and enjoy ad-free browsing.

Hello there  ;D

yes your are right you can do that but using a script has also his benefits.


  • you can combine more then one list
  • you can use a whitlist and a blocklist created by your own
  • you are fail safe if the original list has character like ( / \ ) see: https://github.com/EnergizedProtection/block/issues/326
  • you can optimize the list for Unbound while create a case insensitive list this will shrink the size
and two personal reason
  • i dont like to mess with the crontable im not internally shure but think it is possible that the crontable will be overwrite at update/upgrade
  • maybe you learn a bit in writing scripts

If you like it realy easy and wont do anything it is better use a addon for Opnsense like: https://forum.opnsense.org/index.php?topic=14116.0

Thanks Itow

-edit

I have add a function to check the config file of unbound to be more fail safe


#!/usr/local/bin/bash

#Erstelle Temp Datein

tmp1="$(mktemp)"
tmp2="$(mktemp)"
tmp3="$(mktemp)"
tmp4="$(mktemp)"
tmp5="$(mktemp)"
tmp6="$(mktemp)"
tmp7="$(mktemp)"
file="/var/unbound/adblocklist.conf"
filebackup="/var/unbound/adblocklist.bck"

# Download Blocklist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/ultimate/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/xtreme/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/regional/formats/domains.txt; \
} > $tmp1
# Download Whitelist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/unblock/master/basic/formats/domains.txt; \
        cat /root/whitelist; \
} > $tmp4

# Saeubere die Blocklisten

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp1
cat $tmp1 | tr -d '\r' >> $tmp2
sed -i '' -e 's/ *$//' $tmp2 && sort -uf $tmp2 |tee |uniq -i > $tmp3
sed -i '' -e '/^$/d' $tmp3
rm $tmp1 $tmp2

# Saeubere die Whitelist

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp4
cat $tmp4 | tr -d '\r' >> $tmp5
sed -i '' -e 's/ *$//' $tmp5 && sort -uf $tmp5 |tee |uniq -i > $tmp6
sed -i '' -e '/^$/d' $tmp6
rm $tmp4 $tmp5

# Backup der alten Blocklist
if [ -f $file ]; then
        mv $file $filebackup
fi

# Fuege Blocklist und Whitelist zusammen

comm -23 $tmp3 $tmp6 > $tmp7
rm $tmp3 $tmp6
sed -i '' -e '/^$/d' $tmp7
awk '$0="local-zone: \""$0"\" static"' $tmp7 > $file

# Check Unbound Config

if ! unbound-checkconf /var/unbound/unbound.conf; then
        rm $file
        echo Nutze alte Blockliste
        mv $filebackup $file
else
#       echo Config ist Okey
        rm $filebackup
fi
if [ "$1" == info ]; then
        domains=$(awk '!/^#/ && !/^$/{c++}END{print c}' $file | awk '{ len=length($0); res=""; for (i=0;i<=len;i++) { res=substr($0,len-i+1,1) res; if (i > 0 && i < len && i % 3 == 0) { res = "," res } }; print res }')
        echo "Anzahl zu Blockender Domains = $domains"
fi
# Aendere Benutzer und Gruppe

chown unbound:unbound $file

# Starte Unbound neu
pluginctl dns
exit 0
#4
Hello there  :)

first i'm sorry for my poor english it is not my native language and i'm better in reading it then writing  ::)

I'm using a similar solution and i'm writing here to show you the problems you can have with your script.
And i don't want create another DNS-Block thread.

Quoteunbound-control -c /var/unbound/unbound.conf reload
Is a very bad Solution ... it will fail if the list is to big

QuoteThen:

    Type crontab -e, press Enter and go to the end of the file (you may have to hit the "End" key on the last line)
    Press a
    Press ENTER
    Type: 0     23     *     *     *     (/usr/share/blocklist/getlist.sh) > /dev/null
    Press ESC, then : and finally wq!
It is maybe better to create a action-file in /usr/local/opnsense/service/conf/actions.d to configure cron via web-interface

I will now show the script iam currently using you have to install wget and bash via pkg to use it.
Feel free to edit it to your needs any hints to make the script better is very much appreciated.

ee /root/adblockscript
and insert
#!/usr/local/bin/bash

#Erstelle Temp Datein

tmp1="$(mktemp)"
tmp2="$(mktemp)"
tmp3="$(mktemp)"
tmp4="$(mktemp)"
tmp5="$(mktemp)"
tmp6="$(mktemp)"
tmp7="$(mktemp)"
file="/var/unbound/adblocklist.conf"

# Download Blocklist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/ultimate/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/xtreme/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/regional/formats/domains.txt; \
} > $tmp1
# Download Whitelist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/unblock/master/basic/formats/domains.txt; \
        cat /root/whitelist; \
} > $tmp4

# Saeubere die Blocklisten

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp1
cat $tmp1 | tr -d '\r' >> $tmp2
sed -i '' -e 's/ *$//' $tmp2 && sort -uf $tmp2 |tee |uniq -i > $tmp3
sed -i '' -e '/^$/d' $tmp3
rm $tmp1 $tmp2

# Saeubere die Whitelist

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp4
cat $tmp4 | tr -d '\r' >> $tmp5
sed -i '' -e 's/ *$//' $tmp5 && sort -uf $tmp5 |tee |uniq -i > $tmp6
sed -i '' -e '/^$/d' $tmp6
rm $tmp4 $tmp5

# Fuege Blocklist und Whitelist zusammen

comm -23 $tmp3 $tmp6 > $tmp7
rm $tmp3 $tmp6
sed -i '' -e '/^$/d' $tmp7
awk '$0="local-zone: \""$0"\" static"' $tmp7 > $file
rm $tmp7
if [ "$1" == info ]; then
        domains=$(awk '!/^#/ && !/^$/{c++}END{print c}' $file | awk '{ len=length($0); res=""; for (i=0;i<=len;i++) { res=substr($0,len-i+1,1) res; if (i > 0 && i < len && i % 3 == 0) { res = "," res } }; print res }')
        echo "Anzahl zu Blockender Domains = $domains"
fi
# Aendere Benutzer und Gruppe

chown unbound:unbound $file

# Starte Unbound neu
pluginctl dns
exit 0


To create the Whitelist-file:
echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nichr loeschen!' > /root/whitelist
In the Whitelist-File you can add domains after the first line that shouldn't be blocked.

Create the action-File for cron in "/usr/local/opnsense/service/conf/actions.d"
ee /usr/local/opnsense/service/conf/actions.d/actions_AdBlock.conf
and insert
[reload]
command:/root/adblockscript
parameter:
type:script
message:Update AdBlocklist and load them
description:Update AdBlocklist and load them

then
configctl configd restart
now you can configure the Cron Job via Web Interface

to install bash and wget

pkg lock -y pkg
pkg install bash wget
pkg unlock -y pkg


make the script executable:
chmod +x /root/adblockscript

run the script via
./adblockscript info
it shows how many domains are blocked and create the blockfile for unbound

I'm using the root folder as working environment i know it is kind of laziness  ::)
For the blacklist i am using: https://github.com/EnergizedProtection/block
Output-File is: /var/unbound/adblocklist.conf <= insert this path in your Unbound config

Thanks for your patience and maybe this help someone  ;D
#5
Works fine for me  :)

My Settings:
(keep in mind i have setup up a Lancache with advertisement filtering)

harden-glue: yes
harden-short-bufsize: yes
harden-large-queries: yes

use-caps-for-id: yes
val-clean-additional: yes

cache-min-ttl: 3600
cache-max-ttl: 86400

num-threads: 4
msg-cache-slabs: 8
rrset-cache-slabs: 8
infra-cache-slabs: 8
key-cache-slabs: 8
rrset-cache-size: 256m
msg-cache-size: 128m

include: /var/unbound/ads_and_lancache.conf

unwanted-reply-threshold: 10000
qname-minimisation: yes
do-not-query-localhost: no
ssl-upstream: yes

forward-zone:
name: "."
forward-addr: 9.9.9.9@853


please excuse my bad English

Itow
#6
i had the error since 18.1.5 update
no problems so far
today morning the Firewall installed 18.1.6 and from this point i had no connection through the firewall

i follow the instruction above and now it's working again (i deleted the bogons file change the Firewall Maximum Table Entries)

big thanks @ Reiter der OPNsense and abraxxa

please excuse my bad English

Itow
#7
Hallo Zusammen,

vielen Dank für eure tolle Anleitung :D

Leider hatte ich das Problem, dass die Datei "/usr/local/etc/rc.d/dnscrypt-proxy" während  des updates öfters überschrieben wurde und dadurch der "dnscrypt-proxy" nicht starten konnte, was zum Internetausfall aller Clients führte.
Daher habe ich folgendes Script geschrieben, welches "bash" benötigt, da es in "csh" wohl keine Funktionen gibt. Oder ich habe mich nur blöd angestellt ::)

conf_backup= Das Script von arabesc aus Post 1

Jedem, dessen Englisch besser ist als meines, steht es frei, diesen Text zu übersetzen ;)


#!/usr/local/bin/bash

proxy_tmp=/tmp/tmp_dnscrypt_proxy
proxy_csv=/usr/local/share/dnscrypt-proxy/dnscrypt-resolvers.csv
check_dnscrypt=$(pkg info -ak | grep 'dnscrypt-proxy-' | grep 'yes' | wc -l | tr -d ' ')
conf_backup=/root/dnscrypt_proxy_backup
conf_dnscrypt=/usr/local/etc/rc.d/dnscrypt-proxy
versions_check=$(pkg version -vx dnscrypt-proxy | grep 'up-to-date with remote' | wc -l | tr -d ' ')

update_lock() {
                pkg unlock -qy dnscrypt-proxy
                pkg install dnscrypt-proxy
                pkg lock -qy dnscrypt-proxy
}

update_unlock() {
                pkg install dnscrypt-proxy
                pkg lock -qy dnscrypt-proxy
}

check_proxy_csv() {
                if [ -f $proxy_tmp ]; then
                        rm -rf $proxy_tmp
                fi
                mkdir $proxy_tmp
                wget --no-cache -O $proxy_tmp/dnscrypt-resolvers-new.csv https://raw.githubusercontent.com/jedisct1/dnscrypt-proxy/master/dnscrypt-resolvers.csv
                if ! cmp $proxy_tmp/dnscrypt-resolvers-new.csv $proxy_csv >/dev/null 2>&1; then
                        mv $proxy_tmp/dnscrypt-resolvers-new.csv $proxy_csv
                fi
                rm -rf $proxy_tmp
}

check_conf() {
                checkprocess=$(ps -U_dnscrypt-proxy | grep 'dnscrypt_proxy_' | wc -l | tr -d ' ')
                diff -q $conf_backup $conf_dnscrypt 1>/dev/null
                case "$?" in
                        0)
                                # Alles jut
                                check_proxy_csv
                                if [ "$checkprocess" -eq 0 ]; then
                                service dnscrypt-proxy start
                                else
                                service dnscrypt-proxy restart
                                sleep 3
                                fi
                                ;;
                        1)
                                if [ "$checkprocess" -gt 0 ]; then
                                service dnscrypt-proxy stop
                                sleep 3
                                fi
                                cp "$conf_backup" "$conf_dnscrypt"
                                check_proxy_csv
                                service dnscrypt-proxy start
                                ;;
                esac
}

case "$check_dnscrypt" in
        1)
                if [ "$versions_check" -eq 0 ]; then
                        update_lock
                fi
                check_conf
                ;;
        0)
                if [ "$versions_check" -eq 0 ]; then
                        update_unlock
                else
                        pkg lock -qy dnscrypt-proxy
                fi
                check_conf
                ;;
esac


-edit
Tippfehler