Messages

1

Tutorials and FAQs / Re: Blocking ads using only unbound

« on: January 29, 2020, 06:31:46 am »

Hello there 

yesterday i got following error:

Code: [Select]

[1580222648] unbound-checkconf[27733:0] error: cannot parse name queda212..duckdns.org
[1580222648] unbound-checkconf[27733:0] error: bad zone name queda212..duckdns.org static
[1580222648] unbound-checkconf[27733:0] fatal error: failed local-zone, local-data configuration
So i update the script to replace more then one dot with a single dot.

Code: [Select]

#!/usr/local/bin/bash

oldtmp="$(find /tmp -type f -name 'tmp.*' | wc -l)"
        if [ $oldtmp -gt 0 ]; then
        echo Found $oldtmp old tmp-files.
        echo Delete the old Files
        find /tmp -type f -name 'tmp.*' -exec rm -f {} \;
        fi

#Erstelle Temp Datein
tmp1="$(mktemp)"
tmp2="$(mktemp)"
tmp3="$(mktemp)"
tmp4="$(mktemp)"
tmp5="$(mktemp)"
tmp6="$(mktemp)"
tmp7="$(mktemp)"
file="/var/unbound/adblocklist.conf"
filebackup="/var/unbound/adblocklist.bck"
whitelist="/root/whitelist"
blacklist="/root/blacklist"
actionfile="/usr/local/opnsense/service/conf/actions.d/actions_AdBlock.conf"

# Backup der alten Blocklist
if [ -f $file ]; then
        mv $file $filebackup
else
        touch $filebackup
        echo '# Empty File' > $filebackup
fi

# Pruefe Custom White and Blacklist
if ! [ -f $whitelist ]; then
        touch $whitelist
        echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nicht loeschen!' > $whitelist
fi

if ! [ -f $blacklist ]; then
        touch $blacklist
        echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nicht loeschen!' > $blacklist
fi

# Pruefe und Erstelle Actionfile fuer Cron
if ! [ -f $actionfile ]; then
        touch $actionfile
        printf "[reload]\ncommand:/root/adblockscript\nparameter:\ntype:script\nmessage:Update AdBlocklist and load them\ndescription:Update AdBlocklist and load them" >> $actionfile
        service configd restart
        echo 'Now you can configure the Cron Job via Web Interface'
fi
# Download Blocklist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/ultimate/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/xtreme/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/regional/formats/domains.txt; \
        cat $blacklist; \
} > $tmp1

# Download Whitelist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/unblock/master/basic/formats/domains.txt; \
        cat $whitelist; \
} > $tmp4

# Saeubere die Blocklisten

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp1
cat $tmp1 | sed -r 's/\.+/\./' | tr -d '\r' >> $tmp2
sed -i '' -e 's/ *$//' $tmp2 && sort -uf $tmp2 |tee |uniq -i > $tmp3
sed -i '' -e '/^$/d' $tmp3
rm $tmp1 $tmp2

# Saeubere die Whitelist

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp4
cat $tmp4 | sed -r 's/\.+/\./' | tr -d '\r' >> $tmp5
sed -i '' -e 's/ *$//' $tmp5 && sort -uf $tmp5 |tee |uniq -i > $tmp6
sed -i '' -e '/^$/d' $tmp6
rm $tmp4 $tmp5

# Fuege Blocklist und Whitelist zusammen

comm -23 $tmp3 $tmp6 > $tmp7
rm $tmp3 $tmp6
sed -i '' -e '/^$/d' $tmp7
awk '$0="local-zone: \""$0"\" static"' $tmp7 > $file
rm $tmp7

# Check Unbound Config

if ! unbound-checkconf /var/unbound/unbound.conf; then
        rm $file
        echo Nutze alte Blockliste
        mv $filebackup $file
        exit 1
else
#       echo Config ist Okey
        if ! grep -cq "include: $file" /var/unbound/unbound.conf; then
        echo Please add "include: $file" to your custom Unbound settings.
        fi
        rm $filebackup
fi
if [ "$1" == info ]; then
        domains=$(awk '!/^#/ && !/^$/{c++}END{print c}' $file | awk '{ len=length($0); res=""; for (i=0;i<=len;i++) { res=substr($0,len-i+1,1) res; if (i > 0 && i < len && i % 3 == 0) { res = "," res } }; print res }')
        echo "Anzahl zu Blockender Domains = $domains"
fi
# Aendere Benutzer und Gruppe

chown unbound:unbound $file

# Starte Unbound neu
pluginctl dns
exit 0

Thanks Itow

--Edit 30.01.20 
fix Typos and few Issues add new checks

2

Tutorials and FAQs / Re: Blocking ads using only unbound

« on: January 24, 2020, 02:16:53 am »

if you do not add server: to the start of the first line of .conf file unbound will not start.

I only could start Unbound with that:
example:

Code: [Select]

server:local-zone: "0--ass-cinema-newsp.da.ru" static
local-zone: "0--bondage.dk" static
local-zone: "0--fightingshaving.da.ru" static
local-zone: "0--foodwarez.da.ru" static

Cheers and thanks for your work and scripts, Working like a charm.

Hi,

usually it is not nethethery to add anything to the created Blocklist.

Please check if you have add include: /var/unbound/adblocklist.conf in your Unbound configuration under Custom options in the Web interface. And try remove any addition to the custom options to check if Unbound will function normally without it.

-
I have added a check if the crucial line exist in the unbound config.
Also added checks if Custom Whitelist and Blacklist exist.

Code: [Select]

#!/usr/local/bin/bash

#Erstelle Temp Datein

tmp1="$(mktemp)"
tmp2="$(mktemp)"
tmp3="$(mktemp)"
tmp4="$(mktemp)"
tmp5="$(mktemp)"
tmp6="$(mktemp)"
tmp7="$(mktemp)"
file="/var/unbound/adblocklist.conf"
filebackup="/var/unbound/adblocklist.bck"
whitelist="/root/whitelist"
blacklist="/root/blacklist"

# Backup der alten Blocklist
if [ -f $file ]; then
        mv $file $filebackup
fi

if ! [ -f $whitelist ]; then
        touch $whitelist
        echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nichr loeschen!' > $whitelist
fi

if ! [ -f $blacklist ]; then
        touch $blacklist
        echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nichr loeschen!' > $blacklist
fi

# Pruefe Custom White and Blacklist

# Download Blocklist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/ultimate/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/xtreme/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/regional/formats/domains.txt; \
        cat $blacklist; \
} > $tmp1

# Download Whitelist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/unblock/master/basic/formats/domains.txt; \
        cat $whitelist; \
} > $tmp4

# Saeubere die Blocklisten

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp1
cat $tmp1 | tr -d '\r' >> $tmp2
sed -i '' -e 's/ *$//' $tmp2 && sort -uf $tmp2 |tee |uniq -i > $tmp3
sed -i '' -e '/^$/d' $tmp3
rm $tmp1 $tmp2

# Saeubere die Whitelist

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp4
cat $tmp4 | tr -d '\r' >> $tmp5
sed -i '' -e 's/ *$//' $tmp5 && sort -uf $tmp5 |tee |uniq -i > $tmp6
sed -i '' -e '/^$/d' $tmp6
rm $tmp4 $tmp5

# Fuege Blocklist und Whitelist zusammen

comm -23 $tmp3 $tmp6 > $tmp7
rm $tmp3 $tmp6
sed -i '' -e '/^$/d' $tmp7
awk '$0="local-zone: \""$0"\" static"' $tmp7 > $file

# Check Unbound Config

if ! unbound-checkconf /var/unbound/unbound.conf; then
        rm $file
        echo Nutze alte Blockliste
        mv $filebackup $file
        exit 1
else
#       echo Config ist Okey
        if ! grep -cq "include: $file" /var/unbound/unbound.conf; then
        echo Please add "include: $file" to your custom Unbound settings.
        rm $filebackup
        fi
fi
if [ "$1" == info ]; then
        domains=$(awk '!/^#/ && !/^$/{c++}END{print c}' $file | awk '{ len=length($0); res=""; for (i=0;i<=len;i++) { res=substr($0,len-i+1,1) res; if (i > 0 && i < len && i % 3 == 0) { res = "," res } }; print res }')
        echo "Anzahl zu Blockender Domains = $domains"
fi
# Aendere Benutzer und Gruppe

chown unbound:unbound $file


# Starte Unbound neu
pluginctl dns

Thanks Itow

3

Tutorials and FAQs / Re: Blocking ads using only unbound

« on: January 20, 2020, 02:20:41 pm »

Sinnce https://energized.pro started to publish Unbound-readable blocklists, we should simplify our tutorial to something like this:

1. add include: /var/unbound/ad-blacklist.conf into Custom options of Unbound
2. Create Ad-blacklist-refresh.sh in /var/unbound with:

curl https://raw.githubusercontent.com/EnergizedProtection/block/master/blu/formats/unbound.conf -o /var/unbound/ad-blacklist.conf

3. insert a regular execution of this script into crontab

There are multiple levels of compiled lists by energized.pro team - just pick the right strength, let Unbound use it and enjoy ad-free browsing.

Hello there 

yes your are right you can do that but using a script has also his benefits.

• you can combine more then one list
• you can use a whitlist and a blocklist created by your own
• you are fail safe if the original list has character like ( / \ ) see: https://github.com/EnergizedProtection/block/issues/326
• you can optimize the list for Unbound while create a case insensitive list this will shrink the size

and two personal reason

• i dont like to mess with the crontable im not internally shure but think it is possible that the crontable will be overwrite at update/upgrade
• maybe you learn a bit in writing scripts

If you like it realy easy and wont do anything it is better use a addon for Opnsense like: https://forum.opnsense.org/index.php?topic=14116.0

Thanks Itow

-edit

I have add a function to check the config file of unbound to be more fail safe

Code: [Select]

#!/usr/local/bin/bash

#Erstelle Temp Datein

tmp1="$(mktemp)"
tmp2="$(mktemp)"
tmp3="$(mktemp)"
tmp4="$(mktemp)"
tmp5="$(mktemp)"
tmp6="$(mktemp)"
tmp7="$(mktemp)"
file="/var/unbound/adblocklist.conf"
filebackup="/var/unbound/adblocklist.bck"

# Download Blocklist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/ultimate/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/xtreme/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/regional/formats/domains.txt; \
} > $tmp1
# Download Whitelist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/unblock/master/basic/formats/domains.txt; \
        cat /root/whitelist; \
} > $tmp4

# Saeubere die Blocklisten

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp1
cat $tmp1 | tr -d '\r' >> $tmp2
sed -i '' -e 's/ *$//' $tmp2 && sort -uf $tmp2 |tee |uniq -i > $tmp3
sed -i '' -e '/^$/d' $tmp3
rm $tmp1 $tmp2

# Saeubere die Whitelist

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp4
cat $tmp4 | tr -d '\r' >> $tmp5
sed -i '' -e 's/ *$//' $tmp5 && sort -uf $tmp5 |tee |uniq -i > $tmp6
sed -i '' -e '/^$/d' $tmp6
rm $tmp4 $tmp5

# Backup der alten Blocklist
if [ -f $file ]; then
        mv $file $filebackup
fi

# Fuege Blocklist und Whitelist zusammen

comm -23 $tmp3 $tmp6 > $tmp7
rm $tmp3 $tmp6
sed -i '' -e '/^$/d' $tmp7
awk '$0="local-zone: \""$0"\" static"' $tmp7 > $file

# Check Unbound Config

if ! unbound-checkconf /var/unbound/unbound.conf; then
        rm $file
        echo Nutze alte Blockliste
        mv $filebackup $file
else
#       echo Config ist Okey
        rm $filebackup
fi
if [ "$1" == info ]; then
        domains=$(awk '!/^#/ && !/^$/{c++}END{print c}' $file | awk '{ len=length($0); res=""; for (i=0;i<=len;i++) { res=substr($0,len-i+1,1) res; if (i > 0 && i < len && i % 3 == 0) { res = "," res } }; print res }')
        echo "Anzahl zu Blockender Domains = $domains"
fi
# Aendere Benutzer und Gruppe

chown unbound:unbound $file

# Starte Unbound neu
pluginctl dns
exit 0


4

Tutorials and FAQs / Re: Blocking ads using only unbound

« on: January 16, 2020, 12:59:21 pm »

Hello there 

first i'm sorry for my poor english it is not my native language and i'm better in reading it then writing 

I'm using a similar solution and i'm writing here to show you the problems you can have with your script.
And i don't want create another DNS-Block thread.

unbound-control -c /var/unbound/unbound.conf reload

Is a very bad Solution ... it will fail if the list is to big

Then:

    Type crontab -e, press Enter and go to the end of the file (you may have to hit the "End" key on the last line)
    Press a
    Press ENTER
    Type: 0     23     *     *     *     (/usr/share/blocklist/getlist.sh) > /dev/null
    Press ESC, then : and finally wq!

It is maybe better to create a action-file in /usr/local/opnsense/service/conf/actions.d to configure cron via web-interface

I will now show the script iam currently using you have to install wget and bash via pkg to use it.
Feel free to edit it to your needs any hints to make the script better is very much appreciated.

Code: [Select]

ee /root/adblockscriptand insert

Code: [Select]

#!/usr/local/bin/bash

#Erstelle Temp Datein

tmp1="$(mktemp)"
tmp2="$(mktemp)"
tmp3="$(mktemp)"
tmp4="$(mktemp)"
tmp5="$(mktemp)"
tmp6="$(mktemp)"
tmp7="$(mktemp)"
file="/var/unbound/adblocklist.conf"

# Download Blocklist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/ultimate/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/xtreme/formats/domains.txt; \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/block/master/extensions/regional/formats/domains.txt; \
} > $tmp1
# Download Whitelist
{ \
        wget -qO- https://raw.githubusercontent.com/EnergizedProtection/unblock/master/basic/formats/domains.txt; \
        cat /root/whitelist; \
} > $tmp4

# Saeubere die Blocklisten

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp1
cat $tmp1 | tr -d '\r' >> $tmp2
sed -i '' -e 's/ *$//' $tmp2 && sort -uf $tmp2 |tee |uniq -i > $tmp3
sed -i '' -e '/^$/d' $tmp3
rm $tmp1 $tmp2

# Saeubere die Whitelist

sed -i '' -e '/\//d;/:/d;/(/d;/|/d;/\[/d;/\]/d;/#/d;/^$/d;/[\]/d' $tmp4
cat $tmp4 | tr -d '\r' >> $tmp5
sed -i '' -e 's/ *$//' $tmp5 && sort -uf $tmp5 |tee |uniq -i > $tmp6
sed -i '' -e '/^$/d' $tmp6
rm $tmp4 $tmp5

# Fuege Blocklist und Whitelist zusammen

comm -23 $tmp3 $tmp6 > $tmp7
rm $tmp3 $tmp6
sed -i '' -e '/^$/d' $tmp7
awk '$0="local-zone: \""$0"\" static"' $tmp7 > $file
rm $tmp7
if [ "$1" == info ]; then
        domains=$(awk '!/^#/ && !/^$/{c++}END{print c}' $file | awk '{ len=length($0); res=""; for (i=0;i<=len;i++) { res=substr($0,len-i+1,1) res; if (i > 0 && i < len && i % 3 == 0) { res = "," res } }; print res }')
        echo "Anzahl zu Blockender Domains = $domains"
fi
# Aendere Benutzer und Gruppe

chown unbound:unbound $file

# Starte Unbound neu
pluginctl dns
exit 0

To create the Whitelist-file:

Code: [Select]

echo '# Domains nach dieser Zeile einfuegen. Diese Zeile nichr loeschen!' > /root/whitelistIn the Whitelist-File you can add domains after the first line that shouldn't be blocked.

Create the action-File for cron in "/usr/local/opnsense/service/conf/actions.d"

Code: [Select]

ee /usr/local/opnsense/service/conf/actions.d/actions_AdBlock.confand insert

Code: [Select]

[reload]
command:/root/adblockscript
parameter:
type:script
message:Update AdBlocklist and load them
description:Update AdBlocklist and load them
then

Code: [Select]

configctl configd restartnow you can configure the Cron Job via Web Interface

to install bash and wget

Code: [Select]

pkg lock -y pkg
pkg install bash wget
pkg unlock -y pkg

make the script executable:

Code: [Select]

chmod +x /root/adblockscript
run the script via

Code: [Select]

./adblockscript infoit shows how many domains are blocked and create the blockfile for unbound

I'm using the root folder as working environment i know it is kind of laziness 
For the blacklist i am using: https://github.com/EnergizedProtection/block
Output-File is: /var/unbound/adblocklist.conf <= insert this path in your Unbound config

Thanks for your patience and maybe this help someone 

5

18.1 Legacy Series / Re: ***call for testing*** DNS TLS encryption using Quad9 and Cloudflare DNS servers

« on: April 12, 2018, 12:59:48 pm »

Works fine for me 

My Settings:
(keep in mind i have setup up a Lancache with advertisement filtering)

Code: [Select]

harden-glue: yes
harden-short-bufsize: yes
harden-large-queries: yes

use-caps-for-id: yes
val-clean-additional: yes

cache-min-ttl: 3600
cache-max-ttl: 86400

num-threads: 4
msg-cache-slabs: 8
rrset-cache-slabs: 8
infra-cache-slabs: 8
key-cache-slabs: 8
rrset-cache-size: 256m
msg-cache-size: 128m

include: /var/unbound/ads_and_lancache.conf

unwanted-reply-threshold: 10000
qname-minimisation: yes
do-not-query-localhost: no
ssl-upstream: yes

forward-zone:
name: "."
forward-addr: 9.9.9.9@853
please excuse my bad English

Itow

6

18.1 Legacy Series / Re: 18.1.5 IPv6 bogons invalid after update

« on: April 10, 2018, 06:16:01 pm »

i had the error since 18.1.5 update
no problems so far
today morning the Firewall installed 18.1.6 and from this point i had no connection through the firewall

i follow the instruction above and now it's working again (i deleted the bogons file change the Firewall Maximum Table Entries)

big thanks @ Reiter der OPNsense and abraxxa

please excuse my bad English

Itow

7

Tutorials and FAQs / Re: Multiple dnscrypt-proxy Opnsense 16.7 / 17.1 :-)

« on: August 05, 2017, 07:22:52 pm »

Hallo Zusammen,

vielen Dank für eure tolle Anleitung

Leider hatte ich das Problem, dass die Datei "/usr/local/etc/rc.d/dnscrypt-proxy" während  des updates öfters überschrieben wurde und dadurch der "dnscrypt-proxy" nicht starten konnte, was zum Internetausfall aller Clients führte.
Daher habe ich folgendes Script geschrieben, welches "bash" benötigt, da es in "csh" wohl keine Funktionen gibt. Oder ich habe mich nur blöd angestellt

conf_backup= Das Script von arabesc aus Post 1

Jedem, dessen Englisch besser ist als meines, steht es frei, diesen Text zu übersetzen

Code: [Select]

#!/usr/local/bin/bash

proxy_tmp=/tmp/tmp_dnscrypt_proxy
proxy_csv=/usr/local/share/dnscrypt-proxy/dnscrypt-resolvers.csv
check_dnscrypt=$(pkg info -ak | grep 'dnscrypt-proxy-' | grep 'yes' | wc -l | tr -d ' ')
conf_backup=/root/dnscrypt_proxy_backup
conf_dnscrypt=/usr/local/etc/rc.d/dnscrypt-proxy
versions_check=$(pkg version -vx dnscrypt-proxy | grep 'up-to-date with remote' | wc -l | tr -d ' ')

update_lock() {
                pkg unlock -qy dnscrypt-proxy
                pkg install dnscrypt-proxy
                pkg lock -qy dnscrypt-proxy
}

update_unlock() {
                pkg install dnscrypt-proxy
                pkg lock -qy dnscrypt-proxy
}

check_proxy_csv() {
                if [ -f $proxy_tmp ]; then
                        rm -rf $proxy_tmp
                fi
                mkdir $proxy_tmp
                wget --no-cache -O $proxy_tmp/dnscrypt-resolvers-new.csv https://raw.githubusercontent.com/jedisct1/dnscrypt-proxy/master/dnscrypt-resolvers.csv
                if ! cmp $proxy_tmp/dnscrypt-resolvers-new.csv $proxy_csv >/dev/null 2>&1; then
                        mv $proxy_tmp/dnscrypt-resolvers-new.csv $proxy_csv
                fi
                rm -rf $proxy_tmp
}

check_conf() {
                checkprocess=$(ps -U_dnscrypt-proxy | grep 'dnscrypt_proxy_' | wc -l | tr -d ' ')
                diff -q $conf_backup $conf_dnscrypt 1>/dev/null
                case "$?" in
                        0)
                                # Alles jut
                                check_proxy_csv
                                if [ "$checkprocess" -eq 0 ]; then
                                service dnscrypt-proxy start
                                else
                                service dnscrypt-proxy restart
                                sleep 3
                                fi
                                ;;
                        1)
                                if [ "$checkprocess" -gt 0 ]; then
                                service dnscrypt-proxy stop
                                sleep 3
                                fi
                                cp "$conf_backup" "$conf_dnscrypt"
                                check_proxy_csv
                                service dnscrypt-proxy start
                                ;;
                esac
}

case "$check_dnscrypt" in
        1)
                if [ "$versions_check" -eq 0 ]; then
                        update_lock
                fi
                check_conf
                ;;
        0)
                if [ "$versions_check" -eq 0 ]; then
                        update_unlock
                else
                        pkg lock -qy dnscrypt-proxy
                fi
                check_conf
                ;;
esac

-edit
Tippfehler