Difuse/packages
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #2304 from dibdot/adblock

Browse source 
adblock: update to 0.60.0
This commit is contained in:
Hannu Nyman 2016-01-26 12:17:02 +02:00
commit 5bdd69c1f6
15 changed files with 665 additions and 695 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
net/adblock/Makefile
View file

@ -1,5 +1,5 @@
# #
# Copyright (C) 2015 OpenWrt.org # Copyright (C) 2015-2016 OpenWrt.org
# #
# This is free software, licensed under the GNU General Public License v3. # This is free software, licensed under the GNU General Public License v3.
# #
@ -7,10 +7,10 @@
include $(TOPDIR)/rules.mk include $(TOPDIR)/rules.mk
PKG_NAME:=adblock PKG_NAME:=adblock
PKG_VERSION:=0.40.2 PKG_VERSION:=0.60.0
PKG_RELEASE:=1 PKG_RELEASE:=1
PKG_LICENSE:=GPL-3.0+ PKG_LICENSE:=GPL-3.0+
PKG_MAINTAINER:=Dirk Brenken <dirk@brenken.org> PKG_MAINTAINER:=Dirk Brenken <openwrt@brenken.org>
include $(INCLUDE_DIR)/package.mk include $(INCLUDE_DIR)/package.mk
@ -18,21 +18,23 @@ define Package/$(PKG_NAME)
SECTION:=net SECTION:=net
CATEGORY:=Network CATEGORY:=Network
TITLE:=Powerful adblock script to block ad/abuse domains TITLE:=Powerful adblock script to block ad/abuse domains
DEPENDS:=+curl +wget DEPENDS:=+wget
PKGARCH:=all PKGARCH:=all
endef endef
define Package/$(PKG_NAME)/description define Package/$(PKG_NAME)/description
Powerful adblock script to block ad/abuse domains Powerful adblock script to block ad/abuse domains
When the dns server on your router receives dns requests, we will sort out queries that ask for the [A] When the dns server on your router receives dns requests, we will sort out queries that ask for the resource
resource records of ad servers and return the local ip address of your router and the internal web server records of ad/abuse servers and return the local ip address of your router and the internal web server
delivers a transparent pixel instead. delivers a transparent pixel instead.
The script supports the following domain blacklist sites: The script supports the following domain blacklist sites:
https://adaway.org https://adaway.org
https://disconnect.me https://disconnect.me
http://dshield.org http://dshield.org
https://feodotracker.abuse.ch https://feodotracker.abuse.ch
http://malwaredomains.com http://malwaredomains.com
http://www.malwaredomainlist.com
https://palevotracker.abuse.ch https://palevotracker.abuse.ch
http://www.shallalist.de http://www.shallalist.de
http://www.spam404.com http://www.spam404.com

122
net/adblock/files/README.md
View file

@ -5,20 +5,22 @@ A lot of people already use adblocker plugins within their desktop browsers,
but what if you are using your (smart) phone, tablet, watch or any other wlan gadget... but what if you are using your (smart) phone, tablet, watch or any other wlan gadget...
...getting rid of annoying ads, trackers and other abuse sites (like facebook ;-) is simple: block them with your router. ...getting rid of annoying ads, trackers and other abuse sites (like facebook ;-) is simple: block them with your router.
When the dns server on your router receives dns requests, you’ll sort out queries that ask for the [A] resource records of ad servers and return the local ip address of your router and the internal web server delivers a transparent pixel instead. When the dns server on your router receives dns requests, you will sort out queries that ask for the resource records of ad servers and return the local ip address of your router and the internal web server delivers a transparent pixel instead.
## Main Features ## Main Features
* support of the following domain blocklist sources (free for private usage, for commercial use please check their individual licenses): * support of the following domain blocklist sources (free for private usage, for commercial use please check their individual licenses):
* [adaway.org](https://adaway.org) * [adaway.org](https://adaway.org)
* => infrequent updates, approx. 400 entries * => infrequent updates, approx. 400 entries (enabled by default)
* [disconnect.me](https://disconnect.me) * [disconnect.me](https://disconnect.me)
* => numerous updates on the same day, approx. 6.500 entries * => numerous updates on the same day, approx. 6.500 entries (enabled by default)
* [dshield.org](http://dshield.org) * [dshield.org](http://dshield.org)
* => daily updates, approx. 4.500 entries * => daily updates, approx. 4.500 entries
* [feodotracker.abuse.ch](https://feodotracker.abuse.ch) * [feodotracker.abuse.ch](https://feodotracker.abuse.ch)
* => daily updates, approx. 0-10 entries * => daily updates, approx. 0-10 entries
* [malwaredomains.com](http://malwaredomains.com) * [malwaredomains.com](http://malwaredomains.com)
* => daily updates, approx. 16.000 entries * => daily updates, approx. 16.000 entries
* [malwaredomainlist.com](http://www.malwaredomainlist.com)
* => daily updates, approx. 1.500 entries
* [palevotracker.abuse.ch](https://palevotracker.abuse.ch) * [palevotracker.abuse.ch](https://palevotracker.abuse.ch)
* => daily updates, approx. 15 entries * => daily updates, approx. 15 entries
* [shallalist.de](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default) * [shallalist.de](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default)
@ -33,93 +35,63 @@ When the dns server on your router receives dns requests, you’ll sort out quer
* => weekly updates, approx. 2.500 entries (enabled by default) * => weekly updates, approx. 2.500 entries (enabled by default)
* [zeustracker.abuse.ch](https://zeustracker.abuse.ch) * [zeustracker.abuse.ch](https://zeustracker.abuse.ch)
* => daily updates, approx. 440 entries * => daily updates, approx. 440 entries
* zero-conf like automatic installation & setup, usually no manual changes needed (i.e. ip address, network devices etc.)
* full IPv4 and IPv6 support
* each blocklist source will be updated and processed separately * each blocklist source will be updated and processed separately
* timestamp check to download and process only updated blocklists * timestamp check to download and process only updated adblock list sources
* overall duplicate removal in separate blocklists (will be automatically disabled on low memory systems) * overall duplicate removal in separate adblock lists (will be automatically disabled on low memory systems)
* blocklist parsing by fast & flexible regex rulesets * adblock source list parsing by fast & flexible regex rulesets
* additional white- and blacklist support for manual overrides * additional white- and blacklist support for manual overrides
* use of dynamic adblock network interface * quality checks during/after update of adblock lists to ensure a reliable dnsmasq service
* use of dynamic uhttpd instance as pixel server * wan update check, to wait for an active wan uplink before update
* use of quality checks after adblocklist updates to ensure a reliable dnsmasq service * basic adblock statistics via iptables packet counters
* status & error logging to stdout and syslog
* use of dynamic uhttpd instance as adblock pixel server
* optional features (disabled by default): * optional features (disabled by default):
* powerful adblock list backup/restore handling * adblock list backup/restore
* wan interface check, useful for rc.local based autostart and (mobile) multiwan setups to update only on pre-defined wan interfaces * debug logging to separate file
* domain query logging as a background service to easily identify free and already blocked domains (see example output below)
* adblock statistics (req. query logging)
* ntp time sync
* status & error logging to separate file (req. ntp time sync)
## Prerequisites ## Prerequisites
* [openwrt](https://openwrt.org) (tested only with trunk > r47025), CC should also work * [openwrt](https://openwrt.org), tested with latest stable release (Chaos Calmer 15.05) and with current trunk (Designated Driver > r47025)
* additional software packages: * required software packages:
* curl * wget
* wget (due to an openwrt bug still needed for certain https requests - see ticket #19621) * optional for IPv6 support: kmod-ipt-nat6
* optional: busybox find with *-mtime* support for logfile housekeeping (enabled by default with r47362, will be disabled if not found) * the above dependencies and requirements will be checked during package installation & script runtime, please check console output or *logread -e "adblock"* for errors
* the above dependencies and requirements will be checked during package installation & script startup, please check console output or *logread -e "adblock"* for errors
## Usage ## Usage
* select & install adblock package (*opkg install adblock*) * install the adblock package (*opkg install adblock*)
* configure */etc/config/adblock* to your needs, see additional comments in *adblock.conf.sample* * optional: for an update installation please replace your existing */etc/config/adblock* with a copy of */etc/samples/adblock.conf.sample* to get the latest changes
* at least configure the ip address of the local adblock interface/uhttpd instance, it needs to be a different subnet from the normal LAN * optional: enable/disable your required adblock list sources in */etc/config/adblock* - 'adaway', 'disconnect' and 'yoyo' are enabled by default
* recommendation: there is no need to enable all blacklist sites at once, for normal use one to three lists should be sufficient * start */usr/bin/adblock-update.sh* and check console output or *logread -e "adblock"* for errors
* recommendation: to handle all blacklists at once add an usb stick or any other storage device to supersize your /tmp directory with a swap partition
## Tweaks
* there is no need to enable all blacklist sites at once, for normal use one to three adblock list sources should be sufficient
* if you really need to handle all blacklists at once add an usb stick or any other storage device to supersize your /tmp directory with a swap partition
* => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details * => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details
* add additional domain white- or blacklist entries, one domain per line (wildcards & regex are not allowed!), both lists are located in */etc/adblock* * add static, personal domain white- or blacklist entries, one domain per line (wildcards & regex are not allowed!), by default both lists are located in */etc/adblock*
* by default openwrts main uhttpd instance is bind to all ports of your router, * enable the backup/restore feature, to restore automatically the latest, stable backup of your adblock lists in case of any processing error
* for a working adblock setup you have to bind uhttpd to the standard LAN port only, please change listen_http accordingly * enable the debug log feature for continuous logfile writing to monitor the adblock runs over a longer period
* start /usr/bin/adblock-update.sh and check console output or *logread -e "adblock"* for errors
## Distributed samples ## Distributed samples
* all sample configuration files stored in */etc/adblock/samples* * all sample configuration files stored in */etc/adblock/samples*
* to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see *dhcp.config.sample* * for a fully blown adblock configuration with all explained options see *adblock.conf.sample*
* to bind uhttpd to standard LAN port only, see *uhttpd.config.sample* * for some dnsmasq tweaks see *dhcp.config.sample* and *dnsmasq.conf.sample*
* for rc.local based autostart and /tmp resizing on the fly see *rc.local.sample* * for rc.local based autostart and /tmp resizing on the fly see *rc.local.sample*
* for scheduled call of *adblock-update.sh* see *root.crontab.sample* * for scheduled call of *adblock-update.sh* see *root.crontab.sample*
* to redirect/force all dns queries to your router see *firwall.user.sample*
* for further dnsmasq tweaks see *dnsmasq.conf.sample*
## Examples ## Background
This adblock package is a dns/dnsmasq based adblock solution for openwrt.
Queries to ad/abuse domains are never forwarded and always replied with a local IP address which may be IPv4 or IPv6.
For that purpose adblock uses an ip address from the old reserved [class 'E'](https://tools.ietf.org/html/rfc1700) subnet (254.0.0.1 / ::ffff:fe00:0001) by default.
Furthermore all ad/abuse queries will be filtered by ip(6)tables and redirected to internal adblock pixel server (in PREROUTING chain) or rejected (in FORWARD chain).
All iptables and uhttpd related adblock additions are non-destructive, no hard-coded changes in 'firewall.user', 'uhttpd' config or any other openwrt related config files.
adblock log file excerpt: ## Removal
* remove the adblock package (*opkg remove adblock*)
adblock[11780] info : domain adblock processing started (0.40.1, r47929, 20.12.2015 21:59:56) * remove all script generated adblock lists in */tmp/dnsmasq.d/*
adblock[11780] info : backup/restore will be disabled * kill the running adblock uhttpd instance (ps | grep "[u]httpd.*\-h /www/adblock")
adblock[11780] info : wan update check will be disabled * run /etc/init.d/dnsmasq restart
adblock[11780] info : get ntp time sync (192.168.254.254), after 0 loops * run /etc/init.d/firewall restart
adblock[11780] info : source doesn't change, no update required (whocares)
adblock[11780] info : adblocklists with overall 11619 domains loaded, backups disabled
adblock[11780] info : adblock statistics for query date 20151220 (total: 2495, blocked: 154)
adblock[11780] info : domain adblock processing finished (0.40.1, r47929, 20.12.2015 22:00:03)
domain blocklist for dnsmasq (disconnect.me after overall duplicate removal):
address=/0000mps.webpreview.dsl.net/192.168.2.1
address=/0001.2waky.com/192.168.2.1
address=/001wen.com/192.168.2.1
address=/002it.com/192.168.2.1
address=/00game.net/192.168.2.1
[...]
address=/zzsgssxh.com/192.168.2.1
address=/zzshw.net/192.168.2.1
address=/zztxdown.com/192.168.2.1
address=/zzxcws.com/192.168.2.1
#------------------------------------------------------------------
# adblock-update.sh (0.40.0) - 3710 ad/abuse domains blocked
# source: https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt
# last modified: Thu, 17 Dec 2015 09:21:17 GMT
domain query log excerpt:
query[A] www.seenby.de from fe80::6257:18ff:fe6b:4667
query[A] tarifrechner.heise.de from 192.168.1.131
query[A] www.mittelstandswiki.de from fe80::6257:18ff:fe6b:4667
query[A] ad.doubleclick.net from 192.168.1.131
ad.doubleclick.net is 192.168.2.1
The first three queries are OK (not blocked), the last one has been blocked and answered by local dnsmasq instance.
Have fun! Have fun!
Dirk Dirk

830
net/adblock/files/adblock-helper.sh
View file

File diff suppressed because it is too large Load diff

234
net/adblock/files/adblock-update.sh
View file

@ -1,7 +1,7 @@
#!/bin/sh #!/bin/sh
####################################################### #######################################################
# ad/abuse domain blocking script for dnsmasq/openwrt # # ad/abuse domain blocking script for dnsmasq/openwrt #
# written by Dirk Brenken (dirk@brenken.org) # # written by Dirk Brenken (openwrt@brenken.org) #
####################################################### #######################################################
# LICENSE # LICENSE
@ -25,7 +25,7 @@
# set script version # set script version
# #
adb_version="0.40.2" adb_version="0.60.0"
# get current pid, script directory and openwrt version # get current pid, script directory and openwrt version
# #
@ -39,7 +39,7 @@ if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
then then
. "${adb_scriptdir}/adblock-helper.sh" 2>/dev/null . "${adb_scriptdir}/adblock-helper.sh" 2>/dev/null
else else
rc=600 rc=100
/usr/bin/logger -s -t "adblock[${pid}] error" "adblock function library not found, rc: ${rc}" /usr/bin/logger -s -t "adblock[${pid}] error" "adblock function library not found, rc: ${rc}"
exit ${rc} exit ${rc}
fi fi
@ -50,7 +50,7 @@ fi
# call restore function on trap signals (HUP, INT, QUIT, BUS, SEGV, TERM) # call restore function on trap signals (HUP, INT, QUIT, BUS, SEGV, TERM)
# #
trap "f_log 'trap error' '700'; f_restore" 1 2 3 10 11 15 trap "rc=255; f_log 'trap error' '${rc}'; f_restore" 1 2 3 10 11 15
# start logging # start logging
# #
@ -80,29 +80,24 @@ then
adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}" adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)" list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)"
list_time="${list_time/*: /}" list_time="${list_time/*: /}"
f_log "=> (pre-)processing adblock source '${src_name}'"
# only process shallalist archive with updated timestamp # only process shallalist archive with updated timestamp,
# extract and merge only domains of selected shallalist categories
# #
shalla_time="$(curl ${curl_parm} --max-time 5 --head "${adb_arc_shalla}" 2>/dev/null | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)" shalla_time="$(wget ${wget_parm} --timeout=5 --server-response --spider "${adb_arc_shalla}" 2>&1 | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)"
shalla_time="${shalla_time/*: /}" shalla_time="${shalla_time/*: /}"
if [ -z "${shalla_time}" ] if [ -z "${shalla_time}" ]
then then
shalla_time="$(date)" shalla_time="$(date)"
f_log "no online timestamp received, current date will be used (${src_name})" f_log " no online timestamp received, current date will be used"
fi fi
if [ -z "${list_time}" ] || [ "${list_time}" != "${shalla_time}" ] if [ -z "${list_time}" ] || [ "${list_time}" != "${shalla_time}" ]
then then
f_log "shallalist (pre-)processing started ..." wget ${wget_parm} --timeout="${adb_maxtime}" --tries=1 --output-document="${shalla_archive}" "${adb_arc_shalla}" 2>/dev/null
curl ${curl_parm} --max-time "${adb_maxtime}" "${adb_arc_shalla}" --output "${shalla_archive}" 2>/dev/null
rc=${?} rc=${?}
if [ $((rc)) -ne 0 ] if [ $((rc)) -eq 0 ]
then then
f_log "source download failed (${src_name})" "${rc}"
f_restore
fi
# extract and merge only domains of selected shallalist categories
#
> "${shalla_file}" > "${shalla_file}"
for category in ${adb_cat_shalla} for category in ${adb_cat_shalla}
do do
@ -110,8 +105,8 @@ then
rc=${?} rc=${?}
if [ $((rc)) -ne 0 ] if [ $((rc)) -ne 0 ]
then then
f_log "source archive extraction failed (${category})" "${rc}" f_log " archive extraction failed (${category})"
f_restore break
fi fi
done done
@ -119,32 +114,43 @@ then
# #
rm -f "${shalla_archive}" >/dev/null 2>&1 rm -f "${shalla_archive}" >/dev/null 2>&1
rm -rf "${adb_tmpdir}/BL" >/dev/null 2>&1 rm -rf "${adb_tmpdir}/BL" >/dev/null 2>&1
if [ $((rc)) -eq 0 ]
then
adb_sources="${adb_sources} ${shalla_file}&ruleset=rset_shalla"
f_log " source archive (pre-)processing finished"
else
rc=0
fi
else
f_log " source archive download failed"
rc=0
fi
else
adb_srcfind="! -name ${adb_dnsprefix}.${src_name}"
f_log " source archive doesn't change, no update required"
fi fi
adb_sources="${adb_sources} file:///${shalla_file}&ruleset=rset_shalla"
fi fi
# add blacklist source to active adblock domain sources # add blacklist source to active adblock domain sources
# #
if [ -n "${adb_sources}" ] && [ -s "${adb_blacklist}" ] if [ -s "${adb_blacklist}" ]
then then
adb_sources="${adb_sources} file://${adb_blacklist}&ruleset=rset_blacklist" adb_sources="${adb_sources} ${adb_blacklist}&ruleset=rset_blacklist"
fi fi
# loop through active adblock domain sources, # loop through active adblock domain sources,
# prepare output and store all extracted domains in temp file # download sources, prepare output and store all extracted domains in temp file
# #
for src in ${adb_sources} for src in ${adb_sources}
do do
# download selected adblock sources
#
url="${src/\&ruleset=*/}" url="${src/\&ruleset=*/}"
check_url="$(printf "${url}" | sed -n '/^https:/p' 2>/dev/null)"
src_name="${src/*\&ruleset=rset_/}" src_name="${src/*\&ruleset=rset_/}"
adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}" adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)" list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)"
list_time="${list_time/*: /}" list_time="${list_time/*: /}"
f_log "=> processing adblock source '${src_name}'"
# prepare find statement for all active adblocklist sources # prepare find statement with active adblock list sources
# #
if [ -z "${adb_srcfind}" ] if [ -z "${adb_srcfind}" ]
then then
@ -153,47 +159,40 @@ do
adb_srcfind="${adb_srcfind} -a ! -name ${adb_dnsprefix}.${src_name}" adb_srcfind="${adb_srcfind} -a ! -name ${adb_dnsprefix}.${src_name}"
fi fi
# wget/curl switch # only download adblock list with newer/updated timestamp
# only download blocklist with newer/updated timestamp
# #
if [ -n "${check_url}" ] if [ "${src_name}" = "blacklist" ]
then then
url_time="$(wget ${wget_parm} --timeout=5 --server-response --spider "${url}" 2>&1 | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)" url_time="$(date -r "${adb_blacklist}" 2>/dev/null)"
url_time="${url_time/*: /}" elif [ "${src_name}" = "shalla" ]
if [ -z "${url_time}" ]
then
url_time="$(date)"
f_log "no online timestamp received, current date will be used (${src_name})"
fi
if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ]
then
tmp_domains="$(wget ${wget_parm} --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)"
rc=${?}
else
f_log "source doesn't change, no update required (${src_name})"
continue
fi
else
if [ "${src_name}" = "shalla" ]
then then
url_time="${shalla_time}" url_time="${shalla_time}"
else else
url_time="$(curl ${curl_parm} --max-time 5 --head "${url}" 2>/dev/null | grep -F "Last-Modified: " | tr -d '\r')" url_time="$(wget ${wget_parm} --timeout=5 --server-response --spider "${url}" 2>&1 | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)"
url_time="${url_time/*: /}" url_time="${url_time/*: /}"
fi fi
if [ -z "${url_time}" ] if [ -z "${url_time}" ]
then then
url_time="$(date)" url_time="$(date)"
f_log "no online timestamp received, current date will be used (${src_name})" f_log " no online timestamp received, current date will be used"
fi fi
if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ] if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ]
then then
tmp_domains="$(curl ${curl_parm} --max-time "${adb_maxtime}" "${url}" 2>/dev/null)" if [ "${src_name}" = "blacklist" ]
then
tmp_domains="$(cat "${adb_blacklist}" 2>/dev/null)"
rc=${?}
elif [ "${src_name}" = "shalla" ]
then
tmp_domains="$(cat "${shalla_file}" 2>/dev/null)"
rc=${?} rc=${?}
else else
f_log "source doesn't change, no update required (${src_name})" tmp_domains="$(wget ${wget_parm} --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)"
continue rc=${?}
fi fi
else
f_log " source doesn't change, no update required"
continue
fi fi
# check download result and prepare domain output by regex patterns # check download result and prepare domain output by regex patterns
@ -202,7 +201,7 @@ do
then then
eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')" eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')"
count="$(printf "%s\n" "${tmp_domains}" | tr '[A-Z]' '[a-z]' | eval "${ruleset}" | tee "${adb_tmpfile}" | wc -l)" count="$(printf "%s\n" "${tmp_domains}" | tr '[A-Z]' '[a-z]' | eval "${ruleset}" | tee "${adb_tmpfile}" | wc -l)"
f_log "source download finished (${url}, ${count} entries)" f_log " source download finished (${count} entries)"
if [ "${src_name}" = "shalla" ] if [ "${src_name}" = "shalla" ]
then then
rm -f "${shalla_file}" >/dev/null 2>&1 rm -f "${shalla_file}" >/dev/null 2>&1
@ -210,11 +209,12 @@ do
unset tmp_domains unset tmp_domains
elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ] elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ]
then then
f_log "empty source download finished (${src_name})" f_log " empty source download finished"
continue continue
else else
f_log "source download failed (${src_name})" "${rc}" f_log " source download failed"
f_restore rc=0
continue
fi fi
# remove whitelist domains, sort domains and make them unique, # remove whitelist domains, sort domains and make them unique,
@ -231,7 +231,7 @@ do
rc=${?} rc=${?}
fi fi
# prepare find statement for revised adblocklist sources # prepare find statement with revised adblock list sources
# #
if [ -z "${adb_revsrcfind}" ] if [ -z "${adb_revsrcfind}" ]
then then
@ -240,35 +240,43 @@ do
adb_revsrcfind="${adb_revsrcfind} -o -name ${adb_dnsprefix}.${src_name}" adb_revsrcfind="${adb_revsrcfind} -o -name ${adb_dnsprefix}.${src_name}"
fi fi
# write preliminary adblocklist footer # write preliminary adblock list footer
# #
if [ $((rc)) -eq 0 ] if [ $((rc)) -eq 0 ]
then then
if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
then
count="$(($(wc -l < "${adb_dnsdir}/${adb_dnsprefix}.${src_name}") / 2))"
else
count="$(wc -l < "${adb_dnsdir}/${adb_dnsprefix}.${src_name}")" count="$(wc -l < "${adb_dnsdir}/${adb_dnsprefix}.${src_name}")"
fi
printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsfile}" printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsfile}"
printf "%s\n" "# ${0##*/} (${adb_version}) - ${count} ad/abuse domains blocked" >> "${adb_dnsfile}" printf "%s\n" "# ${0##*/} (${adb_version}) - ${count} ad/abuse domains blocked" >> "${adb_dnsfile}"
printf "%s\n" "# source: ${url}" >> "${adb_dnsfile}" printf "%s\n" "# source: ${url}" >> "${adb_dnsfile}"
printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}" printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}"
f_log "domain merging finished (${src_name})" f_log " domain merging finished"
else else
f_log "domain merging failed (${src_name})" "${rc}" f_log " domain merging failed" "${rc}"
f_restore f_restore
fi fi
else else
f_log "empty domain input received (${src_name})" f_log " empty domain input received"
continue continue
fi fi
done done
# remove old adblocklists and their backups # remove old adblock lists and their backups
# #
if [ -n "${adb_srcfind}" ] if [ -n "${adb_srcfind}" ]
then then
find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srcfind} \) -exec rm -f "{}" \; 2>/dev/null adb_rmfind="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srcfind} \) -print -exec rm -f "{}" \; 2>/dev/null)"
if [ $((rc)) -ne 0 ] if [ $((rc)) -eq 0 ] && [ -n "${adb_rmfind}" ]
then then
f_log "error during removal of old adblocklists" "${rc}" f_log "no longer used adblock lists removed" "${rc}"
f_remove elif [ $((rc)) -ne 0 ]
then
f_log "error during removal of old adblock lists" "${rc}"
f_exit
fi fi
if [ "${backup_ok}" = "true" ] if [ "${backup_ok}" = "true" ]
then then
@ -276,7 +284,7 @@ then
if [ $((rc)) -ne 0 ] if [ $((rc)) -ne 0 ]
then then
f_log "error during removal of old backups" "${rc}" f_log "error during removal of old backups" "${rc}"
f_remove f_exit
fi fi
fi fi
else else
@ -284,19 +292,19 @@ else
if [ "${backup_ok}" = "true" ] if [ "${backup_ok}" = "true" ]
then then
rm -f "${adb_backupdir}/${adb_dnsprefix}."* >/dev/null 2>&1 rm -f "${adb_backupdir}/${adb_dnsprefix}."* >/dev/null 2>&1
f_log "all available adblocklists and backups removed" f_log "all available adblock lists and backups removed"
else else
f_log "all available adblocklists removed" f_log "all available adblock lists removed"
fi fi
fi fi
# make separate adblocklists unique # make separate adblock lists unique
# #
if [ $((adb_unique)) -eq 1 ] if [ $((adb_unique)) -eq 1 ]
then then
if [ -n "${adb_revsrcfind}" ] if [ -n "${adb_revsrcfind}" ]
then then
f_log "remove duplicates in separate adblocklists ..." f_log "remove duplicates in separate adblock lists"
# generate a temporary, unique overall list # generate a temporary, unique overall list
# #
@ -320,7 +328,12 @@ then
# write final adblocklist footer # write final adblocklist footer
# #
if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
then
count="$(($(wc -l < "${adb_dnsdir}/tmp.${list}") / 2))"
else
count="$(wc -l < "${adb_dnsdir}/tmp.${list}")" count="$(wc -l < "${adb_dnsdir}/tmp.${list}")"
fi
printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsdir}/tmp.${list}" printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsdir}/tmp.${list}"
printf "%s\n" "# ${0##*/} (${adb_version}) - ${count} ad/abuse domains blocked" >> "${adb_dnsdir}/tmp.${list}" printf "%s\n" "# ${0##*/} (${adb_version}) - ${count} ad/abuse domains blocked" >> "${adb_dnsdir}/tmp.${list}"
tail -qn -2 "${adb_dnsdir}/$adb_dnsprefix.${list}" 2>/dev/null >> "${adb_dnsdir}/tmp.${list}" tail -qn -2 "${adb_dnsdir}/$adb_dnsprefix.${list}" 2>/dev/null >> "${adb_dnsdir}/tmp.${list}"
@ -330,49 +343,62 @@ then
fi fi
fi fi
# restart dnsmasq with newly generated block lists # get overall count
# #
/etc/init.d/dnsmasq restart >/dev/null 2>&1 if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
sleep 3
# dnsmasq health check
#
dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up" 2>/dev/null)"
if [ -z "${dns_status}" ]
then then
dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep -F "${adb_ip}" 2>/dev/null)" adb_count="$(($(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l) / 2))"
if [ -z "${dns_status}" ] else
then
adb_count="$(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l)" adb_count="$(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l)"
if [ "${backup_ok}" = "true" ] fi
# restart dnsmasq with newly generated or deleted adblock lists,
# check dnsmasq startup afterwards
#
if [ -n "${adb_revsrcfind}" ] || [ -n "${adb_rmfind}" ]
then
/etc/init.d/dnsmasq restart >/dev/null 2>&1
sleep 2
dns_status="$(ps 2>/dev/null | grep "[d]nsmasq" 2>/dev/null)"
if [ -n "${dns_status}" ]
then then
if [ -n "${adb_revsrcfind}" ] f_log "adblock lists with overall ${adb_count} domains loaded"
then
find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrcfind} \) -exec cp -f "{}" "${adb_backupdir}" \; 2>/dev/null
rc=${?}
if [ $((rc)) -ne 0 ]
then
f_log "error during backup of adblocklists" "${rc}"
f_remove
fi
f_log "adblocklists with overall ${adb_count} domains loaded, new backups generated"
else else
f_log "adblocklists with overall ${adb_count} domains loaded, no new backups" rc=105
fi f_log "dnsmasq restart failed, please check 'logread' output" "${rc}"
else
f_log "adblocklists with overall ${adb_count} domains loaded, backups disabled"
fi
else
rc=605
f_log "nslookup probe failed" "${rc}"
f_restore f_restore
fi fi
else else
rc=610 f_log "adblock lists with overall ${adb_count} domains are still valid, no dnsmasq restart required"
f_log "dnsmasq probe failed" "${rc}" fi
f_restore
# create adblock list backups
#
if [ "${backup_ok}" = "true" ] && [ "$(printf "${adb_dnsdir}/${adb_dnsprefix}."*)" != "${adb_dnsdir}/${adb_dnsprefix}.*" ]
then
for file in ${adb_dnsdir}/${adb_dnsprefix}.*
do
filename="${file##*/}"
if [ ! -f "${adb_backupdir}/${filename}" ] || [ "${file}" -nt "${adb_backupdir}/${filename}" ]
then
cp -pf "${file}" "${adb_backupdir}" 2>/dev/null
rc=${?}
if [ $((rc)) -ne 0 ]
then
f_log "error during backup of adblock list (${filename})" "${rc}"
f_exit
fi
backup_done="true"
fi
done
if [ "${backup_done}" = "true" ]
then
f_log "new adblock list backups generated"
else
f_log "adblock list backups are still valid, no new backups required"
fi
fi fi
# remove temporary files and exit # remove temporary files and exit
# #
f_remove f_exit

1
net/adblock/files/adblock.blacklist
View file

@ -1 +0,0 @@

29
net/adblock/files/adblock.conf
View file

@ -1,20 +1,8 @@
# adblock configuration, for further information
# please read /etc/adblock/samples/adblock.conf.sample
# and /etc/adblock/README.md
#
config adblock "global" config adblock "global"
option adb_ip "192.168.2.1"
option adb_domain "heise.de"
option adb_blacklist "/etc/adblock/adblock.blacklist" option adb_blacklist "/etc/adblock/adblock.blacklist"
option adb_whitelist "/etc/adblock/adblock.whitelist" option adb_whitelist "/etc/adblock/adblock.whitelist"
config service "wancheck"
option enabled "0"
list adb_wanlist "wan"
config service "ntpcheck"
option enabled "0"
config service "backup" config service "backup"
option enabled "0" option enabled "0"
option adb_backupdir "/tmp" option adb_backupdir "/tmp"
@ -23,17 +11,12 @@ config service "debuglog"
option enabled "0" option enabled "0"
option adb_logfile "/tmp/adb_debug.log" option adb_logfile "/tmp/adb_debug.log"
config service "querylog"
option enabled "0"
option adb_queryfile "/tmp/adb_query.log"
option adb_queryhistory "1"
config source "adaway" config source "adaway"
option enabled "0" option enabled "1"
option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway" option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
config source "disconnect" config source "disconnect"
option enabled "0" option enabled "1"
option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect" option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
config source "dshield" config source "dshield"
@ -46,7 +29,11 @@ config source "feodo"
config source "malware" config source "malware"
option enabled "0" option enabled "0"
option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware" option adb_src_malware "https://mirror.cedia.org.ec/malwaredomains/justdomains&ruleset=rset_malware"
config source "malwarelist"
option enabled "0"
option adb_src_malwarelist "http://www.malwaredomainlist.com/hostslist/hosts.txt&ruleset=rset_malwarelist"
config source "palevo" config source "palevo"
option enabled "0" option enabled "0"

1
net/adblock/files/adblock.whitelist
View file

@ -1 +0,0 @@

73
net/adblock/files/samples/adblock.conf.sample
View file

@ -1,18 +1,30 @@
# adblock configuration, for further information # adblock configuration, for further information
# please read /etc/adblock/samples/adblock.conf.sample # please see /etc/adblock/samples/adblock.conf.sample
# and /etc/adblock/README.md # and /etc/adblock/README.md
#
# generic options (always required) # generic options (all optional!)
#
config adblock "global" config adblock "global"
# ip address of the local adblock interface/uhttpd instance, # name (or space separated list of names) of the logical wan interface(s)
# needs to be a different subnet from the normal LAN option adb_wanif "wan"
option adb_ip "192.168.2.1"
# name of an "always accessible" domain, # name of the logical lan interface
# this domain will be used for the final nslookup check option adb_lanif "lan"
option adb_domain "heise.de"
# uhttpd port for adblock instance
option adb_port "65535"
# ipv4 blackhole ip address for dnsmasq
# old class 'E' subnet, reserved for future use (certainly not used in normal setups)
option adb_nullipv4 "254.0.0.1"
# ipv6 mapped blackhole ip address for dnsmasq
option adb_nullipv6 "::ffff:fe00:0001"
# download timeout (in seconds) to get the adblock list sources
option adb_maxtime "60"
# max loops/timeout for wan check
option adb_maxloop "20"
# full path to static domain blacklist file (one domain per line) # full path to static domain blacklist file (one domain per line)
# wildcards or regex expressions are not allowed # wildcards or regex expressions are not allowed
@ -22,20 +34,6 @@ config adblock "global"
# wildcards or regex expressions are not allowed # wildcards or regex expressions are not allowed
option adb_whitelist "/etc/adblock/adblock.whitelist" option adb_whitelist "/etc/adblock/adblock.whitelist"
# list of wan devices that are allowed for adblock updates (check /sys/class/net/<dev>),
# if no one found the last adlist backup will be restored,
# useful only for (mobile) multiwan setups
# disabled by default
config service "wancheck"
option enabled "0"
list adb_wanlist "wan"
# check that ntp has adjusted the system time on this device,
# will be used for logfile writing and logfile housekeeping
# disabled by default
config service "ntpcheck"
option enabled "0"
# full path to backup directory for adlist backups # full path to backup directory for adlist backups
# disabled by default # disabled by default
config service "backup" config service "backup"
@ -43,32 +41,23 @@ config service "backup"
option adb_backupdir "/tmp" option adb_backupdir "/tmp"
# full path to debug logfile # full path to debug logfile
# by default adblock logs to syslog and stdout only # normally adblock logs to syslog and stdout only
# disabled by default # disabled by default
config service "debuglog" config service "debuglog"
option enabled "0" option enabled "0"
option adb_logfile "/tmp/adb_debug.log" option adb_logfile "/tmp/adb_debug.log"
# full path to domain query logfile # adblock list source definition
# a background task will trace every dns request to file, to easily identify free and already blocked domains, # please do not change the URLs listed below,
# for this to work, you've to enable the dnsmasq option "logqueries" too.
# the "queryhistory" option deletes query logfiles older than n days (req. busybox find with mtime support)
# disabled by default
config service "querylog"
option enabled "0"
option adb_queryfile "/tmp/adb_query.log"
option adb_queryhistory "1"
# different adblock list sources
# please do not change the urls listed below,
# enable/disable sources as needed # enable/disable sources as needed
# for shallalist you can also enable/disable different ad categories # for shallalist you can also enable/disable different ad categories
# 'adaway', 'disconnect' and 'yoyo' are enabled by default
config source "adaway" config source "adaway"
option enabled "0" option enabled "1"
option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway" option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
config source "disconnect" config source "disconnect"
option enabled "0" option enabled "1"
option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect" option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
config source "dshield" config source "dshield"
@ -81,7 +70,11 @@ config source "feodo"
config source "malware" config source "malware"
option enabled "0" option enabled "0"
option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware" option adb_src_malware "https://mirror.cedia.org.ec/malwaredomains/justdomains&ruleset=rset_malware"
config source "malwarelist"
option enabled "0"
option adb_src_malwarelist "http://www.malwaredomainlist.com/hostslist/hosts.txt&ruleset=rset_malwarelist"
config source "palevo" config source "palevo"
option enabled "0" option enabled "0"

2
net/adblock/files/samples/dhcp.config.sample
View file

@ -4,4 +4,4 @@
config dnsmasq config dnsmasq
option cachesize '1000' option cachesize '1000'
option filterwin2k '0' option filterwin2k '0'
option logqueries '1'

2
net/adblock/files/samples/dnsmasq.conf.sample
View file

@ -1,4 +1,4 @@
# tell DHCP clients to not ask for proxy information # tell DHCP clients not to ask for proxy information
# some clients - like Win7 - will constantly ask if not told "No!" # some clients - like Win7 - will constantly ask if not told "No!"
# configuration found in /etc/dnsmasq # configuration found in /etc/dnsmasq
# #

5
net/adblock/files/samples/firewall.user.sample
View file

@ -1,5 +0,0 @@
# redirect/force all dns queries to port 53 of your router
# configuration found in /etc/firewall.user
#
iptables -t nat -I PREROUTING -p tcp --dport 53 -j REDIRECT --to-ports 53
iptables -t nat -I PREROUTING -p udp --dport 53 -j REDIRECT --to-ports 53

3
net/adblock/files/samples/root.crontab.sample
View file

@ -2,7 +2,6 @@
# configuration found in /etc/crontabs/root # configuration found in /etc/crontabs/root
# #
# start adblock script twice a day # start adblock script once a day at 6 o'clock
# #
0 06 * * * /usr/bin/adblock-update.sh & 0 06 * * * /usr/bin/adblock-update.sh &
0 22 * * * /usr/bin/adblock-update.sh &

4
net/adblock/files/samples/uhttpd.config.sample
View file

@ -1,4 +0,0 @@
# main uhttpd instance listens only to internal LAN
#
config uhttpd 'main'
list listen_http '192.168.1.1:80'

2
net/adblock/files/www/adblock/adblock.html
View file

@ -1,6 +1,6 @@
<html> <html>
<head></head> <head></head>
<body> <body>
<img src="/adblock.png" border=0 alt=""></img> <img src="/adblock.png" border="0" alt=""></img>
</body> </body>
</html> </html>

2
net/adblock/files/www/adblock/index.html
View file

@ -1,6 +1,6 @@
<html> <html>
<head></head> <head></head>
<body> <body>
<img src="/adblock.png" border=0 alt=""></img> <img src="/adblock.png" border="0" alt=""></img>
</body> </body>
</html> </html>