simple-adblock: update to 1.9.4-1

* update default config for new oisd.nl lists
* conf.update file to migrate oisd.nl lists to the new format
* introduce AdBlockPlus lists support (new oisd.nl format)
* longer wait for WAN up/gateway detection
* make load_environemnt only execute once to suppress duplicate
  warnings/errors

PS. While I was testing this, oisd.nl has brought back the old domains
    lists as well, so this version supports both as I'm unclear as to
    why the "big" ABPlus list is only 6.2Mb where as the "big" domains
    list is whopping 19.9Mb.

Signed-off-by: Stan Grishin <stangri@melmac.ca>
This commit is contained in:
Stan Grishin 2023-02-16 22:59:10 +00:00
parent 70009d3586
commit 6c45b40317
3 changed files with 153 additions and 51 deletions

View file

@ -5,8 +5,8 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=simple-adblock
PKG_VERSION:=1.9.3
PKG_RELEASE:=7
PKG_VERSION:=1.9.4
PKG_RELEASE:=1
PKG_MAINTAINER:=Stan Grishin <stangri@melmac.ca>
PKG_LICENSE:=GPL-3.0-or-later

View file

@ -8,8 +8,16 @@ s|raw.githubusercontent.com/StevenBlack/hosts/|cdn.jsdelivr.net/gh/StevenBlack/h
s|raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/|cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@|g
s|raw.githubusercontent.com/jawz101/MobileAdTrackers/|cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@|g
s|http://winhelp2002.mvps.org/hosts.txt|https://winhelp2002.mvps.org/hosts.txt|g
s|list blocked_domains_url 'https://dbl.oisd.nl/basic'|list blocked_adblockplus_url 'https://small.oisd.nl/'|g
s|list blocked_domains_url 'https://dbl.oisd.nl/nsfw'|list blocked_adblockplus_url 'https://nsfw.oisd.nl/'|g
s|list blocked_domains_url 'https://dbl.oisd.nl/'|list blocked_adblockplus_url 'https://big.oisd.nl/'|g
s|list blocked_hosts_url 'https://hosts.oisd.nl/basic'|list blocked_adblockplus_url 'https://small.oisd.nl/'|g
s|list blocked_hosts_url 'https://hosts.oisd.nl/nsfw'|list blocked_adblockplus_url 'https://nsfw.oisd.nl/'|g
s|list blocked_hosts_url 'https://hosts.oisd.nl/'|list blocked_adblockplus_url 'https://big.oisd.nl/'|g
\|dshield.org|d
\|www.malwaredomainlist.com/hostslist/hosts.txt|d
\|https://mirror1.malwaredomains.com/files/justdomains|d
\|lists.disconnect.me|d
\|https://cdn.jsdelivr.net/gh/paulgb/BarbBlock/blacklists/domain-list.txt|d
\|dnsmasq.oisd.nl|d
\|dnsmasq2.oisd.nl|d

View file

@ -62,7 +62,8 @@ readonly sharedMemoryError="/dev/shm/$packageName-error"
readonly sharedMemoryOutput="/dev/shm/$packageName-output"
readonly hostsFilter='/localhost/d;/^#/d;/^[^0-9]/d;s/^0\.0\.0\.0.//;s/^127\.0\.0\.1.//;s/[[:space:]]*#.*$//;s/[[:cntrl:]]$//;s/[[:space:]]//g;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;'
readonly domainsFilter='/^#/d;s/[[:space:]]*#.*$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;'
readonly dnsmasqOISDFilter='\|^server=/[[:alnum:]_.-].*/|!d'
readonly adBlockPlusFilter='/^#/d;/^!/d;s/[[:space:]]*#.*$//;s/^||//;s/\^$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/[`~!@#\$%\^&\*()=+;:"'\'',<>?/\|[{}]/d;/]/d;/\./!d;/^$/d;/[^[:alnum:]_.-]/d;'
readonly dnsmasqFileFilter='\|^server=/[[:alnum:]_.-].*/|!d'
readonly _OK_='\033[0;32m\xe2\x9c\x93\033[0m'
readonly _FAIL_='\033[0;31m\xe2\x9c\x97\033[0m'
readonly __OK__='\033[0;32m[\xe2\x9c\x93]\033[0m'
@ -76,6 +77,20 @@ readonly nft="$(command -v nft)"
readonly canaryDomainsMozilla='use-application-dns.net'
readonly canaryDomainsiCloud='mask.icloud.com mask-h2.icloud.com'
dl_command=
dl_flag=
isSSLSupported=
outputFilter=
outputFilterIPv6=
outputFile=
outputGzip=
outputCache=
awk='awk'
load_environment_flag=
. /lib/functions/network.sh
. /usr/share/libubox/jshn.sh
debug() { local i j; for i in "$@"; do eval "j=\$$i"; echo "${i}: ${j} "; done; }
uci_add_list_if_new() {
@ -112,8 +127,7 @@ get_text() {
errorNoDnsmasqNftset)
r="dnsmasq nft set support is enabled in $packageName, but dnsmasq is either not installed or installed dnsmasq does not support nft set";;
errorNoNft) r="dnsmasq nft sets support is enabled in $packageName, but nft is not installed";;
errorMkdirFail) r="Unable to create directory for";;
errorNoWanGateway) r="The ${serviceName} service failed to discover WAN gateway!";;
errorNoWanGateway) r="The ${serviceName} failed to discover WAN gateway";;
errorOutputDirCreate) r="failed to create directory for %s file";;
errorOutputFileCreate) r="failed to create $outputFile file";;
errorFailDNSReload) r="failed to restart/reload DNS resolver";;
@ -193,9 +207,53 @@ output() {
fi
}
load_network() {
local param="$1"
local i j wan_if wan_gw wan_proto
local counter wan_if_timeout='20' wan_gw_timeout='5'
counter=0
while [ -z "$wan_if" ]; do
network_flush_cache
network_find_wan wan_if
if [ -n "$wan_if" ]; then
output "WAN Interface found: '${wan_if}'.\\n"
break
fi
if [ "$counter" -gt "$wan_if_timeout" ]; then
output "WAN Interface timeout, assuming 'wan'.\\n"
wan_if='wan'
break
fi
counter=$((counter+1))
output "Waiting to discover WAN Interface...\\n"
sleep 1
done
counter=0
wan_proto="$(uci -q get "network.${wan_if}.proto")"
if [ "$wan_proto" = 'pppoe' ]; then
wan_gw_timeout=$((wan_gw_timeout+10))
fi
while [ "$counter" -le "$wan_gw_timeout" ]; do
network_flush_cache
network_get_gateway wan_gw "$wan_if"
if [ -n "$wan_gw" ]; then
output "WAN Gateway found: '${wan_gw}.'\\n"
return 0
fi
counter=$((counter+1))
output "Waiting to discover $wan_if Gateway...\\n"
sleep 1
done
json add error "errorNoWanGateway"
output "${_ERROR_}: $(get_text 'errorNoWanGateway')!\\n"; return 1;
}
load_environment() {
local i j wan_if wan_gw
local validation_result="$1" quiet="$2"
local i j
local validation_result="$1" param="$2"
[ -z "$load_environment_flag" ] || return 0
if [ "$validation_result" != '0' ]; then
json add error "errorConfigValidationFail"
@ -222,7 +280,7 @@ load_environment() {
case "$dns" in
dnsmasq.conf) :;;
*)
if [ -z "$quiet" ]; then
if [ "$param" != 'quiet' ]; then
json add warning "warningExternalDnsmasqConfig"
output "${_WARNING_}: $(get_text 'warningExternalDnsmasqConfig')!\\n"
fi
@ -243,14 +301,14 @@ load_environment() {
case "$dns" in
dnsmasq.ipset)
if dnsmasq -v 2>/dev/null | grep -q 'no-ipset' || ! dnsmasq -v 2>/dev/null | grep -q -w 'ipset'; then
if [ -z "$quiet" ]; then
if [ "$param" != 'quiet' ]; then
json add error "errorNoDnsmasqIpset"
output "${_ERROR_}: $(get_text 'errorNoDnsmasqIpset')!\\n"
fi
dns='dnsmasq.servers'
fi
if ! ipset help hash:net; then
if [ -z "$quiet" ]; then
if [ "$param" != 'quiet' ]; then
json add error "errorNoIpset"
output "${_ERROR_}: $(get_text 'errorNoIpset')!\\n"
fi
@ -259,14 +317,14 @@ load_environment() {
;;
dnsmasq.nftset)
if dnsmasq -v 2>/dev/null | grep -q 'no-nftset' || ! dnsmasq -v 2>/dev/null | grep -q -w 'nftset'; then
if [ -z "$quiet" ]; then
if [ "$param" != 'quiet' ]; then
json add error "errorNoDnsmasqNftset"
output "${_ERROR_}: $(get_text 'errorNoDnsmasqNftset')!\\n"
fi
dns='dnsmasq.servers'
fi
if [ -z "$nft" ]; then
if [ -z "$quiet" ]; then
if [ "$param" != 'quiet' ]; then
json add error "errorNoNft"
output "${_ERROR_}: $(get_text 'errorNoNft')!\\n"
fi
@ -353,14 +411,13 @@ load_environment() {
for i in "$outputFile" "$outputCache" "$outputGzip"; do
if ! mkdir -p "$(dirname "$i")"; then
if [ "$param" != 'quiet' ]; then
json add error "errorOutputDirCreate" "$i"
output "${_ERROR_}: $(get_text 'errorMkdirFail' "$i")!\\n"
output "${_ERROR_}: $(get_text 'errorOutputDirCreate' "$i")!\\n"
fi
fi
done
. /lib/functions/network.sh
. /usr/share/libubox/jshn.sh
is_present 'gawk' && awk='gawk'
if ! is_present '/usr/libexec/grep-gnu' || ! is_present '/usr/libexec/sed-gnu' || \
! is_present '/usr/libexec/sort-coreutils' || ! is_present 'gawk'; then
@ -369,8 +426,8 @@ load_environment() {
is_present '/usr/libexec/grep-gnu' || s="$s grep"
is_present '/usr/libexec/sed-gnu' || s="$s sed"
is_present '/usr/libexec/sort-coreutils' || s="$s coreutils-sort"
if [ -z "$quiet" ]; then
json add warning "errorOutputFileCreate" "${i}"
if [ "$param" != 'quiet' ]; then
json add warning "warningMissingRecommendedPackages" "${i}"
output "${_WARNING_}: $(get_text 'warningMissingRecommendedPackages'), install them by running:\\n"
output "$s;\\n"
fi
@ -405,12 +462,15 @@ load_environment() {
else
unset isSSLSupported
fi
load_environment_flag=1
cache 'test' && return 0
cache 'test_gzip' && return 0
network_flush_cache; network_find_wan wan_if; network_get_gateway wan_gw "$wan_if";
[ -n "$wan_gw" ] && return 0
json add error "errorNoWanGateway"
output "${_ERROR_}: $(get_text 'errorNoWanGateway')!\\n"; return 1;
if [ "$param" = 'on_boot' ]; then
load_network "$param"
return "$?"
else
return 0
fi
}
resolver() {
@ -555,7 +615,7 @@ json() {
case "$param" in
triggers)
curReload="$parallel_downloads $debug $download_timeout $allowed_domain $blocked_domain $allowed_domains_url \
$blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \
$blocked_adblockplus_url $blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \
$dnsmasq_config_file_url $curl_max_file_size $curl_retry"
curRestart="$compressed_cache $force_dns $led $force_dns_port"
if [ ! -s "$jsonFile" ]; then
@ -589,7 +649,7 @@ json() {
case "$param" in
triggers)
reload="$parallel_downloads $debug $download_timeout $allowed_domain $blocked_domain $allowed_domains_url \
$blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \
$blocked_adblockplus_url $blocked_domains_url $blocked_hosts_url $dns $config_update_enabled $config_update_url \
$dnsmasq_config_file_url $curl_max_file_size $curl_retry"
restart="$compressed_cache $force_dns $led $force_dns_port"
;;
@ -658,14 +718,22 @@ process_url() {
if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then return 1; fi
label="${1##*//}"; label="${label%%/*}";
case "$2" in
dnsmasq) label="Dnsmasq: $label"; filter="$dnsmasqOISDFilter";;
domains) label="Domains: $label"; filter="$domainsFilter";;
hosts) label="Hosts: $label"; filter="$hostsFilter";;
adbp) label="ADBPlus: $label"; filter="$adBlockPlusFilter"
;;
dnsmasq) label="Dnsmasq: $label"; filter="$dnsmasqFileFilter"
;;
domains) label="Domains: $label"; filter="$domainsFilter"
;;
hosts) label="Hosts: $label"; filter="$hostsFilter"
;;
esac
case "$3" in
allowed) type='Allowed'; D_TMP="$A_TMP";;
blocked) type='Blocked'; D_TMP="$B_TMP";;
file) type='File'; D_TMP="$B_TMP";;
allowed) type='Allowed'; D_TMP="$A_TMP"
;;
blocked) type='Blocked'; D_TMP="$B_TMP"
;;
file) type='File'; D_TMP="$B_TMP"
;;
esac
if [ "${1:0:5}" = "https" ] && [ -z "$isSSLSupported" ]; then
output 1 "$_FAIL_"
@ -676,7 +744,7 @@ process_url() {
while [ -z "$R_TMP" ] || [ -e "$R_TMP" ]; do
R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)"
done
if ! $dl_command "$1" $dl_flag "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
if ! $dl_command "$1" "$dl_flag" "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
output 1 "$_FAIL_"
output 2 "[DL] $type $label $__FAIL__\\n"
echo "errorDownloadingList|${1}" >> "$sharedMemoryError"
@ -757,6 +825,13 @@ download_lists() {
process_url "$hf" 'hosts' 'blocked'
fi
done
for hf in ${blocked_adblockplus_url}; do
if [ "$parallel_downloads" -gt 0 ]; then
process_url "$hf" 'adbp' 'blocked' &
else
process_url "$hf" 'adbp' 'blocked'
fi
done
for hf in ${blocked_domains_url}; do
if [ "$parallel_downloads" -gt 0 ]; then
process_url "$hf" 'domains' 'blocked' &
@ -1080,8 +1155,12 @@ adb_check() {
adb_config_update() {
local R_TMP label
local param="$1" validation_result="$3"
load_environment "$validation_result" 'quiet' || return 1
local param validation_result="$3"
case "$1" in
on_boot) param="$1";;
*) param='quiet';;
esac
load_environment "$validation_result" "$param" || return 1
label="${config_update_url##*//}"
label="${label%%/*}";
[ "$config_update_enabled" -ne 0 ] || return 0
@ -1094,7 +1173,7 @@ adb_config_update() {
while [ -z "$R_TMP" ] || [ -e "$R_TMP" ]; do
R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)"
done
if ! $dl_command "$config_update_url" $dl_flag "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
if ! $dl_command "$config_update_url" "$dl_flag" "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
output 1 "$_FAIL_\\n"
output 2 "[DL] Config Update: $label $__FAIL__\\n"
json add error "errorDownloadingConfigUpdate"
@ -1118,9 +1197,27 @@ adb_sizes() {
load_environment "$validation_result" 'quiet' || return 1
echo "# $(date)"
for i in $blocked_domains_url; do
for i in ${blocked_adblockplus_url}; do
[ "${i//melmac}" != "$i" ] && continue
if $dl_command "$i" $dl_flag /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
if $dl_command "$i" "$dl_flag" /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
echo "# File size: $(du -sh /tmp/sast | $awk '{print $1}')"
if is_greater "$(du -sk /tmp/sast)" "500"; then
echo "# block-list too big for most routers"
elif is_greater "$(du -sk /tmp/sast)" "100"; then
echo "# block-list may be too big for some routers"
fi
rm -rf /tmp/sast
echo " list blocked_adblockplus_url '$i'"
echo ""
else
echo "# site was down on last check"
echo "# list blocked_adblockplus_url '$i'"
echo ""
fi
done
for i in ${blocked_domains_url}; do
[ "${i//melmac}" != "$i" ] && continue
if $dl_command "$i" "$dl_flag" /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
echo "# File size: $(du -sh /tmp/sast | $awk '{print $1}')"
if is_greater "$(du -sk /tmp/sast)" "500"; then
echo "# block-list too big for most routers"
@ -1136,8 +1233,8 @@ adb_sizes() {
echo ""
fi
done
for i in $blocked_hosts_url; do
if $dl_command "$i" $dl_flag /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
for i in ${blocked_hosts_url}; do
if $dl_command "$i" "$dl_flag" /tmp/sast 2>/dev/null && [ -s /tmp/sast ]; then
echo "# File size: $(du -sh /tmp/sast | $awk '{print $1}')"
if is_greater "$(du -sk /tmp/sast)" "500"; then
echo "# block-list too big for most routers"
@ -1157,9 +1254,9 @@ adb_sizes() {
adb_start() {
local action status error message stats c
local validation_result="$3"
local param="$1" validation_result="$3"
load_environment "$validation_result" || return 1
load_environment "$validation_result" "$param" || return 1
status="$(json get status)"
error="$(json get error)"
@ -1272,7 +1369,11 @@ adb_start() {
procd_open_data
json_add_string 'status' "$(json get status)"
json_add_string 'errors' "$(json get errors)"
if [ -s "$outputFile" ]; then
json_add_int 'entries' "$(wc -l < "$outputFile")"
else
json_add_int 'entries' '0'
fi
json_add_array firewall
if [ "$force_dns" -ne 0 ]; then
for c in $force_dns_port; do
@ -1460,15 +1561,6 @@ stop_service() { load_validate_config 'config' adb_stop "'$*'"; }
version() { echo "$PKG_VERSION"; }
load_validate_config() {
local dl_command
local dl_flag
local isSSLSupported
local outputFilter
local outputFilterIPv6
local outputFile
local outputGzip
local outputCache
local awk='awk'
local enabled
local force_dns
local force_dns_port
@ -1491,6 +1583,7 @@ load_validate_config() {
local dns_instance
local allowed_domain
local allowed_domains_url
local blocked_adblockplus_url
local blocked_domain
local blocked_domains_url
local blocked_hosts_url
@ -1519,6 +1612,7 @@ load_validate_config() {
'allowed_domain:list(string)' \
'allowed_domains_url:list(string)' \
'blocked_domain:list(string)' \
'blocked_adblockplus_url:list(string)' \
'blocked_domains_url:list(string)' \
'blocked_hosts_url:list(string)' \
'dnsmasq_config_file_url:string'