Merge pull request #12333 from stangri/master-simple-adblock

simple-adblock: bugfix: proper error reporting on failed downloads; lists update script

net/simple-adblock/Makefile

@@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=simple-adblock
 PKG_VERSION:=1.8.3
-PKG_RELEASE:=3
+PKG_RELEASE:=9
 PKG_MAINTAINER:=Stan Grishin <stangri@melmac.net>
 PKG_LICENSE:=GPL-3.0-or-later
 
@@ -21,7 +21,7 @@ endef
 
 define Package/simple-adblock/description
 This service provides DNSMASQ or Unbound based ad blocking.
-Please see the project's README at github for further information.
+Please see the project's README on GitHub for further information.
 
 endef
 
@@ -46,6 +46,8 @@ define Package/simple-adblock/install
 	$(INSTALL_BIN) $(PKG_BUILD_DIR)/files/simple-adblock.init $(1)/etc/init.d/simple-adblock
 	$(INSTALL_DIR) $(1)/etc/config
 	$(INSTALL_CONF) ./files/simple-adblock.conf $(1)/etc/config/simple-adblock
+	$(INSTALL_DIR) $(1)/tmp
+	$(INSTALL_CONF) ./files/simple-adblock.conf.update $(1)/tmp/simple-adblock.conf.update
 	$(INSTALL_DIR) $(1)/etc/hotplug.d/iface
 	$(INSTALL_BIN) ./files/simple-adblock.hotplug $(1)/etc/hotplug.d/iface/80-simple-adblock
 endef
@@ -54,6 +56,7 @@ define Package/simple-adblock/postinst
 	#!/bin/sh
 	# check if we are on real system
 	if [ -z "$${IPKG_INSTROOT}" ]; then
+		sed -f /tmp/simple-adblock.conf.update -i /etc/config/simple-adblock || true
 		/etc/init.d/simple-adblock enable
 	fi
 	exit 0

net/simple-adblock/files/README.md

@@ -19,63 +19,70 @@ A simple DNSMASQ/Unbound-based AdBlocking service for OpenWrt/LEDE Project.
 - Blocks ads inside browsers with [DNS-over-HTTPS proxy](https://en.wikipedia.org/wiki/DNS_over_HTTPS) built-in, like [Mozilla Firefox](https://support.mozilla.org/en-US/kb/firefox-dns-over-https#w_about-dns-over-https) or [Google Chrome/Chromium](https://blog.chromium.org/2019/09/experimenting-with-same-provider-dns.html) -- with the ```dnsmasq.ipset``` option.
 - Proudly made in Canada, using locally-sourced electrons.
 
-If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out [net/adblock](https://github.com/openwrt/packages/tree/master/net/adblock/files).
+If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out ```net/adblock``` on [GitHub](https://github.com/openwrt/packages/tree/master/net/adblock/files)/[jsDelivr](https://cdn.jsdelivr.net/gh/openwrt/packages@master/net/adblock/files/README.md).
 
 ## Screenshots (luci-app-simple-adblock)
 
 Service Status
 
-![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-status.png "Service Status")
+![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-status.png "Service Status")
 
 Configuration - Basic Configuration
 
-![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-config-basic.png "Configuration - Basic Configuration")
+![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-config-basic.png "Configuration - Basic Configuration")
 
 Configuration - Advanced Configuration
 
-![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-config-advanced.png "Configuration - Advanced Configuration")
+![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-config-advanced.png "Configuration - Advanced Configuration")
 
 Whitelist and Blocklist Management
 
-![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-lists.png "Whitelist and Blocklist Management")
+![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-lists.png "Whitelist and Blocklist Management")
 
 ## Requirements
 
-This service requires the following packages to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` or ```unbound``` and either ```ca-certificates```, ```wget``` and ```libopenssl``` (for OpenWrt 15.05.1) or ```uclient-fetch``` and ```libustream-mbedtls``` (for LEDE Project and OpenWrt 18.06.xx or newer). Additionally installation of ```coreutils-sort``` is highly recommended as it speeds up blocklist processing.
+To download the hosts/domains files this service requires one of the following packages to be installed on your device: ```wget``` or ```uclient-fetch``` or ```curl```. If you want to download lists over HTTPS (recommended), you also need to install support for HTTPS/SSL downloads on your device. For most devices starting with OpenWrt 15.05.1 and up the following commands will install the minimal requirements:
-
-To satisfy the requirements for connect to your router via ssh and run the following commands:
-
-### How to use DNSMASQ ipset
-
-The ```dnsmasq.ipset``` option requires you to install ```ipset``` and  ```dnsmasq-full``` instead of the ```dnsmasq```. To do that, connect to your router via ssh and run the following command:
 
 ```sh
-opkg update; opkg remove dnsmasq; opkg install dnsmasq-full ipset;
+opkg update; if ubus -S call system board | grep -q '15.05'; then
+opkg install ca-certificates wget libopenssl; else
+opkg install uclient-fetch libustream-mbedtls ca-bundle ca-certificates; fi
 ```
 
-### OpenWrt 15.05.1 Requirements
+### Requirements for file:// Scheme
+
+If you want to include some local files as the blocklists or whitelists, you need to install ```curl```:
 
 ```sh
-opkg update; opkg install ca-certificates wget libopenssl coreutils-sort dnsmasq
+opkg update; opkg install curl;
 ```
 
-### LEDE Project 17.01.x and OpenWrt 18.xx (or newer) Requirements
+### Requirements for DNS Resolver
+
+In order to actually block the ads, this service requires one of the following DNS resolvers to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` or ```unbound```. The ```dnsmasq``` package is usually included in the default OpenWrt installation.
+
+If you want to use the [```dnsmasq.ipset``` option](#dns-resolution-option) you need to install ```ipset``` and  ```dnsmasq-full``` instead of the ```dnsmasq```. To do that, connect to your router via ssh and run the following command:
 
 ```sh
-opkg update; opkg install uclient-fetch libustream-mbedtls coreutils-sort dnsmasq
+opkg update; cd /tmp/; opkg download dnsmasq-full; opkg install ipset;
+opkg remove dnsmasq; opkg install /tmp/dnsmasq-full*; rm -f /tmp/dnsmasq-full*;
 ```
 
-### IPv6 Support
+### Requirements for IPv6 Support
 
 For IPv6 support additionally install ```ip6tables-mod-nat``` and ```kmod-ipt-nat6``` packages from Web UI or run the following in the command line:
 
 ```sh
-opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6
+opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6;
 ```
 
-### Speed Up Blocklist Processing
+### Requirements for Faster Blocklist Processing
 
-The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run ```opkg --force-overwrite install coreutils-sort```.
+The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run the following command: 
+
+```sh
+opkg --force-overwrite install coreutils-sort
+```
 
 ## Unmet Dependencies
 
@@ -89,11 +96,11 @@ Install ```simple-adblock``` and ```luci-app-simple-adblock``` packages from Web
 opkg update; opkg install simple-adblock luci-app-simple-adblock
 ```
 
-If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to [add a custom repo to your router](https://github.com/stangri/openwrt_packages/blob/master/README.md#on-your-router) first.
+If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to add a custom repo to your router following instructions on [GitHub](https://github.com/stangri/openwrt_packages/blob/master/README.md#on-your-router)/[jsDelivr](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/README.md#on-your-router) first.
 
 ## Default Settings
 
-Default configuration has service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1; uci commit simple-adblock;```) and selected ad/malware lists suitable for routers with 64Mb RAM.
+Default configuration has the service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1; uci commit simple-adblock;```) and selected ad/malware lists suitable for routers with 64Mb RAM.
 
 If your router has less then 64Mb RAM, edit the configuration file, located at ```/etc/config/simple-adblock```. The configuration file has lists in ascending order starting with smallest ones and each list has a preceding comment indicating its size, comment out or delete the lists you don't want or your router can't handle.
 

net/simple-adblock/files/simple-adblock.conf

@@ -11,7 +11,7 @@ config simple-adblock 'config'
 	option parallel_downloads '1'
 	option debug '0'
 	option compressed_cache '0'
-	list whitelist_domain 'raw.githubusercontent.com'
+	list whitelist_domain 'cdn.jsdelivr.net'
 
 # Thu Oct  3 17:54:04 PDT 2019
 # File size: 4.0K
@@ -45,7 +45,7 @@ config simple-adblock 'config'
 	list blacklist_hosts_url 'https://adaway.org/hosts.txt'
 	
 # File size: 20.0K
-	list blacklist_hosts_url 'https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt'
+	list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@master/hosts.txt'
 
 # File size: 36.0K
 	list blacklist_hosts_url 'https://www.malwaredomainlist.com/hostslist/hosts.txt'
@@ -56,7 +56,7 @@ config simple-adblock 'config'
 # File size: 388.0K
 # blocklist may be too big for some routers
 # blocklist may block some video-streaming content
-#	list blacklist_hosts_url 'https://raw.githubusercontent.com/jawz101/MobileAdTrackers/master/hosts'
+#	list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@master/hosts'
 
 # File size: 424.0K
 # blocklist may be too big for some routers
@@ -72,7 +72,7 @@ config simple-adblock 'config'
 
 # File size: 1.6M
 # blocklist too big for most routers
-#	list blacklist_hosts_url 'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'
+#	list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/StevenBlack/hosts@master/hosts'
 
 # File size: 3.1M
 # blocklist too big for most routers
@@ -80,7 +80,7 @@ config simple-adblock 'config'
 
 # File size: 23.0M
 # blocklist too big for most routers
-#	list blacklist_hosts_url 'https://dbl.oisd.nl/'
+#	list blacklist_hosts_url 'https://hosts.oisd.nl/'
 
 # site was down on last check
 #	list blacklist_domains_url 'http://support.it-mate.co.uk/downloads/hosts.txt'

net/simple-adblock/files/simple-adblock.conf.update

@@ -0,0 +1,4 @@
+s|dbl.oisd.nl|hosts.oisd.nl|g
+s|raw.githubusercontent.com/StevenBlack/hosts/|cdn.jsdelivr.net/gh/StevenBlack/hosts@|g
+s|raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/|cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@|g
+s|raw.githubusercontent.com/jawz101/MobileAdTrackers/|cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@|g

net/simple-adblock/files/simple-adblock.init

@@ -145,6 +145,7 @@ export whitelist_domains blacklist_domains
 export whitelist_domains_urls blacklist_domains_urls blacklist_hosts_urls
 export wan_if wan_gw wanphysdev dl_command serviceStatus dl_flag
 export outputFilter outputFilterIPv6 outputFile outputGzip outputCache ipv6Enabled
+export is_ssl_supported
 
 load_package_config() {
 	config_load "$packageName"
@@ -231,11 +232,11 @@ load_package_config() {
 	fi
 	. /lib/functions/network.sh
 	. /usr/share/libubox/jshn.sh
-	# Prefer curl because it supports the file: scheme.
+	# Prefer curl because it supports the file:// scheme.
 	if [ -x /usr/bin/curl ]; then
 		dl_command="curl --insecure --retry $curlRetry --connect-timeout $dlTimeout --silent"
 		dl_flag="-o"
-	elif wget -V 2>/dev/null | grep -q "+ssl"; then
+	elif wget --version 2>/dev/null | grep -q "+https"; then
 		dl_command="wget --no-check-certificate --timeout $dlTimeout -q"
 		dl_flag="-O"
 	else
@@ -243,6 +244,15 @@ load_package_config() {
 		dl_flag="-O"
 	fi
 	led="${led:+/sys/class/leds/$led}"
+	if curl --version 2>/dev/null | grep -q "https" \
+		|| wget --version 2>/dev/null | grep -q "+https" \
+		|| grep -q "libustream-mbedtls" /usr/lib/opkg/status \
+		|| grep -q "libustream-openssl" /usr/lib/opkg/status \
+		|| grep -q "libustream-wolfssl" /usr/lib/opkg/status; then
+		is_ssl_supported=1
+	else
+		unset is_ssl_supported
+	fi
 }
 
 is_enabled() {
@@ -668,19 +678,25 @@ process_url() {
 	else
 		type='Allowed'; D_TMP="$A_TMP";
 	fi
+	if [ "${1:0:5}" == "https" ] && [ -z "$is_ssl_supported" ]; then
+		output 1 "$_FAIL_"
+		output 2 "[DL] $type $label $__FAIL__\\n"
+		echo "errorNoSSLSupport|${1}" >> "$sharedMemoryError"
+		return 0
+	fi
 	while [ -z "$R_TMP" ] || [ -e "$R_TMP" ]; do
 		R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)"
 	done
 	if ! $dl_command "$1" $dl_flag "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
 		output 1 "$_FAIL_"
 		output 2 "[DL] $type $label $__FAIL__\\n"
-		echo "errorDownloadingList=${1}" >> "$sharedMemoryError"
+		echo "errorDownloadingList|${1}" >> "$sharedMemoryError"
 	else
 		sed -i "$filter" "$R_TMP"
 		if [ ! -s "$R_TMP" ]; then
 			output 1 "$_FAIL_"
 			output 2 "[DL] $type $label $__FAIL__\\n"
-			echo "errorParsingList=${1}" >> "$sharedMemoryError"
+			echo "errorParsingList|${1}" >> "$sharedMemoryError"
 		else
 			cat "${R_TMP}" >> "$D_TMP"
 			output 1 "$_OK_"
@@ -1115,8 +1131,8 @@ showstatus() {
 	fi
 	if [ -n "$error" ]; then
 		for c in $error; do
-			url="${c##*=}"
+			url="${c##*|}"
-			c="${c%=*}"
+			c="${c%|*}"
 			case "$c" in
 				errorDownloadingList|errorParsingList)
 					output "$_ERROR_: $(getErrorText "$c") $url!\\n";;