banip: release 0.8.3-1

* add the new init command 'lookup', to lookup the IPs of domain names in the local lists and update them
* significant acceleration of the domain lookup function
* multiple small fixes and improvements
* readme update
* luci update (separate commit)

Signed-off-by: Dirk Brenken <dev@brenken.org>

net/banip/Makefile

@@ -7,8 +7,8 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=banip
-PKG_VERSION:=0.8.2
+PKG_VERSION:=0.8.3
-PKG_RELEASE:=6
+PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0-or-later
 PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>
 

net/banip/files/README.md

@@ -75,7 +75,7 @@ IP address blocking is commonly used to protect against brute force attacks, pre
 * Provides a set search engine for certain IPs
 * Feed parsing by fast & flexible regex rulesets
 * Minimal status & error logging to syslog, enable debug logging to receive more output
-* Procd based init system support (start/stop/restart/reload/status/report/search/survey)
+* Procd based init system support (start/stop/restart/reload/status/report/search/survey/lookup)
 * Procd network interface trigger support
 * Ability to add new banIP feeds on your own
 
@@ -114,6 +114,7 @@ Available commands:
 	report          [text|json|mail] Print banIP related set statistics
 	search          [<IPv4 address>|<IPv6 address>] Check if an element exists in a banIP set
 	survey          [<set name>] List all elements of a given banIP set
+	lookup          Lookup the IPs of domain names in the local lists and update them
 	running         Check if service is running
 	status          Service status
 	trace           Start with syscall trace
@@ -226,18 +227,16 @@ Available commands:
 ~# /etc/init.d/banip status
 ::: banIP runtime information
   + status            : active (nft: ✔, monitor: ✔)
-  + version           : 0.8.2-2
+  + version           : 0.8.3-1
-  + element_count     : 211397
+  + element_count     : 281161
-  + active_feeds      : allowlistvMAC, allowlistv4, allowlistv6, adawayv4, adawayv6, adguardv4, adguardtrackersv4, adguardv6, adguardtrackersv
+  + active_feeds      : allowlistvMAC, allowlistv6, allowlistv4, adawayv4, adguardtrackersv4, adawayv6, adguardv6, adguardv4, adguardtrackersv6, antipopadsv6, antipopadsv4, cinsscorev4, deblv4, countryv6, countryv4, deblv6, dohv4, dohv6, iblockadsv4, firehol1v4, oisdbigv4, yoyov6, threatviewv4, yoyov4, oisdbigv6, blocklistvMAC, blocklistv4, blocklistv6
-                        6, antipopadsv4, antipopadsv6, cinsscorev4, countryv6, countryv4, deblv4, deblv6, dohv4, dohv6, firehol1v4, oisdsmallv
+  + active_devices    : br-wan ::: wan, wan6
-                        6, oisdsmallv4, stevenblackv6, stevenblackv4, webclientv4, blocklistvMAC, blocklistv4, blocklistv6
+  + active_subnets    : 91.64.169.252/24, 2a02:710c:0:60:958b:3bd0:9e14:abb/128
-  + active_devices    : eth2 ::: wan, wan6
-  + active_subnets    : 91.64.148.211/24, 2b02:710c:0:80:e442:4b0c:637d:1d33/128
   + nft_info          : priority: -200, policy: memory, loglevel: warn, expiry: -
   + run_info          : base: /mnt/data/banIP, backup: /mnt/data/banIP/backup, report: /mnt/data/banIP/report, feed: /etc/banip/banip.feeds
   + run_flags         : auto: ✔, proto (4/6): ✔/✔, log (wan-inp/wan-fwd/lan-fwd): ✔/✔/✔, dedup: ✔, split: ✘, allowed only: ✘
-  + last_run          : action: restart, duration: 0m 55s, date: 2023-03-10 19:33:08
+  + last_run          : action: reload, duration: 1m 0s, date: 2023-04-06 12:34:10
-  + system_info       : cores: 2, memory: 1830, device: Turris Omnia, OpenWrt SNAPSHOT r22248-bf055fcdca
+  + system_info       : cores: 4, memory: 1822, device: Bananapi BPI-R3, OpenWrt SNAPSHOT r22498-75f7e2d10b
 ```
 
 **banIP search information**  
@@ -288,15 +287,22 @@ list ban_logterm 'SecurityEvent=\"InvalidAccountID\".*RemoteAddress='
 **allow-/blocklist handling**  
 banIP supports local allow and block lists (IPv4, IPv6, CIDR notation or domain names), located in /etc/banip/banip.allowlist and /etc/banip/banip.blocklist.  
 Unsuccessful login attempts or suspicious requests will be tracked and added to the local blocklist (see the 'ban\_autoblocklist' option). The blocklist behaviour can be further tweaked with the 'ban\_nftexpiry' option.  
-Furthermore the uplink subnet will be added to local allowlist (see 'ban\_autowallowlist' option).  
+Furthermore the uplink subnet will be added to local allowlist (see 'ban\_autoallowlist' option).  
-Both lists also accept domain names as input to allow IP filtering based on these names. The corresponding IPs (IPv4 & IPv6) will be extracted in a detached background process and added to the sets.
+Both lists also accept domain names as input to allow IP filtering based on these names. The corresponding IPs (IPv4 & IPv6) will be extracted and added to the sets. You can also start the domain lookup separately via /etc/init.d/banip lookup at any time.
 
 **allowlist-only mode**  
 banIP supports an "allowlist only" mode. This option restricts the internet access from/to a small number of secure websites/IPs, and block access from/to the rest of the internet. All IPs and Domains which are _not_ listed in the allowlist are blocked.
 
-**redirect Asterisk security logs to lodg/logread**   
+**redirect Asterisk security logs to lodg/logread**  
 banIP only supports logfile scanning via logread, so to monitor attacks on Asterisk, its security log must be available via logread. To do this, edit '/etc/asterisk/logger.conf' and add the line 'syslog.local0 = security', then run 'asterisk -rx reload logger' to update the running Asterisk configuration.
 
+**send status E-Mails and update the banIP lists via cron job**  
+For a regular, automatic status mailing and update of the used lists on a daily basis set up a cron job, e.g.
+```
+55 03 * * * /etc/init.d/banip report mail
+00 04 * * * /etc/init.d/banip reload
+```
+
 **tweaks for low memory systems**  
 nftables supports the atomic loading of rules/sets/members, which is cool but unfortunately is also very memory intensive. To reduce the memory pressure on low memory systems (i.e. those with 256-512Mb RAM), you should optimize your configuration with the following options:  
 

net/banip/files/banip-functions.sh

@@ -78,6 +78,7 @@ ban_debug="0"
 f_system() {
 	local cpu core
 
+	[ -z "${ban_dev}" ] && ban_cores="$(uci_get banip global ban_cores)"
 	ban_memory="$("${ban_awkcmd}" '/^MemAvailable/{printf "%s",int($2/1000)}' "/proc/meminfo" 2>/dev/null)"
 	ban_ver="$(${ban_ubuscmd} -S call rpc-sys packagelist '{ "all": true }' 2>/dev/null | jsonfilter -ql1 -e '@.packages.banip')"
 	ban_sysver="$(${ban_ubuscmd} -S call system board 2>/dev/null | jsonfilter -ql1 -e '@.model' -e '@.release.description' |
@@ -426,7 +427,7 @@ f_getsub() {
 f_getelements() {
 	local file="${1}"
 
-	[ -s "${file}" ] && printf "%s" "elements={ $(cat "${file}") };"
+	[ -s "${file}" ] && printf "%s" "elements={ $(cat "${file}" 2>/dev/null) };"
 }
 
 # build initial nft file with base table, chains and rules
@@ -975,8 +976,6 @@ f_getstatus() {
 				done
 				json_select ".."
 			fi
-			value="$(printf "%s" "${value}" |
-				awk '{NR=1;max=118;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{printf"%-24s%s\n","",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')"
 			printf "  + %-17s : %s\n" "${key}" "${value:-"-"}"
 		done
 	else
@@ -987,7 +986,7 @@ f_getstatus() {
 # domain lookup
 #
 f_lookup() {
-	local cnt list domain lookup ip start_time end_time duration cnt_domain="0" cnt_ip="0" feed="${1}"
+	local cnt list domain lookup ip elementsv4 elementsv6 start_time end_time duration cnt_domain="0" cnt_ip="0" feed="${1}"
 
 	start_time="$(date "+%s")"
 	if [ "${feed}" = "allowlist" ]; then
@@ -1004,32 +1003,36 @@ f_lookup() {
 			else
 				if { [ "${feed}" = "allowlist" ] && ! "${ban_grepcmd}" -q "^${ip}" "${ban_allowlist}"; } ||
 					{ [ "${feed}" = "blocklist" ] && ! "${ban_grepcmd}" -q "^${ip}" "${ban_blocklist}"; }; then
-					cnt_ip="$((cnt_ip + 1))"
 					if [ "${ip##*:}" = "${ip}" ]; then
-						if ! "${ban_nftcmd}" add element inet banIP "${feed}v4" "{ ${ip} }" >/dev/null 2>&1; then
+						elementsv4="${elementsv4} ${ip},"
-							f_log "info" "failed to add IP '${ip}' (${domain}) to ${feed}v4 set"
-							continue
-						fi
 					else
-						if ! "${ban_nftcmd}" add element inet banIP "${feed}v6" "{ ${ip} }" >/dev/null 2>&1; then
+						elementsv6="${elementsv6} ${ip},"
-							f_log "info" "failed to add IP '${ip}' (${domain}) to ${feed}v6 set"
-							continue
-						fi
 					fi
 					if [ "${feed}" = "allowlist" ] && [ "${ban_autoallowlist}" = "1" ]; then
 						printf "%-42s%s\n" "${ip}" "# '${domain}' added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_allowlist}"
 					elif [ "${feed}" = "blocklist" ] && [ "${ban_autoblocklist}" = "1" ]; then
 						printf "%-42s%s\n" "${ip}" "# '${domain}' added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_blocklist}"
 					fi
+					cnt_ip="$((cnt_ip + 1))"
 				fi
 			fi
 		done
 		cnt_domain="$((cnt_domain + 1))"
 	done
+	if [ -n "${elementsv4}" ]; then
+		if ! "${ban_nftcmd}" add element inet banIP "${feed}v4" "{ ${elementsv4} }" >/dev/null 2>&1; then
+			f_log "info" "failed to add lookup file to ${feed}v4 set"
+		fi
+	fi
+	if [ -n "${elementsv6}" ]; then
+		if ! "${ban_nftcmd}" add element inet banIP "${feed}v6" "{ ${elementsv6} }" >/dev/null 2>&1; then
+			f_log "info" "failed to add lookup file to ${feed}v6 set"
+		fi
+	fi
 	end_time="$(date "+%s")"
 	duration="$(((end_time - start_time) / 60))m $(((end_time - start_time) % 60))s"
 
-	f_log "debug" "f_lookup  ::: name: ${feed}, cnt_domain: ${cnt_domain}, cnt_ip: ${cnt_ip}, duration: ${duration}"
+	f_log "info" "Lookup summary for the local ${feed}: Domains processed: ${cnt_domain}, IPs added: ${cnt_ip}, Duration: ${duration}"
 }
 
 # table statistics
@@ -1198,7 +1201,7 @@ f_report() {
 # set search
 #
 f_search() {
-	local table_sets ip proto run_search search="${1}"
+	local set table_sets ip proto run_search hold cnt search="${1}"
 
 	if [ -n "${search}" ]; then
 		ip="$(printf "%s" "${search}" | "${ban_awkcmd}" 'BEGIN{RS="(([0-9]{1,3}\\.){3}[0-9]{1,3})+"}{printf "%s",RT}')"
@@ -1215,14 +1218,15 @@ f_search() {
 		return
 	fi
 	printf "%s\n%s\n%s\n" ":::" "::: banIP Search" ":::"
-	printf "%s\n" "    Looking for IP '${ip}' on $(date "+%Y-%m-%d %H:%M:%S")"
+	printf "    %s\n" "Looking for IP '${ip}' on $(date "+%Y-%m-%d %H:%M:%S")"
-	printf "%s\n" "    ---"
+	printf "    %s\n" "---"
 	cnt="1"
 	run_search="/var/run/banIP.search"
 	for set in ${table_sets}; do
+		[ -f "${run_search}" ] && break
 		(
 			if "${ban_nftcmd}" get element inet banIP "${set}" "{ ${ip} }" >/dev/null 2>&1; then
-				printf "%s\n" "    IP found in Set '${set}'"
+				printf "    %s\n" "IP found in Set '${set}'"
 				: >"${run_search}"
 			fi
 		) &
@@ -1231,11 +1235,8 @@ f_search() {
 		cnt="$((cnt + 1))"
 	done
 	wait
-	if [ ! -f "${run_search}" ]; then
+	[ ! -f "${run_search}" ] && printf "    %s\n" "IP not found"
-		printf "%s\n" "    IP not found"
+	rm -f "${run_search}"
-	else
-		rm -f "${run_search}"
-	fi
 }
 
 # set survey
@@ -1243,16 +1244,15 @@ f_search() {
 f_survey() {
 	local set_elements set="${1}"
 
-	[ -n "${set}" ] && set_elements="$("${ban_nftcmd}" -j list set inet banIP "${set}" 2>/dev/null | jsonfilter -qe '@.nftables[*].set.elem[*]')"
+	if [ -z "${set}" ]; then
-
-	if [ -z "${set}" ] || [ -z "${set_elements}" ]; then
 		printf "%s\n%s\n%s\n" ":::" "::: no valid survey input" ":::"
 		return
 	fi
+	[ -n "${set}" ] && set_elements="$("${ban_nftcmd}" -j list set inet banIP "${set}" 2>/dev/null | jsonfilter -qe '@.nftables[*].set.elem[*]')"
 	printf "%s\n%s\n%s\n" ":::" "::: banIP Survey" ":::"
-	printf "%s\n" "    List the elements of Set '${set}' on $(date "+%Y-%m-%d %H:%M:%S")"
+	printf "    %s\n" "List the elements of Set '${set}' on $(date "+%Y-%m-%d %H:%M:%S")"
-	printf "%s\n" "    ---"
+	printf "    %s\n" "---"
-	printf "%s\n" "${set_elements}"
+	[ -n "${set_elements}" ] && printf "%s\n" "${set_elements}" || printf "    %s\n" "empty set"
 }
 
 # send status mails

net/banip/files/banip-service.sh

@@ -124,21 +124,25 @@ for feed in allowlist ${ban_feed} blocklist; do
 	fi
 done
 wait
-
-# start background domain lookup
-#
-f_log "info" "start detached banIP domain lookup"
-(f_lookup "allowlist") &
-hold="$((cnt % ban_cores))"
-[ "${hold}" = "0" ] && wait
-(f_lookup "blocklist") &
-
-# end processing
-#
 f_rmset
 f_rmdir "${ban_tmpdir}"
 f_genstatus "active"
 f_log "info" "finished banIP download processes"
+
+# start domain lookup
+#
+f_log "info" "start banIP domain lookup"
+cnt="1"
+for list in allowlist blocklist; do
+	(f_lookup "${list}") &
+	hold="$((cnt % ban_cores))"
+	[ "${hold}" = "0" ] && wait
+	cnt="$((cnt + 1))"
+done
+wait
+
+# end processing
+#
 if [ "${ban_mailnotification}" = "1" ] && [ -n "${ban_mailreceiver}" ] && [ -x "${ban_mailcmd}" ]; then
 	(
 		sleep ${ban_triggerdelay}

net/banip/files/banip.init

@@ -12,6 +12,7 @@ USE_PROCD=1
 extra_command "report" "[text|json|mail] Print banIP related set statistics"
 extra_command "search" "[<IPv4 address>|<IPv6 address>] Check if an element exists in a banIP set"
 extra_command "survey" "[<set name>] List all elements of a given banIP set"
+extra_command "lookup" "Lookup the IPs of domain names in the local lists and update them"
 
 ban_init="/etc/init.d/banip"
 ban_service="/usr/bin/banip-service.sh"
@@ -20,10 +21,10 @@ ban_pidfile="/var/run/banip.pid"
 ban_lock="/var/run/banip.lock"
 
 [ "${action}" = "boot" ] && /etc/init.d/banip running && exit 0
-[ "${action}" = "stop" ] && ! /etc/init.d/banip running && exit 0
+{ [ "${action}" = "stop" ] || [ "${action}" = "lookup" ]; } && ! /etc/init.d/banip running && exit 0
-[ ! -r "${ban_funlib}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "stop" ] || [ "${action}" = "report" ] || [ "${action}" = "search" ] || [ "${action}" = "survey" ] || [ "${action}" = "status" ]; } && exit 1
+[ ! -r "${ban_funlib}" ] && [ "${action}" != "boot" ] && exit 1
-[ -d "${ban_lock}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ]; } && exit 1
+[ -d "${ban_lock}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "lookup" ]; } && exit 1
-[ ! -d "${ban_lock}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ]; } && mkdir -p "${ban_lock}"
+[ ! -d "${ban_lock}" ] && { [ "${action}" = "start" ] || [ "${action}" = "restart" ] || [ "${action}" = "reload" ] || [ "${action}" = "lookup" ]; } && mkdir -p "${ban_lock}"
 
 boot() {
 	: >"${ban_pidfile}"
@@ -61,7 +62,7 @@ stop_service() {
 	"${ban_nftcmd}" delete table inet banIP >/dev/null 2>&1
 	f_genstatus "stopped"
 	f_rmpid
-	rm -rf "${ban_lock}"
+	[ "${action}" = "stop" ] && rm -rf "${ban_lock}"
 }
 
 restart() {
@@ -74,10 +75,8 @@ status() {
 }
 
 status_service() {
-	local actual="${1}"
-
 	[ -z "$(command -v "f_system")" ] && . "${ban_funlib}"
-	[ -n "${actual}" ] && f_actual || f_getstatus
+	f_getstatus
 }
 
 report() {
@@ -95,6 +94,20 @@ survey() {
 	f_survey "${1}"
 }
 
+lookup() {
+	local list hold cnt="1"
+
+	[ -z "$(command -v "f_system")" ] && . "${ban_funlib}"
+	for list in allowlist blocklist; do
+		(f_lookup "${list}") &
+		hold="$((cnt % ban_cores))"
+		[ "${hold}" = "0" ] && wait
+		cnt="$((cnt + 1))"
+	done
+	wait
+	rm -rf "${ban_lock}"
+}
+
 service_triggers() {
 	local iface trigger trigger_action delay
 

net/banip/files/banip.tpl

@@ -6,7 +6,7 @@
 #
 local banip_info report_info log_info system_info mail_text
 
-banip_info="$(/etc/init.d/banip status 2>/dev/null)"
+banip_info="$(/etc/init.d/banip status 2>/dev/null | awk '{NR=1;max=140;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{print substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')"
 report_info="$(cat ${ban_reportdir}/ban_report.txt 2>/dev/null)"
 log_info="$("${ban_logreadcmd}" -l 100 -e "banIP/" 2>/dev/null | awk '{NR=1;max=140;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{print substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')"
 system_info="$(
@@ -14,11 +14,17 @@ system_info="$(
 	ubus call system board | awk 'BEGIN{FS="[{}\"]"}{if($2=="kernel"||$2=="hostname"||$2=="system"||$2=="model"||$2=="description")printf "  + %-12s: %s\n",$2,$4}'
 )"
 
-# mail body
+# content header
 #
 mail_text="$(printf "%s\n" "<html><body><pre style='display:block;font-family:monospace;font-size:1rem;padding:20;background-color:#f3eee5;white-space:pre'>")"
+
+# content body
+#
 mail_text="$(printf "%s\n" "${mail_text}\n<strong>++\n++ System Information ++\n++</strong>\n${system_info:-"-"}")"
 mail_text="$(printf "%s\n" "${mail_text}\n\n<strong>++\n++ banIP Status ++\n++</strong>\n${banip_info:-"-"}")"
-mail_text="$(printf "%s\n" "${mail_text}\n\n<strong>++\n++ banIP Report ++\n++</strong>\n${report_info:-"-"}")"
+[ -n "${report_info}" ] && mail_text="$(printf "%s\n" "${mail_text}\n\n<strong>++\n++ banIP Report ++\n++</strong>\n${report_info}")"
-mail_text="$(printf "%s\n" "${mail_text}\n\n<strong>++\n++ Logfile Information ++\n++</strong>\n${log_info}")"
+[ -n "${log_info}" ] && mail_text="$(printf "%s\n" "${mail_text}\n\n<strong>++\n++ Logfile Information ++\n++</strong>\n${log_info}")"
+
+# content footer
+#
 mail_text="$(printf "%s\n" "${mail_text}</pre></body></html>")"