luci-app-adblock: sync with release 3.6.0

* add adblock dns query reporting via tcpdump (see readme for details)
* fix tld compression on low memory systems (< 64 MB)
* fix various small issues

Signed-off-by: Dirk Brenken <dev@brenken.org>
This commit is contained in:
Dirk Brenken 2018-12-19 17:33:29 +01:00
parent efb4d7c40a
commit 87ec92d1ed
5 changed files with 419 additions and 72 deletions

View file

@ -16,7 +16,8 @@ function index()
end
entry({"admin", "services", "adblock"}, firstchild(), _("Adblock"), 30).dependent = false
entry({"admin", "services", "adblock", "tab_from_cbi"}, cbi("adblock/overview_tab", {hideresetbtn=true, hidesavebtn=true}), _("Overview"), 10).leaf = true
entry({"admin", "services", "adblock", "log"}, template("adblock/logread"), _("View Logfile"), 20).leaf = true
entry({"admin", "services", "adblock", "report"}, template("adblock/report"), _("DNS Query Report"), 20).leaf = true
entry({"admin", "services", "adblock", "log"}, template("adblock/logread"), _("Logfile"), 30).leaf = true
entry({"admin", "services", "adblock", "advanced"}, firstchild(), _("Advanced"), 100)
entry({"admin", "services", "adblock", "advanced", "blacklist"}, form("adblock/blacklist_tab"), _("Edit Blacklist"), 110).leaf = true
entry({"admin", "services", "adblock", "advanced", "whitelist"}, form("adblock/whitelist_tab"), _("Edit Whitelist"), 120).leaf = true
@ -24,17 +25,49 @@ function index()
entry({"admin", "services", "adblock", "advanced", "query"}, template("adblock/query"), _("Query domains"), 140).leaf = true
entry({"admin", "services", "adblock", "advanced", "result"}, call("queryData"), nil, 150).leaf = true
entry({"admin", "services", "adblock", "logread"}, call("logread"), nil).leaf = true
entry({"admin", "services", "adblock", "status"}, call("status_update"), nil).leaf = true
entry({"admin", "services", "adblock", "status_update"}, call("status_update"), nil).leaf = true
entry({"admin", "services", "adblock", "report_json"}, call("report_json"), nil).leaf = true
entry({"admin", "services", "adblock", "report_text"}, call("report_text"), nil).leaf = true
entry({"admin", "services", "adblock", "action"}, call("adb_action"), nil).leaf = true
end
function adb_action(name)
function adb_action(name, domain)
if name == "do_suspend" then
luci.sys.call("/etc/init.d/adblock suspend >/dev/null 2>&1")
elseif name == "do_resume" then
luci.sys.call("/etc/init.d/adblock resume >/dev/null 2>&1")
elseif name == "do_refresh" then
luci.sys.call("/etc/init.d/adblock reload >/dev/null 2>&1")
local pid_file = "/var/run/adblock.pid"
if nixio.fs.access(pid_file) then
repeat
nixio.nanosleep(1)
until nixio.fs.readfile(pid_file) == ""
end
elseif name == "do_report" then
luci.sys.call("/etc/init.d/adblock report false >/dev/null 2>&1")
local rep_dir = uci:get("adblock", "extra", "adb_repdir") or "/tmp"
repeat
nixio.nanosleep(1)
until not nixio.fs.access(rep_dir.. "/adb_report.raw")
elseif name == "add_blacklist" then
local file = uci:get("adblock", "blacklist", "adb_src") or "/etc/adblock/adblock.blacklist"
if nixio.fs.access(file) then
local blacklist = nixio.fs.readfile(file)
if not string.find(blacklist, domain, 1, plain)
then
nixio.fs.writefile(file, blacklist.. domain.. "\n")
end
end
elseif name == "add_whitelist" then
local file = uci:get("adblock", "global", "adb_whitelist") or "/etc/adblock/adblock.whitelist"
if nixio.fs.access(file) then
local whitelist = nixio.fs.readfile(file)
if not string.find(whitelist, domain, 1, plain)
then
nixio.fs.writefile(file, whitelist.. domain.. "\n")
end
end
end
luci.http.prepare_content("text/plain")
luci.http.write("0")
@ -53,6 +86,43 @@ function status_update()
end
end
function report_json()
local rep_dir
local rep_file
local content
rep_dir = uci:get("adblock", "extra", "adb_repdir") or "/tmp"
rep_file = rep_dir.. "/adb_report.json"
http.prepare_content("application/json")
if nixio.fs.access(rep_file) then
content = json.parse(nixio.fs.readfile(rep_file) or "")
http.write_json(content)
else
http.write_json("{}")
end
end
function report_text()
local file
local rep_dir
local rep_file
local content
rep_dir = uci:get("adblock", "extra", "adb_repdir") or "/tmp"
rep_file = rep_dir.. "/adb_report"
http.prepare_content("text/plain")
if nixio.fs.access(rep_file) then
file = io.open(rep_file, "r")
content = file:read("*all")
file:close()
http.write(content)
else
http.write("")
end
end
function logread()
local content

View file

@ -1,14 +1,16 @@
-- Copyright 2017-2018 Dirk Brenken (dev@brenken.org)
-- This is free software, licensed under the Apache License, Version 2.0
local fs = require("nixio.fs")
local uci = require("luci.model.uci").cursor()
local util = require("luci.util")
local dump = util.ubus("network.interface", "dump", {})
local fs = require("nixio.fs")
local uci = require("luci.model.uci").cursor()
local util = require("luci.util")
local net = require "luci.model.network".init()
local sys = require("luci.sys")
local devices = sys.net:devices()
m = Map("adblock", translate("Adblock"),
translate("Configuration of the adblock package to block ad/abuse domains by using DNS. ")
.. translatef("For further information "
..translatef("For further information "
.. "<a href=\"%s\" target=\"_blank\">"
.. "check the online documentation</a>", "https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md"))
@ -22,7 +24,7 @@ o1.rmempty = false
o2 = s:option(ListValue, "adb_dns", translate("DNS Backend (DNS Directory)"),
translate("List of supported DNS backends with their default list export directory. ")
.. translate("To overwrite the default path use the 'DNS Directory' option in the extra section below."))
..translate("To overwrite the default path use the 'DNS Directory' option in the extra section below."))
o2:value("dnsmasq", "dnsmasq (/tmp)")
o2:value("unbound", "unbound (/var/lib/unbound)")
o2:value("named", "named (/var/lib/bind)")
@ -44,14 +46,18 @@ o3.rmempty = false
o4 = s:option(ListValue, "adb_trigger", translate("Startup Trigger"),
translate("List of available network interfaces. Usually the startup will be triggered by the 'wan' interface. ")
.. translate("Choose 'none' to disable automatic startups, 'timed' to use a classic timeout (default 30 sec.) or select another trigger interface."))
..translate("Choose 'none' to disable automatic startups, 'timed' to use a classic timeout (default 30 sec.) or select another trigger interface."))
o4:value("none")
o4:value("timed")
if dump then
local i, v
for i, v in ipairs(dump.interface) do
if v.interface ~= "loopback" then
o4:value(v.interface)
for _, dev in ipairs(devices) do
if dev ~= "lo" then
local iface = net:get_interface(dev)
if iface then
iface = iface:get_networks() or {}
for k, v in pairs(iface) do
iface[k] = iface[k].sid
o4:value(iface[k], iface[k].. " (" ..dev.. ")")
end
end
end
end
@ -94,90 +100,117 @@ e = m:section(NamedSection, "extra", "adblock", translate("Extra Options"),
e1 = e:option(Flag, "adb_debug", translate("Verbose Debug Logging"),
translate("Enable verbose debug logging in case of any processing error."))
e1.default = e1.disabled
e1.rmempty = false
e2 = e:option(Flag, "adb_nice", translate("Low Priority Service"),
translate("Set the nice level to 'low priority' and the adblock background processing will take less resources from the system. ")
..translate("This change requires a manual service stop/re-start to take effect."))
e2.default = e2.disabled
e2.disabled = "0"
e2.enabled = "10"
e2.rmempty = false
e3 = e:option(Flag, "adb_forcedns", translate("Force Local DNS"),
translate("Redirect all DNS queries from 'lan' zone to the local resolver, apply to udp and tcp protocol on ports 53, 853 and 5353."))
e3.default = e3.disabled
e3.rmempty = false
e4 = e:option(Flag, "adb_forcesrt", translate("Force Overall Sort"),
translate("Enable memory intense overall sort / duplicate removal on low memory devices (&lt; 64 MB free RAM)"))
e4.default = e4.disabled
e4 = e:option(Flag, "adb_backup", translate("Enable Blocklist Backup"),
translate("Create compressed blocklist backups, they will be used in case of download errors or during startup in backup mode."))
e4.rmempty = false
e5 = e:option(Flag, "adb_backup", translate("Enable Blocklist Backup"),
translate("Create compressed blocklist backups, they will be used in case of download errors or during startup in backup mode."))
e5.default = e5.disabled
e5.rmempty = false
e6 = e:option(Value, "adb_backupdir", translate("Backup Directory"),
translate("Target directory for adblock backups. Please use only non-volatile disks, e.g. an external usb stick."))
e6:depends("adb_backup", 1)
e6.datatype = "directory"
e6.default = "/mnt"
e5 = e:option(Value, "adb_backupdir", translate("Backup Directory"),
translate("Target directory for adblock backups. Please use only a non-volatile disk, e.g. an external usb stick."))
e5:depends("adb_backup", 1)
e5.datatype = "directory"
e5.default = "/mnt"
e5.rmempty = true
e7 = e:option(Flag, "adb_backup_mode", translate("Backup Mode"),
e6 = e:option(Flag, "adb_backup_mode", translate("Backup Mode"),
translate("Do not automatically update blocklists during startup, use blocklist backups instead."))
e7:depends("adb_backup", 1)
e7.default = e7.disabled
e7.rmempty = true
e6:depends("adb_backup", 1)
e6.rmempty = true
e8 = e:option(Value, "adb_maxqueue", translate("Max. Download Queue"),
translate("Size of the download queue to handle downloads &amp; list processing in parallel (default '4'). ")
.. translate("For further performance improvements you can raise this value, e.g. '8' or '16' should be safe."))
e8.default = 4
e8.datatype = "range(1,32)"
e7 = e:option(Value, "adb_maxqueue", translate("Max. Download Queue"),
translate("Size of the download queue to handle downloads &amp; list processing in parallel (default '8'). ")
..translate("For further performance improvements you can raise this value, e.g. '8' or '16' should be safe."))
e7.default = 8
e7.datatype = "range(1,32)"
e7.rmempty = false
e8 = e:option(Flag, "adb_report", translate("Enable DNS Query Report"),
translate("Gather dns related network traffic via tcpdump to provide a DNS Query Report on demand. ")
..translate("Please note: this needs manual 'tcpdump-mini' package installation."))
e8.rmempty = false
e9 = e:option(Flag, "adb_jail", translate("'Jail' Blocklist Creation"),
translate("Builds an additional 'Jail' list (/tmp/adb_list.jail) to block access to all domains except those listed in the whitelist file. ")
.. translate("You can use this restrictive blocklist manually e.g. for guest wifi or kidsafe configurations."))
e9.default = e9.disabled
e9.rmempty = true
e9 = e:option(Flag, "adb_dnsflush", translate("Flush DNS Cache"),
translate("Flush DNS Cache after adblock processing."))
e9.default = e9.disabled
e9 = e:option(Value, "adb_repdir", translate("Report Directory"),
translate("Target directory for dns related report files. Please use preferably a non-volatile disk, e.g. an external usb stick."))
e9:depends("adb_report", 1)
e9.datatype = "directory"
e9.default = "/tmp"
e9.rmempty = true
e10 = e:option(Flag, "adb_notify", translate("Email Notification"),
translate("Send notification emails in case of a processing error or if domain count is &le; 0. ")
.. translate("Please note: this needs additional 'msmtp' package installation and setup."))
e10.default = e10.disabled
.. translate("Please note: this needs manual 'msmtp' package installation and setup."))
e10.rmempty = true
e11 = e:option(Value, "adb_notifycnt", translate("Email Notification Count"),
-- Optional Extra Options
e20 = e:option(Flag, "adb_jail", translate("'Jail' Blocklist Creation"),
translate("Builds an additional 'Jail' list (/tmp/adb_list.jail) to block access to all domains except those listed in the whitelist file. ")
.. translate("You can use this restrictive blocklist e.g. for guest wifi or kidsafe configurations."))
e20.optional = true
e20.default = nil
e21 = e:option(Value, "adb_notifycnt", translate("Email Notification Count"),
translate("Raise the minimum email notification count, to get emails if the overall count is less or equal to the given limit (default 0), ")
.. translate("e.g. to receive an email notification with every adblock update set this value to 150000."))
e11.default = 0
e11.datatype = "min(0)"
e11.optional = true
e21.default = 0
e21.datatype = "min(0)"
e21.optional = true
e12 = e:option(Value, "adb_dnsdir", translate("DNS Directory"),
e22 = e:option(Value, "adb_dnsdir", translate("DNS Directory"),
translate("Target directory for the generated blocklist 'adb_list.overall'."))
e12.datatype = "directory"
e12.optional = true
e22.datatype = "directory"
e22.optional = true
e13 = e:option(Value, "adb_whitelist", translate("Whitelist File"),
e23 = e:option(Value, "adb_whitelist", translate("Whitelist File"),
translate("Full path to the whitelist file."))
e13.datatype = "file"
e13.default = "/etc/adblock/adblock.whitelist"
e13.optional = true
e23.datatype = "file"
e23.default = "/etc/adblock/adblock.whitelist"
e23.optional = true
e14 = e:option(Value, "adb_triggerdelay", translate("Trigger Delay"),
e24 = e:option(Value, "adb_triggerdelay", translate("Trigger Delay"),
translate("Additional trigger delay in seconds before adblock processing begins."))
e14.datatype = "range(1,60)"
e14.optional = true
e24.datatype = "range(1,60)"
e24.optional = true
e25 = e:option(Flag, "adb_dnsflush", translate("Flush DNS Cache"),
translate("Flush DNS Cache after adblock processing."))
e25.optional = true
e25.default = nil
e26 = e:option(ListValue, "adb_repiface", translate("Report Interface"),
translate("Reporting interface used by tcpdump (default 'br-lan')."))
for _, dev in ipairs(devices) do
if dev ~= "lo" then
e26:value(dev)
end
end
e26.optional = true
e28 = e:option(Value, "adb_repchunkcnt", translate("Report Chunk Count"),
translate("Report chunk count used by tcpdump (default '5')."))
e28.datatype = "range(1,10)"
e28.optional = true
e29 = e:option(Value, "adb_repchunksize", translate("Report Chunk Size"),
translate("Report chunk size used by tcpdump in MB (default '1')."))
e29.datatype = "range(1,10)"
e29.optional = true
e30 = e:option(Flag, "adb_forcesrt", translate("Force Overall Sort"),
translate("Enable memory intense overall sort / duplicate removal on low memory devices (&lt; 64 MB free RAM)"))
e30.optional = true
e30.default = nil
return m

View file

@ -1,8 +1,8 @@
<style type="text/css">
textarea
{
width: 100%;
height: 450px;
width: 100% !important;
height: 450px !important;
border: 1px solid #cccccc;
padding: 5px;
font-size: 12px;
@ -16,8 +16,8 @@
select[readonly],
textarea[readonly]
{
width: 100%;
height: 450px;
width: 100% !important;
height: 450px !important;
border: 1px solid #cccccc;
padding: 5px;
font-size: 12px;
@ -33,7 +33,7 @@
.cbi-section-table-row,
.tr[data-title]::before
{
text-align: left;
text-align: left !important;
vertical-align: top;
margin-left: 0px;
padding-left: 2px;

View file

@ -0,0 +1,244 @@
<%#
Copyright 2017-2018 Dirk Brenken (dev@brenken.org)
This is free software, licensed under the Apache License, Version 2.0
-%>
<%+header%>
<%+adblock/adblock_css%>
<script type="text/javascript">
//<![CDATA[
function report_json(json)
{
document.getElementById("value_1").innerHTML = json.data.start_date + ", " + json.data.start_time || "-";
document.getElementById("value_2").innerHTML = json.data.end_date + ", " + json.data.end_time || "-";
document.getElementById("value_3").innerHTML = json.data.total || "-";
document.getElementById("value_4").innerHTML = json.data.blocked + " (" + json.data.percent + ")" || "-";
var a_cnt;
var b_cnt;
var c_cnt;
var a_val;
var b_val;
var c_val;
var s = '<div class="tr table-titles">';
s += '<div class="th left"><%:Count%></div>';
s += '<div class="th left"><%:Name / IP-Address%></div>';
s += '<div class="th left"><%:Count%></div>';
s += '<div class="th left"><%:Domain%></div>';
s += '<div class="th left"><%:Count%></div>';
s += '<div class="th left"><%:Blocked Domain%></div></div>';
for(var i=0;i<10;i++)
{
if (i < json.top_clients.length)
{
a_cnt = json.top_clients[i].count;
a_val = json.top_clients[i].address;
}
else
{
a_cnt = "-";
a_val = "";
}
if (i < json.top_domains.length)
{
b_cnt = json.top_domains[i].count;
b_val = json.top_domains[i].address;
}
else
{
b_cnt = "-";
b_val = "";
}
if (i < json.top_blocked.length)
{
c_cnt = json.top_blocked[i].count;
c_val = json.top_blocked[i].address;
}
else
{
c_cnt = "-";
c_val = "";
}
s += '<div class="tr left">';
s += '<div class="td left" data-title="<%:Count%>">' + a_cnt + '</div>';
s += '<div class="td left" data-title="<%:Name / IP-Address%>">' + a_val + '</div>';
s += '<div class="td left" data-title="<%:Count%>">' + b_cnt + '</div>';
s += '<div class="td left" data-title="<%:Domain%>">' + b_val + '</div>';
s += '<div class="td left" data-title="<%:Count%>">' + c_cnt + '</div>';
s += '<div class="td left" data-title="<%:Blocked Domain%>">' + c_val + '</div></div>';
}
document.getElementById("value_5").innerHTML = s;
}
function report_text(text)
{
var s = '<div class="tr table-titles">';
s += '<div class="th left"><%:Date%></div>';
s += '<div class="th left"><%:Time%></div>';
s += '<div class="th left"><%:Client%></div>';
s += '<div class="th left"><%:Domain%></div>';
s += '<div class="th left"><%:Answer%></div>';
s += '<div class="th left"><%:Action%></div></div>';
var btn;
var record;
var title_arr = ["<%:Date%>", "<%:Time%>", "<%:Client%>", "<%:Domain%>", "<%:Answer%>", "<%:Action%>"];
var array = text.split("\n", 50);
for (var i=0;i<array.length;i++)
{
record = array[i].split("\t");
if (record[0] && record[0] != "undefined")
{
s += '<div class="tr left">';
for (var j=0;j<5;j++)
{
s += '<div class="td left" data-title="' + title_arr[j] + '">' + record[j] + '</div>';
if (record[4] === "NX")
{
btn = '<div class="td left" data-title="' + title_arr[5] + '"><input type="button" class="cbi-button cbi-button-edit" name="add_whitelist,' + record[3] + '" value="<%:Whitelist%>" onclick="btn_action(this)" /></div>';
}
else
{
btn = '<div class="td left" data-title="' + title_arr[5] + '"><input type="button" class="cbi-button cbi-button-remove" name="add_blacklist,' + record[3] + '" value="<%:Blacklist%>" onclick="btn_action(this)" /></div>';
}
}
s += btn + '</div>'
}
}
document.getElementById("value_6").innerHTML = s;
}
function btn_action(value)
{
var action = value.name.split(",");
if (action[0] === "do_report")
{
var btn = document.getElementById("btn");
var btn_running = document.getElementById("btn_running");
btn.disabled = true;
running(btn_running, 1);
action[1] = "-"
}
new XHR.get('<%=luci.dispatcher.build_url("admin", "services", "adblock")%>/action/' + action[0] + "/" + action[1], null,
function(x)
{
if (!x)
{
return;
}
if (action[0] === "do_report")
{
running(btn_running, 0);
btn.disabled = false;
}
});
}
function running(element, state)
{
if (state === 1)
{
var running_html = '<img src="<%=resource%>/icons/loading.gif" alt="<%:Loading%>" width="16" height="16" style="vertical-align:middle" />';
element.innerHTML = running_html;
}
else
{
element.innerHTML = '';
}
}
XHR.get('<%=luci.dispatcher.build_url("admin", "services", "adblock", "report_json")%>', null,
function(x, json_info)
{
if (!x || !json_info || !json_info.data)
{
running(btn_running, 0);
btn.disabled = false;
return;
}
report_json(json_info);
});
XHR.poll(5, '<%=luci.dispatcher.build_url("admin", "services", "adblock", "report_json")%>', null,
function(x, json_info)
{
if (!x || !json_info || !json_info.data)
{
running(btn_running, 0);
btn.disabled = false;
return;
}
report_json(json_info);
});
XHR.get('<%=luci.dispatcher.build_url("admin", "services", "adblock", "report_text")%>', null,
function(x)
{
if (!x || !x.responseText)
{
return;
}
report_text(x.responseText);
});
XHR.poll(5, '<%=luci.dispatcher.build_url("admin", "services", "adblock", "report_text")%>', null,
function(x)
{
if (!x || !x.responseText)
{
return;
}
report_text(x.responseText);
});
//]]>
</script>
<div class="cbi-section">
<div class="cbi-value" id="status_1">
<label class="cbi-value-title" for="status_1"><%:Start Date%></label>
<div class="cbi-value-field">
<span class="runtime" id="value_1">-</span>
</div>
</div>
<div class="cbi-value" id="status_2">
<label class="cbi-value-title" for="status_2"><%:End Date%></label>
<div class="cbi-value-field">
<span class="runtime" id="value_2">-</span>
</div>
</div>
<div class="cbi-value" id="status_3">
<label class="cbi-value-title" for="status_3"><%:Total DNS Queries%></label>
<div class="cbi-value-field">
<span class="runtime" id="value_3">-</span>
</div>
</div>
<div class="cbi-value" id="status_4">
<label class="cbi-value-title" for="status_4"><%:Blocked DNS Queries%></label>
<div class="cbi-value-field">
<span class="runtime" id="value_4">-</span>
</div>
</div>
<br />
<div id="button">
<input type="button" class="cbi-button cbi-button-action important" id="btn" name="do_report" value="<%:Refresh Report%>" onclick="btn_action(this)" />
<span id="btn_running" class="btn_running"></span>
</div>
</div>
<br />
<div class="cbi-section">
<h3><%:Top 10 Reporting%></h3>
<div class="table" id="value_5"></div>
</div>
<br />
<div class="cbi-section">
<h3><%:Latest DNS Queries%></h3>
<div class="table" id="value_6"></div>
</div>
<%+footer%>

View file

@ -112,7 +112,7 @@ This is free software, licensed under the Apache License, Version 2.0
document.getElementById("value_6").innerHTML = "-";
}
XHR.get('<%=luci.dispatcher.build_url("admin", "services", "adblock", "status")%>', null,
XHR.get('<%=luci.dispatcher.build_url("admin", "services", "adblock", "status_update")%>', null,
function(x, json_info)
{
if (!x || !json_info || !json_info.data)
@ -123,7 +123,7 @@ This is free software, licensed under the Apache License, Version 2.0
status_update(json_info);
});
XHR.poll(5, '<%=luci.dispatcher.build_url("admin", "services", "adblock", "status")%>', null,
XHR.poll(5, '<%=luci.dispatcher.build_url("admin", "services", "adblock", "status_update")%>', null,
function(x, json_info)
{
if (!x || !json_info || !json_info.data)