[PATCH] Allow smarter node creation based on visibility during createtree

As I've brought up on the mailing list thread "High latency caused by full tree creation", there is a large amount of delay per LuCI request which is spent building the node tree in createtree(). Most nodes created aren't 
needed for the view presented to the user and only serve to consume memory and CPU time during a page load.

My idea is to provide an easy mechanism for index()ers to determine which needs to be created and what isn't. Due to the constraints of the standard LuCI web interface, this optimization needs to establish a few rules:

 * The page requested must have its node created
 * All parents of the page being requested must be created, so the children inherit the track
 * All the top-level nodes "Status", "System", "Services", "Network" (and others added by extensions) must be created in order to have their top-level tabs in the UI
 * All peers of second-level nodes need to be created as well for the same reason, to display their links on the subindexes 
                
To make this easy to implement in each controller, the attached patch adds an "inreq" field to each created node to indicate if it lies on the request path. To satisfy the "top level node" requirement, we always 
add the top level node, then check its inreq property if the top-level node is not "in request", then the controller can exit index() early.

libs/web/luasrc/dispatcher.lua

@@ -130,6 +130,8 @@ function httpdispatch(request, prefix)
 
 	local r = {}
 	context.request = r
+	context.urltoken = {}
+	
 	local pathinfo = http.urldecode(request:getenv("PATH_INFO") or "", true)
 
 	if prefix then
@@ -138,8 +140,18 @@ function httpdispatch(request, prefix)
 		end
 	end
 
+	local tokensok = true
 	for node in pathinfo:gmatch("[^/]+") do
-		r[#r+1] = node
+		local tkey, tval
+		if tokensok then
+			tkey, tval = node:match(";(%w+)=([a-fA-F0-9]*)")
+		end
+		if tkey then
+			context.urltoken[tkey] = tval
+		else
+			tokensok = false
+			r[#r+1] = node
+		end
 	end
 
 	local stat, err = util.coxpcall(function()
@@ -157,7 +169,6 @@ function dispatch(request)
 	--context._disable_memtrace = require "luci.debug".trap_memtrace("l")
 	local ctx = context
 	ctx.path = request
-	ctx.urltoken   = ctx.urltoken or {}
 
 	local conf = require "luci.config"
 	assert(conf.main,
@@ -187,34 +198,23 @@ function dispatch(request)
 	ctx.args = args
 	ctx.requestargs = ctx.requestargs or args
 	local n
-	local t = true
 	local token = ctx.urltoken
 	local preq = {}
 	local freq = {}
 
 	for i, s in ipairs(request) do
-		local tkey, tval
-		if t then
-			tkey, tval = s:match(";(%w+)=([a-fA-F0-9]*)")
+		preq[#preq+1] = s
+		freq[#freq+1] = s
+		c = c.nodes[s]
+		n = i
+		if not c then
+			break
 		end
 
-		if tkey then
-			token[tkey] = tval
-		else
-			t = false
-			preq[#preq+1] = s
-			freq[#freq+1] = s
-			c = c.nodes[s]
-			n = i
-			if not c then
-				break
-			end
+		util.update(track, c)
 
-			util.update(track, c)
-
-			if c.leaf then
-				break
-			end
+		if c.leaf then
+			break
 		end
 	end
 
@@ -513,7 +513,7 @@ function createtree()
 	end
 
 	local ctx  = context
-	local tree = {nodes={}}
+	local tree = {nodes={}, inreq=true}
 	local modi = {}
 
 	ctx.treecache = setmetatable({}, {__mode="v"})
@@ -625,6 +625,11 @@ function _create_node(path)
 		local parent = _create_node(path)
 
 		c = {nodes={}, auto=true}
+		-- the node is "in request" if the request path matches
+		-- at least up to the length of the node path
+		if parent.inreq and context.path[#path+1] == last then
+		  c.inreq = true
+		end
 		parent.nodes[last] = c
 		context.treecache[name] = c
 	end