* Introducing LuCI HTTPD as testing environment

* Several coroutine-safety fixes

Makefile

@@ -34,9 +34,19 @@ hostcopy:
 	rm -f host/luci
 	ln -s .$(LUCI_MODULEDIR) host/luci
 
-run: host
+runboa: host
 	libs/sgi-webuci/host/buildconfig.sh `pwd`/host  > host/etc/boa/boa.conf
 	./host/usr/bin/boa -c ./host/etc/boa -d
 
+runluci: luahost
+	libs/httpd/host/runluci host$(HTDOCS)
+
 hostclean: clean
 	rm -rf host
+
+run:
+	#	make run is deprecated			#
+	#	Please use:				#
+	#						#
+	#	make runluci to use LuCI HTTPD		#
+	#	make runboa  to use Boa / Webuci	#

libs/http/luasrc/http/protocol.lua

@@ -16,7 +16,6 @@ $Id$
 module("luci.http.protocol", package.seeall)
 
 require("ltn12")
-require("luci.util")
 require("luci.http.protocol.filter")
 
 HTTP_MAX_CONTENT      = 1024*4		-- 4 kB maximum content size
@@ -49,7 +48,7 @@ function urldecode_params( url, tbl )
 		url = url:gsub( "^.+%?([^?]+)", "%1" )
 	end
 
-	for i, pair in ipairs(luci.util.split( url, "[&;]+", nil, true )) do
+	for pair in url:gmatch( "[^&;]+" ) do
 
 		-- find key and value
 		local key = urldecode( pair:match("^([^=]+)")     )
@@ -501,7 +500,8 @@ process_states['urldecode-value'] = function( msg, chunk, filecb )
 			-- We're somewhere within a data section and our buffer is full
 			if #buffer > #chunk then
 				-- Flush buffered data
-				msg._urldeccallback( buffer:sub( 1, #buffer - #chunk ), false )
+				-- Send EOF if chunk is empty
+				msg._urldeccallback( buffer:sub( 1, #buffer - #chunk ), ( #chunk == 0 ) )
 
 				-- Store new data
 				msg._urldeclength = msg._urldeclength + #buffer - #chunk
@@ -769,3 +769,34 @@ function parse_message_body( source, msg, filecb )
 		end
 	end
 end
+
+
+-- Push a response to a socket
+function push_response(request, response, sourceout, sinkout, sinkerr)
+	local code = response.status
+	sinkout(request.env.SERVER_PROTOCOL .. " " .. code .. " " .. statusmsg[code] .. "\r\n")
+
+	-- FIXME: Add support for keep-alive
+	response.headers["Connection"] = "close"
+
+	for k,v in pairs(response.headers) do
+		sinkout(k .. ": " .. v .. "\r\n")
+	end
+
+	sinkout("\r\n")
+
+	if sourceout then
+		ltn12.pump.all(sourceout, sinkout)
+	end
+end
+
+
+-- Status codes
+statusmsg = {
+	[200] = "OK",
+	[400] = "Bad Request",
+	[403] = "Forbidden",
+	[404] = "Not Found",
+	[500] = "Internal Server Error",
+	[503] = "Server Unavailable",
+}

libs/httpd/host/runluci

@@ -0,0 +1,32 @@
+#!/usr/bin/lua
+require("luci.httpd")
+require("luci.httpd.server")
+require("luci.httpd.handler.file")
+require("luci.httpd.handler.luci")
+
+DOCROOT = arg[1]
+PORT = 8080
+
+
+serversocket = luci.httpd.Socket("0.0.0.0", PORT)
+
+
+server  = luci.httpd.server.Server()
+vhost   = luci.httpd.server.VHost()
+
+server:set_default_vhost(vhost)
+
+
+filehandler = luci.httpd.handler.file.Simple(DOCROOT)
+vhost:set_default_handler(filehandler)
+
+lucihandler = luci.httpd.handler.luci.Luci()
+vhost:set_handler("/luci", lucihandler)
+
+io.stderr:write("Starting LuCI HTTPD on port " .. PORT .. "...\n")
+io.stderr:write("Point your browser to http://localhost:" .. PORT .. "/luci\n")
+
+daemon = luci.httpd.Daemon()
+daemon.debug = true
+daemon:register(serversocket, server:create_daemon_handlers())
+daemon:run()

libs/httpd/luasrc/httpd.lua

@@ -13,23 +13,41 @@ $Id$
 
 ]]--
 
-require("ltn12")
+module("luci.httpd", package.seeall)
 require("socket")
 require("luci.util")
 
+function Socket(ip, port)
+	local sock, err = socket.bind( ip, port )
+
+	if sock then
+		sock:settimeout( 0, "t" )
+	end
+
+	return sock, err
+end
+
 
 Daemon = luci.util.class()
 
-function Daemon.__init__(self, threadlimit)
+function Daemon.__init__(self, threadlimit, timeout)
 	self.reading = {}
 	self.running = {}
 	self.handler = {}
+	self.debug   = false
 	self.threadlimit = threadlimit
+	self.timeout = timeout or 0.1
 end
 
-function Daemon.register(self, socket, clhandler, errhandler)
+function Daemon.dprint(self, msg)
-	table.insert( self.reading, socket )
+	if self.debug then
-	self.handler[socket] = { clhandler = clhandler, errhandler = errhandler }
+		io.stderr:write("[daemon] " .. msg .. "\n")
+	end
+end
+
+function Daemon.register(self, sock, clhandler, errhandler)
+	table.insert( self.reading, sock )
+	self.handler[sock] = { clhandler = clhandler, errhandler = errhandler }
 end
 
 function Daemon.run(self)
@@ -39,7 +57,11 @@ function Daemon.run(self)
 end
 
 function Daemon.step(self)	
-	local input = socket.select( self.reading, nil, 0 )
+	local input, output, err = socket.select( self.reading, nil, 0 )
+
+	if err == "timeout" and #self.running == 0 then
+		socket.sleep(self.timeout)
+	end
 
 	-- accept new connections
 	for i, connection in ipairs(input) do
@@ -47,15 +69,21 @@ function Daemon.step(self)
 		local sock = connection:accept()
 
 		-- check capacity
-		if self.threadlimit and #running < self.threadlimit then
+		if not self.threadlimit or #self.running < self.threadlimit then
+
+			self:dprint("Accepted incoming connection from " .. sock:getpeername())
 
 			table.insert( self.running, {
 				coroutine.create( self.handler[connection].clhandler ),
 				sock
 			} )
 
+			self:dprint("Created " .. tostring(self.running[#self.running][1]))
+
 		-- reject client
 		else
+			self:dprint("Rejected incoming connection from " .. sock:getpeername())
+
 			if self.handler[connection].errhandler then
 				self.handler[connection].errhandler( sock )
 			end
@@ -69,9 +97,18 @@ function Daemon.step(self)
 
 		-- reap dead clients
 		if coroutine.status( client[1] ) == "dead" then
+			self:dprint("Completed " .. tostring(client[1]))
 			table.remove( self.running, i )
-		end
+		else
+			self:dprint("Resuming " .. tostring(client[1]))
 
-		coroutine.resume( client[1], client[2] )
+			local stat, err = coroutine.resume( client[1], client[2] )
+
+			self:dprint(tostring(client[1]) .. " returned")
+
+			if not stat then
+				self:dprint("Error in " .. tostring(client[1]) .. " " .. err)
+			end
+		end
 	end
 end

libs/httpd/luasrc/httpd/handler/file.lua

@@ -12,11 +12,17 @@ function Simple.__init__(self, docroot)
 end
 
 function Simple.handle(self, request, sourcein, sinkerr)
-	local file = self.docroot .. request.env.REQUEST_URI:gsub("../", "")
+	local uri  = request.env.PATH_INFO
-	local size = luci.fs.stat(file, "size")
+	local file = self.docroot .. uri:gsub("%.%./", "")
-	if size then
+	local stat = luci.fs.stat(file)
-		return Response(200, {["Content-Length"] = size}), ltn12.source.file(io.open(file))
+
+	if stat then
+		if stat.type == "regular" then
+			return Response(200, {["Content-Length"] = stat.size}), ltn12.source.file(io.open(file))
 		else
-		return Response(404)
+			return self:failure(403, "Unable to transmit " .. stat.type .. " " .. uri)
+		end
+	else
+		return self:failure(404, "No such file: " .. uri)
 	end
 end

libs/httpd/luasrc/httpd/handler/luci.lua

@@ -0,0 +1,56 @@
+module("luci.httpd.handler.luci", package.seeall)
+require("luci.dispatcher")
+require("luci.http")
+require("ltn12")
+
+Luci = luci.util.class(luci.httpd.module.Handler)
+Response = luci.httpd.module.Response
+
+function Luci.__init__(self)
+	luci.httpd.module.Handler.__init__(self)
+end
+
+function Luci.handle(self, request, sourcein, sinkerr)	
+	local r = luci.http.Request(
+		request.env,
+		sourcein,
+		sinkerr
+	)
+		
+	local res, id, data1, data2 = true, 0, nil, nil
+	local headers = {}
+	local status = 200
+	
+	local x = coroutine.create(luci.dispatcher.httpdispatch)
+	while id < 3 do
+		coroutine.yield()
+		
+		res, id, data1, data2 = coroutine.resume(x, r)
+		
+		if not res then
+			status = 500
+			headers["Content-Type"] = "text/plain"
+			local err = {id}
+			return status, headers, function() local x = table.remove(err) return x end
+		end
+		
+		if id == 1 then
+			status = data1
+		elseif id == 2 then
+			headers[data1] = data2
+		end
+	end
+	
+	local function iter()
+		local res, id, data = coroutine.resume(x)
+		if not res then
+			return nil, id
+		elseif id == 5 then
+			return nil
+		else
+			return data
+		end
+	end
+	
+	return Response(status, headers), iter
+end

libs/httpd/luasrc/httpd/module.lua

@@ -34,15 +34,9 @@ end
 
 
 -- Creates a failure reply
-function Handler.failure(self, message)
+function Handler.failure(self, code, message)
-	response = {
+	local response = Response(code, { ["Content-Type"] = "text/plain" })
-		status = 500,
+	local sourceout = ltn12.source.string(message)
-		headers = {
-			["Content-Type"] = "text/plain"
-		}
-	}
-	
-	sourceout = ltn12.source.string(message)
 	
 	return response, sourceout 
 end
@@ -70,12 +64,12 @@ function Handler.process(self, request, sourcein, sinkout, sinkerr)
 	
 	-- Check for any errors
 	if not stat then
-		response, sourceout = self:failure(response)
+		response, sourceout = self:failure(500, response)
 	end
 	
 	-- Check data
 	if not luci.util.instanceof(response, Response) then
-		response, sourceout = self:failure("Core error: Invalid module response!")
+		response, sourceout = self:failure(500, "Core error: Invalid module response!")
 	end
 	
 	-- Process outgoing filters
@@ -132,11 +126,3 @@ end
 function Response.setstatus(self, status)
 	self.status = status
 end
-
-
--- Status codes
-statusmsg = {
-	[200] = "OK",
-	[404] = "Not Found",
-	[500] = "Internal Server Error",
-}

libs/httpd/luasrc/httpd/server.lua

@@ -18,6 +18,7 @@ require("luci.util")
 
 READ_BUFSIZE = 1024
 
+
 VHost = luci.util.class()
 
 function VHost.__init__(self, handler)
@@ -25,76 +26,75 @@ function VHost.__init__(self, handler)
 	self.dhandler = {}
 end
 
-function VHost.process(self, ...)
+function VHost.process(self, request, sourcein, sinkout, sinkerr)
-	-- TODO: Dispatch handler
+	local handler = self.handler
+
+	local uri = request.env.REQUEST_URI:match("^([^?]*)")
+
+	-- SCRIPT_NAME
+	request.env.SCRIPT_NAME = ""
+
+	-- Call URI part
+	request.env.PATH_INFO = uri
+
+	for k, dhandler in pairs(self.dhandler) do
+		if k == uri or k.."/" == uri:sub(1, #k+1) then
+			handler = dhandler
+			request.env.SCRIPT_NAME = k
+			request.env.PATH_INFO   = uri:sub(#k+1)
+			break;
+		end
 	end
 
-function VHost.sethandler(self, handler, match)
+	if handler then
-	if match then
+		handler:process(request, sourcein, sinkout, sinkerr)
-		self.dhandler[match] = handler
+		return true
 	else
+		return false
+	end
+end
+
+
+function VHost.set_default_handler(self, handler)
 	self.handler = handler
 end
+
+
+function VHost.set_handler(self, match, handler)
+	self.dhandler[match] = handler
 end
 
 
 
 Server = luci.util.class()
 
-function Server.__init__(self, ip, port, base)
+function Server.__init__(self, host)
-	self.socket = socket.bind(ip, port)
-	self.socket:settimeout(0, "t")
 	self.clhandler = client_handler
 	self.errhandler = error503
-	self.host = nil
+	self.host = host
 	self.vhosts = {}
-	
-	-- Clone another server
-	if base then
-		getmetatable(self).__index = base 
 end
+
+function Server.set_default_vhost(self, vhost)
+	self.host = vhost
 end
 
 -- Sets a vhost
-function Server.setvhost(self, vhost, name)
+function Server.set_vhost(self, name, vhost)
-	if name then
 	self.vhosts[name] = vhost
-	else
-		self.host = vhost
-	end
 end
 
-
+function Server.create_daemon_handlers(self)
-function Server.error400(self, client, msg)
+	return function(...) return self:process(...) end,
-	client:send( "HTTP/1.0 400 Bad request\r\n" )
+		function(...) return self:error503(...) end
-	client:send( "Content-Type: text/plain\r\n\r\n" )
-
-	if msg then
-		client:send( msg .. "\r\n" )
 end
 
-	client:close()
+function Server.create_client_sources(self, client)
-end
-
-function Server.error503(self, client)
-	client:send( "HTTP/1.0 503 Server unavailable\r\n" )
-	client:send( "Content-Type: text/plain\r\n\r\n" )
-	client:send( "There are too many clients connected, try again later\r\n" )
-end
-
-function Server.process(self, ...)
-	-- TODO: Dispatch vhost
-end
-
-
-function Server.client_handler(self, client)
-
-	client:settimeout( 0 )
-
 	-- Create LTN12 block source
 	local block_source = function()
 
-		coroutine.yield()
+		-- Yielding here may cause chaos in coroutine based modules, be careful
+		-- coroutine.yield()
 
 		local chunk, err, part = client:receive( READ_BUFSIZE )
 
@@ -108,6 +108,7 @@ function Server.client_handler(self, client)
 
 	end
 
+
 	-- Create LTN12 line source
 	local line_source = ltn12.source.simplify( function()
 
@@ -139,14 +140,55 @@ function Server.client_handler(self, client)
 		end
 	end )
 
-	coroutine.yield(client)
+	return block_source, line_source
+end
 
 
-	-- parse message
+function Server.error400(self, socket, msg)
-	local message, err = luci.http.protocol.parse_message_header( line_source )
+	socket:send( "HTTP/1.0 400 Bad request\r\n" )
+	socket:send( "Content-Type: text/plain\r\n\r\n" )
+
+	if msg then
+		socket:send( msg .. "\r\n" )
+	end
+
+	socket:close()
+end
+
+function Server.error500(self, socket, msg)
+	socket:send( "HTTP/1.0 500 Internal Server Error\r\n" )
+	socket:send( "Content-Type: text/plain\r\n\r\n" )
+
+	if msg then
+		socket:send( msg .. "\r\n" )
+	end
+
+	socket:close()
+end
+
+function Server.error503(self, socket)
+	socket:send( "HTTP/1.0 503 Server unavailable\r\n" )
+	socket:send( "Content-Type: text/plain\r\n\r\n" )
+	socket:send( "There are too many clients connected, try again later\r\n" )
+	socket:close()
+end
+
+
+function Server.process(self, client)
+
+	client:settimeout( 0 )
+	local sourcein, sourcehdr = self:create_client_sources(client)
+	local sinkerr = ltn12.sink.file(io.stderr)
+
+	-- FIXME: Add keep-alive support
+	local sinkout = socket.sink("close-when-done", client)
+
+	coroutine.yield()
+
+	-- parse headers
+	local message, err = luci.http.protocol.parse_message_header( sourcehdr )
 
 	if message then
-
 		-- If we have a HTTP/1.1 client and an Expect: 100-continue header then
 		-- respond with HTTP 100 Continue message
 		if message.http_version == 1.1 and message.headers['Expect'] and
@@ -155,19 +197,18 @@ function Server.client_handler(self, client)
 			client:send("HTTP/1.1 100 Continue\r\n\r\n")
 		end
 
-
+		local host = self.vhosts[message.env.HTTP_HOST] or self.host
-		local s, e = luci.http.protocol.parse_message_body( block_source, message )
+		if host then
-
+			if host:process(message, sourcein, sinkout, sinkerr) then
-		-- XXX: debug
+				sinkout()
-		luci.util.dumptable( message )
+			else
-
+				self:error500( client, "No suitable path handler found" )
-		if not s and e then
+			end
-			self:error400( client, e )
+		else
+			self:error500( client, "No suitable host handler found" )
 		end
 	else
 		self:error400( client, err )
+		return nil
 	end
-
-	-- send response
-	self:error400( client, "Dummy response" )
 end

libs/web/luasrc/dispatcher.lua

@@ -267,7 +267,7 @@ function createtree()
 	luci.i18n.loadc("default")
 	
 	local scope = luci.util.clone(_G)
-	for k,v in pairs(_M) do
+	for k,v in pairs(luci.dispatcher) do
 		if type(v) == "function" then
 			scope[k] = v
 		end
@@ -279,7 +279,7 @@ function createtree()
 
 		local stat, err = luci.util.copcall(v)
 		if not stat then
-			error500(err)	
+			error500("createtree failed: " .. k .. ": " .. err)
 			os.exit(1)
 		end
 	end

libs/web/luasrc/http.lua

@@ -53,13 +53,14 @@ function Request.__init__(self, env, sourcein, sinkerr)
 	
 	setmetatable(self.message.params, {__index =
 		function(tbl, key)
+			setmetatable(tbl, nil)
+
 			luci.http.protocol.parse_message_body(
 			 self.input,
 			 self.message,
 			 self.filehandler
 			)
 
-			setmetatable(tbl, nil)
 			return rawget(tbl, key)
 		end
 	})