luci-base: fold luci.http.protocol into luci.http

With only the decoder routines remaining in luci.http.protocol, it makes no
sense to keep the low level protocol class around, so fold the remaining code
into the central luci.http class.

Also adjust the few direct users of luci.http.protocol accordingly.

Signed-off-by: Jo-Philipp Wich <jo@mein.io>
This commit is contained in:
Jo-Philipp Wich 2018-04-18 16:11:42 +02:00
parent 8f66de12c1
commit eb4571c6dc
5 changed files with 420 additions and 438 deletions

View file

@ -7,7 +7,7 @@ local nixio = require "nixio"
local ltn12 = require "luci.ltn12"
local util = require "luci.util"
local table = require "table"
local http = require "luci.http.protocol"
local http = require "luci.http"
local date = require "luci.http.protocol.date"
local type, pairs, ipairs, tonumber = type, pairs, ipairs, tonumber

View file

@ -1,19 +1,21 @@
-- Copyright 2008 Steven Barth <steven@midlink.org>
-- Copyright 2010-2018 Jo-Philipp Wich <jo@mein.io>
-- Licensed to the public under the Apache License 2.0.
local ltn12 = require "luci.ltn12"
local protocol = require "luci.http.protocol"
local util = require "luci.util"
local string = require "string"
local coroutine = require "coroutine"
local table = require "table"
local lhttp = require "lucihttp"
local nixio = require "nixio"
local ltn12 = require "luci.ltn12"
local ipairs, pairs, next, type, tostring, error =
ipairs, pairs, next, type, tostring, error
local table, ipairs, pairs, type, tostring, tonumber, error =
table, ipairs, pairs, type, tostring, tonumber, error
module "luci.http"
HTTP_MAX_CONTENT = 1024*8 -- 8 kB maximum content size
context = util.threadlocal()
Request = util.class()
@ -29,7 +31,7 @@ function Request.__init__(self, env, sourcein, sinkerr)
self.message = {
env = env,
headers = {},
params = protocol.urldecode_params(env.QUERY_STRING or ""),
params = urldecode_params(env.QUERY_STRING or ""),
}
self.parsed_input = false
@ -115,7 +117,7 @@ function Request.setfilehandler(self, callback)
end
function Request._parse_input(self)
protocol.parse_message_body(
parse_message_body(
self.input,
self.message,
self.filehandler
@ -266,3 +268,272 @@ urlencode = util.urlencode
function write_json(x)
util.serialize_json(x, write)
end
-- from given url or string. Returns a table with urldecoded values.
-- Simple parameters are stored as string values associated with the parameter
-- name within the table. Parameters with multiple values are stored as array
-- containing the corresponding values.
function urldecode_params(url, tbl)
local parser, name
local params = tbl or { }
parser = lhttp.urlencoded_parser(function (what, buffer, length)
if what == parser.TUPLE then
name, value = nil, nil
elseif what == parser.NAME then
name = lhttp.urldecode(buffer)
elseif what == parser.VALUE and name then
params[name] = lhttp.urldecode(buffer) or ""
end
return true
end)
if parser then
parser:parse((url or ""):match("[^?]*$"))
parser:parse(nil)
end
return params
end
-- separated by "&". Tables are encoded as parameters with multiple values by
-- repeating the parameter name with each value.
function urlencode_params(tbl)
local k, v
local n, enc = 1, {}
for k, v in pairs(tbl) do
if type(v) == "table" then
local i, v2
for i, v2 in ipairs(v) do
if enc[1] then
enc[n] = "&"
n = n + 1
end
enc[n+0] = lhttp.urlencode(k)
enc[n+1] = "="
enc[n+2] = lhttp.urlencode(v2)
n = n + 3
end
else
if enc[1] then
enc[n] = "&"
n = n + 1
end
enc[n+0] = lhttp.urlencode(k)
enc[n+1] = "="
enc[n+2] = lhttp.urlencode(v)
n = n + 3
end
end
return table.concat(enc, "")
end
-- Content-Type. Stores all extracted data associated with its parameter name
-- in the params table within the given message object. Multiple parameter
-- values are stored as tables, ordinary ones as strings.
-- If an optional file callback function is given then it is feeded with the
-- file contents chunk by chunk and only the extracted file name is stored
-- within the params table. The callback function will be called subsequently
-- with three arguments:
-- o Table containing decoded (name, file) and raw (headers) mime header data
-- o String value containing a chunk of the file data
-- o Boolean which indicates wheather the current chunk is the last one (eof)
function mimedecode_message_body(src, msg, file_cb)
local parser, header, field
local len, maxlen = 0, tonumber(msg.env.CONTENT_LENGTH or nil)
parser, err = lhttp.multipart_parser(msg.env.CONTENT_TYPE, function (what, buffer, length)
if what == parser.PART_INIT then
field = { }
elseif what == parser.HEADER_NAME then
header = buffer:lower()
elseif what == parser.HEADER_VALUE and header then
if header:lower() == "content-disposition" and
lhttp.header_attribute(buffer, nil) == "form-data"
then
field.name = lhttp.header_attribute(buffer, "name")
field.file = lhttp.header_attribute(buffer, "filename")
end
if field.headers then
field.headers[header] = buffer
else
field.headers = { [header] = buffer }
end
elseif what == parser.PART_BEGIN then
return not field.file
elseif what == parser.PART_DATA and field.name and length > 0 then
if field.file then
if file_cb then
file_cb(field, buffer, false)
msg.params[field.name] = msg.params[field.name] or field
else
if not field.fd then
field.fd = nixio.mkstemp(field.name)
end
if field.fd then
field.fd:write(buffer)
msg.params[field.name] = msg.params[field.name] or field
end
end
else
field.value = buffer
end
elseif what == parser.PART_END and field.name then
if field.file and msg.params[field.name] then
if file_cb then
file_cb(field, "", true)
elseif field.fd then
field.fd:seek(0, "set")
end
else
msg.params[field.name] = field.value or ""
end
field = nil
elseif what == parser.ERROR then
err = buffer
end
return true
end)
return ltn12.pump.all(src, function (chunk)
len = len + (chunk and #chunk or 0)
if maxlen and len > maxlen + 2 then
return nil, "Message body size exceeds Content-Length"
end
if not parser or not parser:parse(chunk) then
return nil, err
end
return true
end)
end
-- Content-Type. Stores all extracted data associated with its parameter name
-- in the params table within the given message object. Multiple parameter
-- values are stored as tables, ordinary ones as strings.
function urldecode_message_body(src, msg)
local err, name, value, parser
local len, maxlen = 0, tonumber(msg.env.CONTENT_LENGTH or nil)
parser = lhttp.urlencoded_parser(function (what, buffer, length)
if what == parser.TUPLE then
name, value = nil, nil
elseif what == parser.NAME then
name = lhttp.urldecode(buffer)
elseif what == parser.VALUE and name then
msg.params[name] = lhttp.urldecode(buffer) or ""
elseif what == parser.ERROR then
err = buffer
end
return true
end)
return ltn12.pump.all(src, function (chunk)
len = len + (chunk and #chunk or 0)
if maxlen and len > maxlen + 2 then
return nil, "Message body size exceeds Content-Length"
elseif len > HTTP_MAX_CONTENT then
return nil, "Message body size exceeds maximum allowed length"
end
if not parser or not parser:parse(chunk) then
return nil, err
end
return true
end)
end
-- This function will examine the Content-Type within the given message object
-- to select the appropriate content decoder.
-- Currently the application/x-www-urlencoded and application/form-data
-- mime types are supported. If the encountered content encoding can't be
-- handled then the whole message body will be stored unaltered as "content"
-- property within the given message object.
function parse_message_body(src, msg, filecb)
local ctype = lhttp.header_attribute(msg.env.CONTENT_TYPE, nil)
-- Is it multipart/mime ?
if msg.env.REQUEST_METHOD == "POST" and
ctype == "multipart/form-data"
then
return mimedecode_message_body(src, msg, filecb)
-- Is it application/x-www-form-urlencoded ?
elseif msg.env.REQUEST_METHOD == "POST" and
ctype == "application/x-www-form-urlencoded"
then
return urldecode_message_body(src, msg)
-- Unhandled encoding
-- If a file callback is given then feed it chunk by chunk, else
-- store whole buffer in message.content
else
local sink
-- If we have a file callback then feed it
if type(filecb) == "function" then
local meta = {
name = "raw",
encoding = msg.env.CONTENT_TYPE
}
sink = function( chunk )
if chunk then
return filecb(meta, chunk, false)
else
return filecb(meta, nil, true)
end
end
-- ... else append to .content
else
msg.content = ""
msg.content_length = 0
sink = function( chunk )
if chunk then
if ( msg.content_length + #chunk ) <= HTTP_MAX_CONTENT then
msg.content = msg.content .. chunk
msg.content_length = msg.content_length + #chunk
return true
else
return nil, "POST data exceeds maximum allowed length"
end
end
return true
end
end
-- Pump data...
while true do
local ok, err = ltn12.pump.step( src, sink )
if not ok and err then
return nil, err
elseif not ok then -- eof
return true
end
end
return true
end
end

View file

@ -6,25 +6,24 @@ module "luci.http"
---[[
Close the HTTP-Connection.
@class function
@name close
@class function
@name close
]]
---[[
Return the request content if the request was of unknown type.
@class function
@name content
@return HTTP request body
@return HTTP request body length
@class function
@name content
@return HTTP request body
@return HTTP request body length
]]
---[[
Get a certain HTTP input value or a table of all input values.
@class function
@name formvalue
@class function
@name formvalue
@param name Name of the GET or POST variable to fetch
@param noparse Don't parse POST data before getting the value
@return HTTP input value or table of all input value
@ -33,8 +32,8 @@ Get a certain HTTP input value or a table of all input values.
---[[
Get a table of all HTTP input values with a certain prefix.
@class function
@name formvaluetable
@class function
@name formvaluetable
@param prefix Prefix
@return Table of all HTTP input values with given prefix
]]
@ -42,18 +41,18 @@ Get a table of all HTTP input values with a certain prefix.
---[[
Get the value of a certain HTTP-Cookie.
@class function
@name getcookie
@class function
@name getcookie
@param name Cookie Name
@return String containing cookie data
]]
---[[
Get the value of a certain HTTP environment variable
or the environment table itself.
@class function
@name getenv
@class function
@name getenv
@param name Environment variable
@return HTTP environment value or environment table
]]
@ -61,41 +60,41 @@ or the environment table itself.
---[[
Set a handler function for incoming user file uploads.
@class function
@name setfilehandler
@class function
@name setfilehandler
@param callback Handler function
]]
---[[
Send a HTTP-Header.
@class function
@name header
@param key Header key
@param value Header value
@class function
@name header
@param key Header key
@param value Header value
]]
---[[
Set the mime type of following content data.
@class function
@name prepare_content
@param mime Mimetype of following content
@class function
@name prepare_content
@param mime Mimetype of following content
]]
---[[
Get the RAW HTTP input source
@class function
@name source
@return HTTP LTN12 source
@class function
@name source
@return HTTP LTN12 source
]]
---[[
Set the HTTP status code and status message.
@class function
@name status
@class function
@name status
@param code Status code
@param message Status message
]]
@ -105,8 +104,9 @@ Send a chunk of content data to the client.
This function is as a valid LTN12 sink.
If the content chunk is nil this function will automatically invoke close.
@class function
@name write
@class function
@name write
@param content Content chunk
@param src_err Error object from source (optional)
@see close
@ -115,51 +115,146 @@ If the content chunk is nil this function will automatically invoke close.
---[[
Splice data from a filedescriptor to the client.
@class function
@name splice
@param fp File descriptor
@param size Bytes to splice (optional)
@class function
@name splice
@param fp File descriptor
@param size Bytes to splice (optional)
]]
---[[
Redirects the client to a new URL and closes the connection.
@class function
@name redirect
@param url Target URL
@class function
@name redirect
@param url Target URL
]]
---[[
Create a querystring out of a table of key - value pairs.
@class function
@name build_querystring
@param table Query string source table
@class function
@name build_querystring
@param table Query string source table
@return Encoded HTTP query string
]]
---[[
Return the URL-decoded equivalent of a string.
@class function
@name urldecode
@param str URL-encoded string
@param no_plus Don't decode + to " "
@return URL-decoded string
@see urlencode
@see urlencode
]]
---[[
Return the URL-encoded equivalent of a string.
@class function
@name urlencode
@param str Source string
@return URL-encoded string
@see urldecode
@see urldecode
]]
---[[
Send the given data as JSON encoded string.
@class function
@name write_json
@class function
@name write_json
@param data Data to send
]]
---[[
Extract and split urlencoded data pairs, separated bei either "&" or ";"
from given url or string. Returns a table with urldecoded values.
Simple parameters are stored as string values associated with the parameter
name within the table. Parameters with multiple values are stored as array
containing the corresponding values.
@class function
@name urldecode_params
@param url The url or string which contains x-www-urlencoded form data
@param tbl Use the given table for storing values (optional)
@return Table containing the urldecoded parameters
@see urlencode_params
]]
---[[
Encode each key-value-pair in given table to x-www-urlencoded format,
separated by "&".
Tables are encoded as parameters with multiple values by repeating the
parameter name with each value.
@class function
@name urlencode_params
@param tbl Table with the values
@return String containing encoded values
@see urldecode_params
]]
---[[
Decode a mime encoded http message body with multipart/form-data Content-Type.
Stores all extracted data associated with its parameter name
in the params table within the given message object. Multiple parameter
values are stored as tables, ordinary ones as strings.
If an optional file callback function is given then it is feeded with the
file contents chunk by chunk and only the extracted file name is stored
within the params table. The callback function will be called subsequently
with three arguments:
o Table containing decoded (name, file) and raw (headers) mime header data
o String value containing a chunk of the file data
o Boolean which indicates wheather the current chunk is the last one (eof)
@class function
@name mimedecode_message_body
@param src Ltn12 source function
@param msg HTTP message object
@param filecb File callback function (optional)
@return Value indicating successful operation (not nil means "ok")
@return String containing the error if unsuccessful
@see parse_message_header
]]
---[[
Decode an urlencoded http message body with application/x-www-urlencoded
Content-Type.
Stores all extracted data associated with its parameter name in the params
table within the given message object. Multiple parameter values are stored
as tables, ordinary ones as strings.
@class function
@name urldecode_message_body
@param src Ltn12 source function
@param msg HTTP message object
@return Value indicating successful operation (not nil means "ok")
@return String containing the error if unsuccessful
@see parse_message_header
]]
---[[
Try to extract and decode a http message body from the given ltn12 source.
This function will examine the Content-Type within the given message object
to select the appropriate content decoder.
Currently the application/x-www-urlencoded and application/form-data
mime types are supported. If the encountered content encoding can't be
handled then the whole message body will be stored unaltered as "content"
property within the given message object.
@class function
@name parse_message_body
@param src Ltn12 source function
@param msg HTTP message object
@param filecb File data callback (optional, see mimedecode_message_body())
@return Value indicating successful operation (not nil means "ok")
@return String containing the error if unsuccessful
@see parse_message_header
]]

View file

@ -1,285 +0,0 @@
-- Copyright 2008-2018 Jo-Philipp Wich <jo@mein.io>
-- Licensed to the public under the Apache License 2.0.
-- This class contains several functions useful for http message- and content
-- decoding and to retrive form data from raw http messages.
local require, type, tonumber = require, type, tonumber
local table, pairs, ipairs, pcall = table, pairs, ipairs, pcall
module "luci.http.protocol"
local ltn12 = require "luci.ltn12"
local lhttp = require "lucihttp"
HTTP_MAX_CONTENT = 1024*8 -- 8 kB maximum content size
-- from given url or string. Returns a table with urldecoded values.
-- Simple parameters are stored as string values associated with the parameter
-- name within the table. Parameters with multiple values are stored as array
-- containing the corresponding values.
function urldecode_params(url, tbl)
local parser, name
local params = tbl or { }
parser = lhttp.urlencoded_parser(function (what, buffer, length)
if what == parser.TUPLE then
name, value = nil, nil
elseif what == parser.NAME then
name = lhttp.urldecode(buffer)
elseif what == parser.VALUE and name then
params[name] = lhttp.urldecode(buffer) or ""
end
return true
end)
if parser then
parser:parse((url or ""):match("[^?]*$"))
parser:parse(nil)
end
return params
end
-- separated by "&". Tables are encoded as parameters with multiple values by
-- repeating the parameter name with each value.
function urlencode_params(tbl)
local k, v
local n, enc = 1, {}
for k, v in pairs(tbl) do
if type(v) == "table" then
local i, v2
for i, v2 in ipairs(v) do
if enc[1] then
enc[n] = "&"
n = n + 1
end
enc[n+0] = lhttp.urlencode(k)
enc[n+1] = "="
enc[n+2] = lhttp.urlencode(v2)
n = n + 3
end
else
if enc[1] then
enc[n] = "&"
n = n + 1
end
enc[n+0] = lhttp.urlencode(k)
enc[n+1] = "="
enc[n+2] = lhttp.urlencode(v)
n = n + 3
end
end
return table.concat(enc, "")
end
-- Content-Type. Stores all extracted data associated with its parameter name
-- in the params table within the given message object. Multiple parameter
-- values are stored as tables, ordinary ones as strings.
-- If an optional file callback function is given then it is feeded with the
-- file contents chunk by chunk and only the extracted file name is stored
-- within the params table. The callback function will be called subsequently
-- with three arguments:
-- o Table containing decoded (name, file) and raw (headers) mime header data
-- o String value containing a chunk of the file data
-- o Boolean which indicates wheather the current chunk is the last one (eof)
function mimedecode_message_body(src, msg, file_cb)
local parser, header, field
local len, maxlen = 0, tonumber(msg.env.CONTENT_LENGTH or nil)
parser, err = lhttp.multipart_parser(msg.env.CONTENT_TYPE, function (what, buffer, length)
if what == parser.PART_INIT then
field = { }
elseif what == parser.HEADER_NAME then
header = buffer:lower()
elseif what == parser.HEADER_VALUE and header then
if header:lower() == "content-disposition" and
lhttp.header_attribute(buffer, nil) == "form-data"
then
field.name = lhttp.header_attribute(buffer, "name")
field.file = lhttp.header_attribute(buffer, "filename")
end
if field.headers then
field.headers[header] = buffer
else
field.headers = { [header] = buffer }
end
elseif what == parser.PART_BEGIN then
return not field.file
elseif what == parser.PART_DATA and field.name and length > 0 then
if field.file then
if file_cb then
file_cb(field, buffer, false)
msg.params[field.name] = msg.params[field.name] or field
else
if not field.fd then
local ok, nx = pcall(require, "nixio")
field.fd = ok and nx.mkstemp(field.name)
end
if field.fd then
field.fd:write(buffer)
msg.params[field.name] = msg.params[field.name] or field
end
end
else
field.value = buffer
end
elseif what == parser.PART_END and field.name then
if field.file and msg.params[field.name] then
if file_cb then
file_cb(field, "", true)
elseif field.fd then
field.fd:seek(0, "set")
end
else
msg.params[field.name] = field.value or ""
end
field = nil
elseif what == parser.ERROR then
err = buffer
end
return true
end)
return ltn12.pump.all(src, function (chunk)
len = len + (chunk and #chunk or 0)
if maxlen and len > maxlen + 2 then
return nil, "Message body size exceeds Content-Length"
end
if not parser or not parser:parse(chunk) then
return nil, err
end
return true
end)
end
-- Content-Type. Stores all extracted data associated with its parameter name
-- in the params table within the given message object. Multiple parameter
-- values are stored as tables, ordinary ones as strings.
function urldecode_message_body(src, msg)
local err, name, value, parser
local len, maxlen = 0, tonumber(msg.env.CONTENT_LENGTH or nil)
parser = lhttp.urlencoded_parser(function (what, buffer, length)
if what == parser.TUPLE then
name, value = nil, nil
elseif what == parser.NAME then
name = lhttp.urldecode(buffer)
elseif what == parser.VALUE and name then
msg.params[name] = lhttp.urldecode(buffer) or ""
elseif what == parser.ERROR then
err = buffer
end
return true
end)
return ltn12.pump.all(src, function (chunk)
len = len + (chunk and #chunk or 0)
if maxlen and len > maxlen + 2 then
return nil, "Message body size exceeds Content-Length"
elseif len > HTTP_MAX_CONTENT then
return nil, "Message body size exceeds maximum allowed length"
end
if not parser or not parser:parse(chunk) then
return nil, err
end
return true
end)
end
-- This function will examine the Content-Type within the given message object
-- to select the appropriate content decoder.
-- Currently the application/x-www-urlencoded and application/form-data
-- mime types are supported. If the encountered content encoding can't be
-- handled then the whole message body will be stored unaltered as "content"
-- property within the given message object.
function parse_message_body(src, msg, filecb)
local ctype = lhttp.header_attribute(msg.env.CONTENT_TYPE, nil)
-- Is it multipart/mime ?
if msg.env.REQUEST_METHOD == "POST" and
ctype == "multipart/form-data"
then
return mimedecode_message_body( src, msg, filecb )
-- Is it application/x-www-form-urlencoded ?
elseif msg.env.REQUEST_METHOD == "POST" and
ctype == "application/x-www-form-urlencoded"
then
return urldecode_message_body( src, msg, filecb )
-- Unhandled encoding
-- If a file callback is given then feed it chunk by chunk, else
-- store whole buffer in message.content
else
local sink
-- If we have a file callback then feed it
if type(filecb) == "function" then
local meta = {
name = "raw",
encoding = msg.env.CONTENT_TYPE
}
sink = function( chunk )
if chunk then
return filecb(meta, chunk, false)
else
return filecb(meta, nil, true)
end
end
-- ... else append to .content
else
msg.content = ""
msg.content_length = 0
sink = function( chunk )
if chunk then
if ( msg.content_length + #chunk ) <= HTTP_MAX_CONTENT then
msg.content = msg.content .. chunk
msg.content_length = msg.content_length + #chunk
return true
else
return nil, "POST data exceeds maximum allowed length"
end
end
return true
end
end
-- Pump data...
while true do
local ok, err = ltn12.pump.step( src, sink )
if not ok and err then
return nil, err
elseif not ok then -- eof
return true
end
end
return true
end
end

View file

@ -1,99 +0,0 @@
---[[
LuCI http protocol class.
This class contains several functions useful for http message- and content
decoding and to retrive form data from raw http messages.
]]
module "luci.http.protocol"
---[[
Extract and split urlencoded data pairs, separated bei either "&" or ";"
from given url or string. Returns a table with urldecoded values.
Simple parameters are stored as string values associated with the parameter
name within the table. Parameters with multiple values are stored as array
containing the corresponding values.
@class function
@name urldecode_params
@param url The url or string which contains x-www-urlencoded form data
@param tbl Use the given table for storing values (optional)
@return Table containing the urldecoded parameters
@see urlencode_params
]]
---[[
Encode each key-value-pair in given table to x-www-urlencoded format,
separated by "&".
Tables are encoded as parameters with multiple values by repeating the
parameter name with each value.
@class function
@name urlencode_params
@param tbl Table with the values
@return String containing encoded values
@see urldecode_params
]]
---[[
Decode a mime encoded http message body with multipart/form-data Content-Type.
Stores all extracted data associated with its parameter name
in the params table within the given message object. Multiple parameter
values are stored as tables, ordinary ones as strings.
If an optional file callback function is given then it is feeded with the
file contents chunk by chunk and only the extracted file name is stored
within the params table. The callback function will be called subsequently
with three arguments:
o Table containing decoded (name, file) and raw (headers) mime header data
o String value containing a chunk of the file data
o Boolean which indicates wheather the current chunk is the last one (eof)
@class function
@name mimedecode_message_body
@param src Ltn12 source function
@param msg HTTP message object
@param filecb File callback function (optional)
@return Value indicating successful operation (not nil means "ok")
@return String containing the error if unsuccessful
@see parse_message_header
]]
---[[
Decode an urlencoded http message body with application/x-www-urlencoded
Content-Type.
Stores all extracted data associated with its parameter name in the params
table within the given message object. Multiple parameter values are stored
as tables, ordinary ones as strings.
@class function
@name urldecode_message_body
@param src Ltn12 source function
@param msg HTTP message object
@return Value indicating successful operation (not nil means "ok")
@return String containing the error if unsuccessful
@see parse_message_header
]]
---[[
Try to extract and decode a http message body from the given ltn12 source.
This function will examine the Content-Type within the given message object
to select the appropriate content decoder.
Currently the application/x-www-urlencoded and application/form-data
mime types are supported. If the encountered content encoding can't be
handled then the whole message body will be stored unaltered as "content"
property within the given message object.
@class function
@name parse_message_body
@param src Ltn12 source function
@param msg HTTP message object
@param filecb File data callback (optional, see mimedecode_message_body())
@return Value indicating successful operation (not nil means "ok")
@return String containing the error if unsuccessful
@see parse_message_header
]]