nixio: Fixes, use POSIX calls for file i/o

httpclient: resume support, splice() support, cookie support
This commit is contained in:
Steven Barth 2009-02-27 14:51:37 +00:00
parent 30421d38dd
commit 7196b2cd84
11 changed files with 441 additions and 119 deletions

View file

@ -1,5 +1,5 @@
--[[
LuCI - Lua Configuration Interface
LuCI - Lua Development Framework
Copyright 2009 Steven Barth <steven@midlink.org>
@ -19,8 +19,9 @@ local ltn12 = require "luci.ltn12"
local util = require "luci.util"
local table = require "table"
local http = require "luci.http.protocol"
local date = require "luci.http.protocol.date"
local type, pairs, tonumber, print = type, pairs, tonumber, print
local type, pairs, ipairs, tonumber = type, pairs, ipairs, tonumber
module "luci.httpclient"
@ -93,7 +94,7 @@ function request_to_source(uri, options)
return nil, status, response
end
if response["Transfer-Encoding"] == "chunked" then
if response.headers["Transfer-Encoding"] == "chunked" then
return chunksource(sock, buffer)
else
return ltn12.source.cat(ltn12.source.string(buffer), sock:blocksource())
@ -114,7 +115,7 @@ function request_raw(uri, options)
return nil, -2, "protocol not supported"
end
port = #port > 0 and port or (pr == "https" and "443" or "80")
port = #port > 0 and port or (pr == "https" and 443 or 80)
path = #path > 0 and path or "/"
options.depth = options.depth or 10
@ -163,10 +164,28 @@ function request_raw(uri, options)
local message = {method .. " " .. path .. " " .. protocol}
for k, v in pairs(headers) do
if v then
if type(v) == "string" then
message[#message+1] = k .. ": " .. v
elseif type(v) == "table" then
for i, j in ipairs(v) do
message[#message+1] = k .. ": " .. j
end
end
end
if options.cookies then
for _, c in ipairs(options.cookies) do
local cdo = c.flags.domain
local cpa = c.flags.path
if (cdo == host or cdo == "."..host or host:sub(-#cdo) == cdo)
and (cpa == "/" or cpa .. "/" == path:sub(#cpa+1))
and (not c.flags.secure or pr == "https")
then
message[#message+1] = "Cookie: " .. c.key .. "=" .. c.value
end
end
end
message[#message+1] = ""
message[#message+1] = ""
@ -191,13 +210,19 @@ function request_raw(uri, options)
return nil, -3, "invalid response magic: " .. line
end
local response = {Status=line}
local response = {status = line, headers = {}, code = 0, cookies = {}}
line = linesrc()
while line and line ~= "" do
local key, val = line:match("^([%w-]+)%s?:%s?(.*)")
if key and key ~= "Status" then
response[key] = val
if type(response[key]) == "string" then
response.headers[key] = {response.headers[key], val}
elseif type(response[key]) == "table" then
response.headers[key][#response.headers[key]+1] = val
else
response.headers[key] = val
end
end
line = linesrc()
end
@ -206,11 +231,54 @@ function request_raw(uri, options)
return nil, -4, "protocol error"
end
-- Parse cookies
if response.headers["Set-Cookie"] then
local cookies = response.headers["Set-Cookie"]
for _, c in ipairs(type(cookies) == "table" and cookies or {cookies}) do
local cobj = cookie_parse(c)
cobj.flags.path = cobj.flags.path or path:match("(/.*)/?[^/]*")
if not cobj.flags.domain or cobj.flags.domain == "" then
cobj.flags.domain = host
response.cookies[#response.cookies+1] = cobj
else
local hprt, cprt = {}, {}
-- Split hostnames and save them in reverse order
for part in host:gmatch("[^.]*") do
table.insert(hprt, 1, part)
end
for part in cobj.flags.domain:gmatch("[^.]*") do
table.insert(cprt, 1, part)
end
local valid = true
for i, part in ipairs(cprt) do
-- If parts are different and no wildcard
if hprt[i] ~= part and #part ~= 0 then
valid = false
break
-- Wildcard on invalid position
elseif hprt[i] ~= part and #part == 0 then
if i ~= #cprt or (#hprt ~= i and #hprt+1 ~= i) then
valid = false
break
end
end
end
-- No TLD cookies
if valid and #cprt > 1 and #cprt[2] > 0 then
response.cookies[#response.cookies+1] = cobj
end
end
end
end
-- Follow
local code = tonumber(status)
if code and options.depth > 0 then
if code == 301 or code == 302 or code == 307 and response.Location then
local nexturi = response.Location
response.code = tonumber(status)
if response.code and options.depth > 0 then
if response.code == 301 or response.code == 302 or response.code == 307
and response.headers.Location then
local nexturi = response.headers.Location
if not nexturi:find("https?://") then
nexturi = pr .. "://" .. host .. ":" .. port .. nexturi
end
@ -222,5 +290,36 @@ function request_raw(uri, options)
end
end
return code, response, linesrc(true), sock
return response.code, response, linesrc(true), sock
end
function cookie_parse(cookiestr)
local key, val, flags = cookiestr:match("%s?([^=;]+)=?([^;]*)(.*)")
if not key then
return nil
end
local cookie = {key = key, value = val, flags = {}}
for fkey, fval in flags:gmatch(";%s?([^=;]+)=?([^;]*)") do
fkey = fkey:lower()
if fkey == "expires" then
fval = date.to_unix(fval:gsub("%-", " "))
end
cookie.flags[fkey] = fval
end
return cookie
end
function cookie_create(cookie)
local cookiedata = {cookie.key .. "=" .. cookie.value}
for k, v in pairs(cookie.flags) do
if k == "expires" then
v = date.to_http(v):gsub(", (%w+) (%w+) (%w+) ", ", %1-%2-%3 ")
end
cookiedata[#cookiedata+1] = k .. ((#v > 0) and ("=" .. v) or "")
end
return table.concat(cookiedata, "; ")
end

View file

@ -0,0 +1,197 @@
--[[
LuCI - Lua Development Framework
Copyright 2009 Steven Barth <steven@midlink.org>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
$Id$
]]--
require "nixio.util"
local nixio = require "nixio"
local httpclient = require "luci.httpclient"
local ltn12 = require "luci.ltn12"
local print = print
module "luci.httpclient.receiver"
local function prepare_fd(target)
-- Open fd for appending
local file, code, msg = nixio.open(target, "r+")
if not file and code == nixio.const.ENOENT then
file, code, msg = nixio.open(target, "w")
if file then
file:flush()
end
end
if not file then
return file, code, msg
end
-- Acquire lock
local stat, code, msg = file:lock("ex", "nb")
if not stat then
return stat, code, msg
end
file:seek(0, "end")
return file
end
function request_to_file(uri, target, options, cbs)
options = options or {}
cbs = cbs or {}
options.headers = options.headers or {}
local hdr = options.headers
local file, code, msg = prepare_fd(target)
if not file then
return file, code, msg
end
local off = file:tell()
-- Set content range
if off > 0 then
hdr.Range = hdr.Range or ("bytes=" .. off .. "-")
end
local code, resp, buffer, sock = httpclient.request_raw(uri, options)
if not code then
-- No success
file:close()
return code, resp, buffer
elseif hdr.Range and code ~= 206 then
-- We wanted a part but we got the while file
sock:close()
file:close()
return nil, -4, code, resp
elseif not hdr.Range and code ~= 200 then
-- We encountered an error
sock:close()
file:close()
return nil, -4, code, resp
end
if cbs.on_header then
cbs.on_header(file, code, resp)
end
local chunked = resp.headers["Transfer-Encoding"] == "chunked"
-- Write the buffer to file
file:writeall(buffer)
print ("Buffered data: " .. #buffer .. " Byte")
repeat
if not sock:is_socket() or chunked then
break
end
-- This is a plain TCP socket and there is no encoding so we can splice
local pipein, pipeout, msg = nixio.pipe()
if not pipein then
sock:close()
file:close()
return pipein, pipeout, msg
end
-- Disable blocking for the pipe otherwise we might end in a deadlock
local stat, code, msg = pipein:setblocking(false)
if stat then
stat, code, msg = pipeout:setblocking(false)
end
if not stat then
sock:close()
file:close()
return stat, code, msg
end
-- Adjust splice values
local ssize = 65536
local smode = nixio.splice_flags("move", "more", "nonblock")
local stat, code, msg = nixio.splice(sock, pipeout, ssize, smode)
if stat == nil then
break
end
local pollsock = {
{fd=sock, events=nixio.poll_flags("in")}
}
local pollfile = {
{fd=file, events=nixio.poll_flags("out")}
}
local done
repeat
-- Socket -> Pipe
repeat
nixio.poll(pollsock, 15000)
stat, code, msg = nixio.splice(sock, pipeout, ssize, smode)
if stat == nil then
sock:close()
file:close()
return stat, code, msg
elseif stat == 0 then
done = true
break
end
until stat == false
-- Pipe -> File
repeat
nixio.poll(pollfile, 15000)
stat, code, msg = nixio.splice(pipein, file, ssize, smode)
if stat == nil then
sock:close()
file:close()
return stat, code, msg
end
until stat == false
if cbs.on_write then
cbs.on_write(file)
end
until done
file:close()
sock:close()
return true
until true
print "Warning: splice() failed, falling back to read/write mode"
local src = chunked and httpclient.chunksource(sock) or sock:blocksource()
local snk = file:sink()
if cbs.on_write then
src = ltn12.source.chain(src, function(chunk)
cbs.on_write(file)
return chunk
end)
end
-- Fallback to read/write
local stat, code, msg = ltn12.pump.all(src, snk)
if stat then
file:close()
sock:close()
end
return stat, code, msg
end

View file

@ -2,3 +2,9 @@ src/libaxtls.a
.depend
.config.*
_stage
conf
lex.zconf.c
lkc_defs.h
mconf
zconf.tab.h
zconf.tab.c

View file

@ -14,26 +14,31 @@ $Id$
local table = require "table"
local nixio = require "nixio"
local getmetatable, assert = getmetatable, assert
local getmetatable, assert, pairs = getmetatable, assert, pairs
module "nixio.util"
local BUFFERSIZE = 8096
local socket = nixio.socket_meta
local tls_socket = nixio.tls_socket_meta
local socket = nixio.meta_socket
local tls_socket = nixio.meta_tls_socket
local file = nixio.meta_file
function socket.is_socket(self)
local meta = {}
function meta.is_socket(self)
return (getmetatable(self) == socket)
end
tls_socket.is_socket = socket.is_socket
function socket.is_tls_socket(self)
function meta.is_tls_socket(self)
return (getmetatable(self) == tls_socket)
end
tls_socket.is_tls_socket = socket.is_tls_socket
function socket.recvall(self, len)
local block, code, msg = self:recv(len)
function meta.is_file(self)
return (getmetatable(self) == file)
end
function meta.readall(self, len)
local block, code, msg = self:read(len)
if not block then
return "", code, msg, len
@ -44,7 +49,7 @@ function socket.recvall(self, len)
local data, total = {block}, #block
while len > total do
block, code, msg = self:recv(len - total)
block, code, msg = self:read(len - total)
if not block then
return data, code, msg, len - #data
@ -57,11 +62,11 @@ function socket.recvall(self, len)
return (#data > 1 and table.concat(data) or data[1]), nil, nil, 0
end
tls_socket.recvall = socket.recvall
meta.recvall = meta.readall
function socket.sendall(self, data)
function meta.writeall(self, data)
local total, block = 0
local sent, code, msg = self:send(data)
local sent, code, msg = self:write(data)
if not sent then
return total, code, msg, data
@ -69,7 +74,7 @@ function socket.sendall(self, data)
while sent < #data do
block, total = data:sub(sent + 1), total + sent
sent, code, msg = self:send(block)
sent, code, msg = self:write(block)
if not sent then
return total, code, msg, block
@ -78,9 +83,9 @@ function socket.sendall(self, data)
return total + sent, nil, nil, ""
end
tls_socket.sendall = socket.sendall
meta.sendall = meta.writeall
function socket.linesource(self, limit)
function meta.linesource(self, limit)
limit = limit or BUFFERSIZE
local buffer = ""
local bpos = 0
@ -100,7 +105,7 @@ function socket.linesource(self, limit)
bpos = endp
return line
elseif #buffer < limit + bpos then
local newblock, code = self:recv(limit + bpos - #buffer)
local newblock, code = self:read(limit + bpos - #buffer)
if not newblock then
return nil, code
elseif #newblock == 0 then
@ -114,9 +119,8 @@ function socket.linesource(self, limit)
end
end
end
tls_socket.linesource = socket.linesource
function socket.blocksource(self, bs, limit)
function meta.blocksource(self, bs, limit)
bs = bs or BUFFERSIZE
return function()
local toread = bs
@ -128,7 +132,7 @@ function socket.blocksource(self, bs, limit)
end
end
local block, code, msg = self:recv(toread)
local block, code, msg = self:read(toread)
if not block then
return nil, code
@ -143,9 +147,25 @@ function socket.blocksource(self, bs, limit)
end
end
end
tls_socket.blocksource = socket.blocksource
function meta.sink(self, close)
return function(chunk, src_err)
if not chunk and not src_err and close then
self:close()
elseif chunk and #chunk > 0 then
return self:writeall(chunk)
end
return true
end
end
function tls_socket.close(self)
self:shutdown()
return self.socket:close()
end
for k, v in pairs(meta) do
file[k] = v
socket[k] = v
tls_socket[k] = v
end

View file

@ -78,51 +78,44 @@ static int nixio_pipe(lua_State *L) {
}
static int nixio_file_write(lua_State *L) {
FILE *fp = nixio__checkfile(L);
size_t len, written;
int fd = nixio__checkfd(L, 1);
size_t len;
ssize_t sent;
const char *data = luaL_checklstring(L, 2, &len);
written = fwrite(data, sizeof(char), len, fp);
if (written < 0) {
do {
sent = write(fd, data, len);
} while(sent == -1 && errno == EINTR);
if (sent >= 0) {
lua_pushinteger(L, sent);
return 1;
} else {
return nixio__perror(L);
}
}
static int nixio_file_read(lua_State *L) {
int fd = nixio__checkfd(L, 1);
char buffer[NIXIO_BUFFERSIZE];
int req = luaL_checkinteger(L, 2);
int readc;
/* We limit the readsize to NIXIO_BUFFERSIZE */
req = (req > NIXIO_BUFFERSIZE) ? NIXIO_BUFFERSIZE : req;
do {
readc = read(fd, buffer, req);
} while (readc == -1 && errno == EINTR);
if (readc < 0) {
return nixio__perror(L);
} else {
lua_pushnumber(L, written);
lua_pushlstring(L, buffer, readc);
return 1;
}
}
/* Some code borrowed from Lua 5.1.4 liolib.c */
static int nixio_file_read(lua_State *L) {
FILE *f = nixio__checkfile(L);
size_t n = (size_t)luaL_checkinteger(L, 2);
luaL_argcheck(L, 2, n >= 0, "invalid length");
if (n == 0) {
if (feof(f)) {
return 0;
} else {
lua_pushliteral(L, "");
return 1;
}
}
size_t rlen; /* how much to read */
size_t nr; /* number of chars actually read */
luaL_Buffer b;
luaL_buffinit(L, &b);
rlen = LUAL_BUFFERSIZE; /* try to read that much each time */
do {
char *p = luaL_prepbuffer(&b);
if (rlen > n) rlen = n; /* cannot read more than asked */
nr = fread(p, sizeof(char), rlen, f);
luaL_addsize(&b, nr);
n -= nr; /* still have to read `n' chars */
} while (n > 0 && nr == rlen); /* until end of count or eof */
luaL_pushresult(&b); /* close buffer */
return (n == 0 || lua_objlen(L, -1) > 0);
}
static int nixio_file_seek(lua_State *L) {
FILE *f = nixio__checkfile(L);
off_t len = (off_t)luaL_checknumber(L, 2);
@ -176,7 +169,7 @@ static int nixio_file_lock(lua_State *L) {
}
}
return nixio__pstatus(L, flock(fd, flags));
return nixio__pstatus(L, !flock(fd, flags));
}
static int nixio_file_close(lua_State *L) {
@ -232,5 +225,5 @@ void nixio_open_file(lua_State *L) {
luaL_register(L, NULL, M);
lua_pushvalue(L, -1);
lua_setfield(L, -2, "__index");
lua_pop(L, 1);
lua_setfield(L, -2, "meta_file");
}

View file

@ -169,6 +169,8 @@ static const luaL_reg M[] = {
{"sendto", nixio_sock_sendto},
{"recv", nixio_sock_recv},
{"recvfrom",nixio_sock_recvfrom},
{"write", nixio_sock_send},
{"read", nixio_sock_recv},
{NULL, NULL}
};

View file

@ -107,7 +107,7 @@ LUALIB_API int luaopen_nixio(lua_State *L) {
/* register metatable as socket_meta */
lua_pushvalue(L, -2);
lua_setfield(L, -2, "socket_meta");
lua_setfield(L, -2, "meta_socket");
/* register methods */
nixio_open_file(L);
@ -126,7 +126,7 @@ LUALIB_API int luaopen_nixio(lua_State *L) {
lua_setfield(L, -2, "version");
/* some constants */
lua_createtable(L, 0, 1);
lua_createtable(L, 0, 7);
NIXIO_PUSH_CONSTANT(EACCES);
NIXIO_PUSH_CONSTANT(ENOSYS);
@ -134,6 +134,7 @@ LUALIB_API int luaopen_nixio(lua_State *L) {
NIXIO_PUSH_CONSTANT(EWOULDBLOCK);
NIXIO_PUSH_CONSTANT(EAGAIN);
NIXIO_PUSH_CONSTANT(ENOMEM);
NIXIO_PUSH_CONSTANT(ENOENT);
lua_setfield(L, -2, "const");

View file

@ -20,6 +20,7 @@
#include <poll.h>
#include <time.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include "nixio.h"
@ -48,18 +49,6 @@ static int nixio_nanosleep(lua_State *L) {
}
}
/**
* Checks whether a flag is set in the table and translates it into a bitmap
*/
static void nixio_poll_flags__w(lua_State *L, int *map, int f, const char *t) {
lua_pushstring(L, t);
lua_rawget(L, -2);
if (lua_toboolean(L, -1)) {
*map |= f;
}
lua_pop(L, 1);
}
/**
* Checks whether a flag is set in the bitmap and sets the matching table value
*/
@ -78,17 +67,7 @@ static void nixio_poll_flags__r(lua_State *L, int *map, int f, const char *t) {
*/
static int nixio_poll_flags(lua_State *L) {
int flags;
if (lua_istable(L, 1)) {
lua_settop(L, 1);
flags = 0;
nixio_poll_flags__w(L, &flags, POLLIN, "in");
nixio_poll_flags__w(L, &flags, POLLPRI, "pri");
nixio_poll_flags__w(L, &flags, POLLOUT, "out");
nixio_poll_flags__w(L, &flags, POLLERR, "err");
nixio_poll_flags__w(L, &flags, POLLHUP, "hup");
nixio_poll_flags__w(L, &flags, POLLNVAL, "nval");
lua_pushinteger(L, flags);
} else {
if (lua_isnumber(L, 1)) {
flags = luaL_checkinteger(L, 1);
lua_newtable(L);
nixio_poll_flags__r(L, &flags, POLLIN, "in");
@ -97,6 +76,29 @@ static int nixio_poll_flags(lua_State *L) {
nixio_poll_flags__r(L, &flags, POLLERR, "err");
nixio_poll_flags__r(L, &flags, POLLHUP, "hup");
nixio_poll_flags__r(L, &flags, POLLNVAL, "nval");
} else {
flags = 0;
const int j = lua_gettop(L);
for (int i=1; i<=j; i++) {
const char *flag = luaL_checkstring(L, i);
if (!strcmp(flag, "in")) {
flags |= POLLIN;
} else if (!strcmp(flag, "pri")) {
flags |= POLLPRI;
} else if (!strcmp(flag, "out")) {
flags |= POLLOUT;
} else if (!strcmp(flag, "err")) {
flags |= POLLERR;
} else if (!strcmp(flag, "hup")) {
flags |= POLLHUP;
} else if (!strcmp(flag, "nval")) {
flags |= POLLNVAL;
} else {
return luaL_argerror(L, i,
"supported values: in, pri, out, err, hup, nval");
}
}
lua_pushinteger(L, flags);
}
return 1;
}

View file

@ -30,6 +30,7 @@
*/
static int nixio_sock_setblocking(lua_State *L) {
int fd = nixio__checkfd(L, 1);
luaL_checkany(L, 2);
int set = lua_toboolean(L, 2);
int flags = fcntl(fd, F_GETFL);
@ -37,7 +38,7 @@ static int nixio_sock_setblocking(lua_State *L) {
return nixio__perror(L);
}
if (set) {
if (!set) {
flags |= O_NONBLOCK;
} else {
flags &= ~O_NONBLOCK;

View file

@ -20,6 +20,7 @@
#include "nixio.h"
#include <fcntl.h>
#include <string.h>
#include <sys/sendfile.h>
/* guess what sucks... */
@ -45,28 +46,24 @@ ssize_t splice(int __fdin, __off64_t *__offin, int __fdout,
#endif /* __UCLIBC__ */
/**
* Checks whether a flag is set in the table and translates it into a bitmap
*/
static void nixio_splice_flags__w(lua_State *L, int *m, int f, const char *t) {
lua_pushstring(L, t);
lua_rawget(L, -2);
if (lua_toboolean(L, -1)) {
*m |= f;
}
lua_pop(L, 1);
}
/**
* Translate integer to poll flags and vice versa
* Translate splice flags to integer
*/
static int nixio_splice_flags(lua_State *L) {
const int j = lua_gettop(L);
int flags = 0;
luaL_checktype(L, 1, LUA_TTABLE);
lua_settop(L, 1);
nixio_splice_flags__w(L, &flags, SPLICE_F_MOVE, "move");
nixio_splice_flags__w(L, &flags, SPLICE_F_NONBLOCK, "nonblock");
nixio_splice_flags__w(L, &flags, SPLICE_F_MORE, "more");
for (int i=1; i<=j; i++) {
const char *flag = luaL_checkstring(L, i);
if (!strcmp(flag, "move")) {
flags |= SPLICE_F_MOVE;
} else if (!strcmp(flag, "nonblock")) {
flags |= SPLICE_F_NONBLOCK;
} else if (!strcmp(flag, "more")) {
flags |= SPLICE_F_MORE;
} else {
return luaL_argerror(L, i, "supported values: "
"move, nonblock, more");
}
}
lua_pushinteger(L, flags);
return 1;

View file

@ -96,13 +96,15 @@ static int nixio_tls_sock_recv(lua_State *L) {
/* There is an error */
free(t->pbuffer);
t->pbuffer = t->pbufpos = NULL;
t->pbufsiz = 0;
if (axread != SSL_ERROR_CONN_LOST) {
t->pbufsiz = 0;
return nixio__tls_sock_perror(L, sock, axread);
} else {
if (!t->pbufsiz) {
lua_pushliteral(L, "");
} else {
t->pbufsiz = 0;
}
}
} else {
@ -198,6 +200,8 @@ static int nixio_tls_sock__tostring(lua_State *L) {
static const luaL_reg M[] = {
{"recv", nixio_tls_sock_recv},
{"send", nixio_tls_sock_send},
{"read", nixio_tls_sock_recv},
{"write", nixio_tls_sock_send},
{"accept", nixio_tls_sock_accept},
{"connect", nixio_tls_sock_connect},
{"shutdown", nixio_tls_sock_shutdown},
@ -213,5 +217,5 @@ void nixio_open_tls_socket(lua_State *L) {
luaL_register(L, NULL, M);
lua_pushvalue(L, -1);
lua_setfield(L, -2, "__index");
lua_setfield(L, -2, "tls_socket_meta");
lua_setfield(L, -2, "meta_tls_socket");
}