Difuse/luci
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

libs/web: rewrite template engine, merge lmo library

Browse source 
- template parser: merge lmo library
	- template parser: rewrite to operate on memory mapped files
	- template parser: implement proper line number reporting on syntax errors
	- template parser: process translate tags directly and bypass Lua
	- template lmo: introduce load_catalog(), change_catalog() and close_catalog()
	- template lmo: rewrite index processing to operate directly on the memory mapped file
	- template lmo: implement binary search keys, reducing the lookup complexity to O(log n)
	- po2lmo: write sorted indixes when generating *.lmo archives
	- i18n: use the template parser for translations
	- i18n: stub load(), loadc() and clear()
	- i18n: map setlanguage() to load_catalog()
This commit is contained in:
Jo-Philipp Wich 2012-11-25 19:17:55 +00:00
parent c647ff9f0e
commit 0e50aa690a
19 changed files with 975 additions and 1269 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
libs/lmo/Makefile
View file

@ -1,46 +0,0 @@
ifneq (,$(wildcard ../../build/config.mk))
include ../../build/config.mk
include ../../build/module.mk
include ../../build/gccconfig.mk
else
include standalone.mk
endif
LMO_LDFLAGS =
LMO_CFLAGS =
LMO_SO = lmo.so
LMO_PO2LMO = po2lmo
LMO_LOOKUP = lookup
LMO_COMMON_OBJ = src/lmo_core.o src/lmo_hash.o
LMO_PO2LMO_OBJ = src/lmo_po2lmo.o
LMO_LOOKUP_OBJ = src/lmo_lookup.o
LMO_LUALIB_OBJ = src/lmo_lualib.o
%.o: %.c
$(COMPILE) $(LMO_CFLAGS) $(LUA_CFLAGS) $(FPIC) -c -o $@ $<
compile: build-clean $(LMO_COMMON_OBJ) $(LMO_PO2LMO_OBJ) $(LMO_LOOKUP_OBJ) $(LMO_LUALIB_OBJ)
$(LINK) $(SHLIB_FLAGS) $(LMO_LDFLAGS) -o src/$(LMO_SO) \
$(LMO_COMMON_OBJ) $(LMO_LUALIB_OBJ)
$(LINK) $(LMO_LDFLAGS) -o src/$(LMO_PO2LMO) $(LMO_COMMON_OBJ) $(LMO_PO2LMO_OBJ)
$(LINK) $(LMO_LDFLAGS) -o src/$(LMO_LOOKUP) $(LMO_COMMON_OBJ) $(LMO_LOOKUP_OBJ)
mkdir -p dist$(LUA_LIBRARYDIR)
cp src/$(LMO_SO) dist$(LUA_LIBRARYDIR)/$(LMO_SO)
install: build
cp -pR dist$(LUA_LIBRARYDIR)/* $(LUA_LIBRARYDIR)
clean: build-clean
build-clean:
rm -f src/*.o src/lookup src/po2lmo src/lmo.so
host-compile: build-clean host-clean $(LMO_COMMON_OBJ) $(LMO_PO2LMO_OBJ)
$(LINK) $(LMO_LDFLAGS) -o src/$(LMO_PO2LMO) $(LMO_COMMON_OBJ) $(LMO_PO2LMO_OBJ)
host-install: host-compile
cp src/$(LMO_PO2LMO) ../../build/$(LMO_PO2LMO)
host-clean:
rm -f ../../build/$(LMO_PO2LMO)

234
libs/lmo/src/lmo_core.c
View file

@ -1,234 +0,0 @@
/*
* lmo - Lua Machine Objects - Base functions
*
* Copyright (C) 2009-2010 Jo-Philipp Wich <xm@subsignal.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "lmo.h"
extern char _lmo_error[1024];
static int lmo_read32( int fd, uint32_t *val )
{
if( read(fd, val, 4) < 4 )
return -1;
*val = ntohl(*val);
return 4;
}
static char * error(const char *message, int add_errno)
{
memset(_lmo_error, 0, sizeof(_lmo_error));
if( add_errno )
snprintf(_lmo_error, sizeof(_lmo_error),
"%s: %s", message, strerror(errno));
else
snprintf(_lmo_error, sizeof(_lmo_error), "%s", message);
return NULL;
}
const char * lmo_error(void)
{
return _lmo_error;
}
lmo_archive_t * lmo_open(const char *file)
{
int in = -1;
uint32_t idx_offset = 0;
uint32_t i;
struct stat s;
lmo_archive_t *ar = NULL;
lmo_entry_t *head = NULL;
lmo_entry_t *entry = NULL;
if( stat(file, &s) == -1 )
{
error("Can not stat file", 1);
goto cleanup;
}
if( (in = open(file, O_RDONLY)) == -1 )
{
error("Can not open file", 1);
goto cleanup;
}
if( lseek(in, -sizeof(uint32_t), SEEK_END) == -1 )
{
error("Can not seek to eof", 1);
goto cleanup;
}
if( lmo_read32(in, &idx_offset) != 4 )
{
error("Unexpected EOF while reading index offset", 0);
goto cleanup;
}
if( lseek(in, (off_t)idx_offset, SEEK_SET) == -1 )
{
error("Can not seek to index offset", 1);
goto cleanup;
}
if( (ar = (lmo_archive_t *) malloc(sizeof(lmo_archive_t))) != NULL )
{
ar->fd = in;
ar->length = idx_offset;
fcntl(ar->fd, F_SETFD, fcntl(ar->fd, F_GETFD) | FD_CLOEXEC);
for( i = idx_offset;
i < (s.st_size - sizeof(uint32_t));
i += (4 * sizeof(uint32_t))
) {
if( (entry = (lmo_entry_t *) malloc(sizeof(lmo_entry_t))) != NULL )
{
if( (lmo_read32(ar->fd, &entry->key_id) == 4) &&
(lmo_read32(ar->fd, &entry->val_id) == 4) &&
(lmo_read32(ar->fd, &entry->offset) == 4) &&
(lmo_read32(ar->fd, &entry->length) == 4)
) {
entry->next = head;
head = entry;
}
else
{
error("Unexpected EOF while reading index entry", 0);
goto cleanup;
}
}
else
{
error("Out of memory", 0);
goto cleanup;
}
}
ar->index = head;
if( lseek(ar->fd, 0, SEEK_SET) == -1 )
{
error("Can not seek to start", 1);
goto cleanup;
}
if( (ar->mmap = mmap(NULL, ar->length, PROT_READ, MAP_PRIVATE, ar->fd, 0)) == MAP_FAILED )
{
error("Failed to memory map archive contents", 1);
goto cleanup;
}
return ar;
}
else
{
error("Out of memory", 0);
goto cleanup;
}
cleanup:
if( in > -1 )
close(in);
if( head != NULL )
{
entry = head;
while( entry != NULL )
{
head = entry->next;
free(entry);
entry = head;
}
head = entry = NULL;
}
if( ar != NULL )
{
if( (ar->mmap != NULL) && (ar->mmap != MAP_FAILED) )
munmap(ar->mmap, ar->length);
free(ar);
ar = NULL;
}
return NULL;
}
void lmo_close(lmo_archive_t *ar)
{
lmo_entry_t *head = NULL;
lmo_entry_t *entry = NULL;
if( ar != NULL )
{
entry = ar->index;
while( entry != NULL )
{
head = entry->next;
free(entry);
entry = head;
}
head = entry = NULL;
if( (ar->mmap != NULL) && (ar->mmap != MAP_FAILED) )
munmap(ar->mmap, ar->length);
close(ar->fd);
free(ar);
ar = NULL;
}
}
int lmo_lookup(lmo_archive_t *ar, const char *key, char *dest, int len)
{
uint32_t look_key = sfh_hash(key, strlen(key));
int copy_len = -1;
lmo_entry_t *entry;
if( !ar )
return copy_len;
entry = ar->index;
while( entry != NULL )
{
if( entry->key_id == look_key )
{
copy_len = ((len - 1) > entry->length) ? entry->length : (len - 1);
memcpy(dest, &ar->mmap[entry->offset], copy_len);
dest[copy_len] = '\0';
break;
}
entry = entry->next;
}
return copy_len;
}

53
libs/lmo/src/lmo_hash.c
View file

@ -1,53 +0,0 @@
/*
* Hash function from http://www.azillionmonkeys.com/qed/hash.html
* Copyright (C) 2004-2008 by Paul Hsieh
*/
#include "lmo.h"
uint32_t sfh_hash(const char * data, int len)
{
uint32_t hash = len, tmp;
int rem;
if (len <= 0 || data == NULL) return 0;
rem = len & 3;
len >>= 2;
/* Main loop */
for (;len > 0; len--) {
hash += sfh_get16(data);
tmp = (sfh_get16(data+2) << 11) ^ hash;
hash = (hash << 16) ^ tmp;
data += 2*sizeof(uint16_t);
hash += hash >> 11;
}
/* Handle end cases */
switch (rem) {
case 3: hash += sfh_get16(data);
hash ^= hash << 16;
hash ^= data[sizeof(uint16_t)] << 18;
hash += hash >> 11;
break;
case 2: hash += sfh_get16(data);
hash ^= hash << 11;
hash += hash >> 17;
break;
case 1: hash += *data;
hash ^= hash << 10;
hash += hash >> 1;
}
/* Force "avalanching" of final 127 bits */
hash ^= hash << 3;
hash += hash >> 5;
hash ^= hash << 4;
hash += hash >> 17;
hash ^= hash << 25;
hash += hash >> 6;
return hash;
}

58
libs/lmo/src/lmo_lookup.c
View file

@ -1,58 +0,0 @@
/*
* lmo - Lua Machine Objects - Lookup utility
*
* Copyright (C) 2009 Jo-Philipp Wich <xm@subsignal.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "lmo.h"
extern char _lmo_error[1024];
static void die(const char *msg)
{
printf("Error: %s\n", msg);
exit(1);
}
static void usage(const char *name)
{
printf("Usage: %s input.lmo key\n", name);
exit(1);
}
int main(int argc, char *argv[])
{
char val[4096];
lmo_archive_t *ar = NULL;
if( argc != 3 )
usage(argv[0]);
if( (ar = (lmo_archive_t *) lmo_open(argv[1])) != NULL )
{
if( lmo_lookup(ar, argv[2], val, sizeof(val)) > -1 )
{
printf("%s\n", val);
}
lmo_close(ar);
}
else
{
die(lmo_error());
}
return 0;
}

263
libs/lmo/src/lmo_lualib.c
View file

@ -1,263 +0,0 @@
/*
* lmo - Lua Machine Objects - Lua binding
*
* Copyright (C) 2009-2012 Jo-Philipp Wich <xm@subsignal.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "lmo_lualib.h"
extern char _lmo_error[1024];
static int lmo_L_open(lua_State *L) {
const char *filename = luaL_checklstring(L, 1, NULL);
lmo_archive_t *ar, **udata;
if( (ar = lmo_open(filename)) != NULL )
{
if( (udata = lua_newuserdata(L, sizeof(lmo_archive_t *))) != NULL )
{
*udata = ar;
luaL_getmetatable(L, LMO_ARCHIVE_META);
lua_setmetatable(L, -2);
return 1;
}
lmo_close(ar);
lua_pushnil(L);
lua_pushstring(L, "out of memory");
return 2;
}
lua_pushnil(L);
lua_pushstring(L, lmo_error());
return 2;
}
static uint32_t _lmo_hash_string(lua_State *L, int n) {
size_t len;
const char *str = luaL_checklstring(L, n, &len);
char res[4096];
char *ptr, prev;
if (!str || len >= sizeof(res))
return 0;
for (prev = ' ', ptr = res; *str; prev = *str, str++)
{
if (isspace(*str))
{
if (!isspace(prev))
*ptr++ = ' ';
}
else
{
*ptr++ = *str;
}
}
if ((ptr > res) && isspace(*(ptr-1)))
ptr--;
return sfh_hash(res, ptr - res);
}
static int lmo_L_hash(lua_State *L) {
uint32_t hash = _lmo_hash_string(L, 1);
lua_pushinteger(L, (lua_Integer)hash);
return 1;
}
static lmo_luaentry_t *_lmo_push_entry(lua_State *L) {
lmo_luaentry_t *le;
if( (le = lua_newuserdata(L, sizeof(lmo_luaentry_t))) != NULL )
{
luaL_getmetatable(L, LMO_ENTRY_META);
lua_setmetatable(L, -2);
return le;
}
return NULL;
}
static int _lmo_lookup(lua_State *L, lmo_archive_t *ar, uint32_t hash) {
lmo_entry_t *e = ar->index;
lmo_luaentry_t *le = NULL;
while( e != NULL )
{
if( e->key_id == hash )
{
if( (le = _lmo_push_entry(L)) != NULL )
{
le->archive = ar;
le->entry = e;
return 1;
}
else
{
lua_pushnil(L);
lua_pushstring(L, "out of memory");
return 2;
}
}
e = e->next;
}
lua_pushnil(L);
return 1;
}
static int lmo_L_get(lua_State *L) {
lmo_archive_t **ar = luaL_checkudata(L, 1, LMO_ARCHIVE_META);
uint32_t hash = (uint32_t) luaL_checkinteger(L, 2);
return _lmo_lookup(L, *ar, hash);
}
static int lmo_L_lookup(lua_State *L) {
lmo_archive_t **ar = luaL_checkudata(L, 1, LMO_ARCHIVE_META);
uint32_t hash = _lmo_hash_string(L, 2);
return _lmo_lookup(L, *ar, hash);
}
static int lmo_L_foreach(lua_State *L) {
lmo_archive_t **ar = luaL_checkudata(L, 1, LMO_ARCHIVE_META);
lmo_entry_t *e = (*ar)->index;
if( lua_isfunction(L, 2) )
{
while( e != NULL )
{
lua_pushvalue(L, 2);
lua_pushinteger(L, e->key_id);
lua_pushlstring(L, &(*ar)->mmap[e->offset], e->length);
lua_pcall(L, 2, 0, 0);
e = e->next;
}
}
return 0;
}
static int lmo_L__gc(lua_State *L) {
lmo_archive_t **ar = luaL_checkudata(L, 1, LMO_ARCHIVE_META);
if( (*ar) != NULL )
lmo_close(*ar);
*ar = NULL;
return 0;
}
static int lmo_L__tostring(lua_State *L) {
lmo_archive_t **ar = luaL_checkudata(L, 1, LMO_ARCHIVE_META);
lua_pushfstring(L, "LMO Archive (%d bytes)", (*ar)->length);
return 1;
}
static int _lmo_convert_entry(lua_State *L, int idx) {
lmo_luaentry_t *le = luaL_checkudata(L, idx, LMO_ENTRY_META);
lua_pushlstring(L,
&le->archive->mmap[le->entry->offset],
le->entry->length
);
return 1;
}
static int lmo_L_entry__tostring(lua_State *L) {
return _lmo_convert_entry(L, 1);
}
static int lmo_L_entry__concat(lua_State *L) {
if( lua_isuserdata(L, 1) )
_lmo_convert_entry(L, 1);
else
lua_pushstring(L, lua_tostring(L, 1));
if( lua_isuserdata(L, 2) )
_lmo_convert_entry(L, 2);
else
lua_pushstring(L, lua_tostring(L, 2));
lua_concat(L, 2);
return 1;
}
static int lmo_L_entry__len(lua_State *L) {
lmo_luaentry_t *le = luaL_checkudata(L, 1, LMO_ENTRY_META);
lua_pushinteger(L, le->entry->length);
return 1;
}
static int lmo_L_entry__gc(lua_State *L) {
lmo_luaentry_t *le = luaL_checkudata(L, 1, LMO_ENTRY_META);
le->archive = NULL;
le->entry = NULL;
return 0;
}
/* lmo method table */
static const luaL_reg M[] = {
{"close", lmo_L__gc},
{"get", lmo_L_get},
{"lookup", lmo_L_lookup},
{"foreach", lmo_L_foreach},
{"__tostring", lmo_L__tostring},
{"__gc", lmo_L__gc},
{NULL, NULL}
};
/* lmo.entry method table */
static const luaL_reg E[] = {
{"__tostring", lmo_L_entry__tostring},
{"__concat", lmo_L_entry__concat},
{"__len", lmo_L_entry__len},
{"__gc", lmo_L_entry__gc},
{NULL, NULL}
};
/* module table */
static const luaL_reg R[] = {
{"open", lmo_L_open},
{"hash", lmo_L_hash},
{NULL, NULL}
};
LUALIB_API int luaopen_lmo(lua_State *L) {
luaL_newmetatable(L, LMO_ARCHIVE_META);
luaL_register(L, NULL, M);
lua_pushvalue(L, -1);
lua_setfield(L, -2, "__index");
lua_setglobal(L, LMO_ARCHIVE_META);
luaL_newmetatable(L, LMO_ENTRY_META);
luaL_register(L, NULL, E);
lua_pushvalue(L, -1);
lua_setfield(L, -2, "__index");
lua_setglobal(L, LMO_ENTRY_META);
luaL_register(L, LMO_LUALIB_META, R);
return 1;
}

43
libs/lmo/src/lmo_lualib.h
View file

@ -1,43 +0,0 @@
/*
* lmo - Lua Machine Objects - Lua library header
*
* Copyright (C) 2009-2012 Jo-Philipp Wich <xm@subsignal.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef _LMO_LUALIB_H_
#define _LMO_LUALIB_H_
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
#include <ctype.h>
#include "lmo.h"
#define LMO_LUALIB_META "lmo"
#define LMO_ARCHIVE_META "lmo.archive"
#define LMO_ENTRY_META "lmo.entry"
struct lmo_luaentry {
lmo_archive_t *archive;
lmo_entry_t *entry;
};
typedef struct lmo_luaentry lmo_luaentry_t;
LUALIB_API int luaopen_lmo(lua_State *L);
#endif

22
libs/web/Makefile
View file

@ -1,19 +1,28 @@
ifneq (,$(wildcard ../../build/config.mk))
include ../../build/config.mk
include ../../build/module.mk
include ../../build/gccconfig.mk
else
include standalone.mk
endif
TPL_LDFLAGS =
TPL_CFLAGS =
TPL_SO = parser.so
TPL_PO2LMO = po2lmo
TPL_PO2LMO_OBJ = src/po2lmo.o
TPL_LMO_OBJ = src/template_lmo.o
TPL_COMMON_OBJ = src/template_parser.o src/template_utils.o
TPL_LUALIB_OBJ = src/template_lualib.o
%.o: %.c
$(COMPILE) $(TPL_CFLAGS) $(LUA_CFLAGS) $(FPIC) -c -o $@ $<
compile: build-clean $(TPL_COMMON_OBJ) $(TPL_LUALIB_OBJ)
compile: build-clean $(TPL_COMMON_OBJ) $(TPL_LUALIB_OBJ) $(TPL_LMO_OBJ) $(TPL_PO2LMO_OBJ)
$(LINK) $(SHLIB_FLAGS) $(TPL_LDFLAGS) -o src/$(TPL_SO) \
$(TPL_COMMON_OBJ) $(TPL_LUALIB_OBJ)
$(TPL_COMMON_OBJ) $(TPL_LMO_OBJ) $(TPL_LUALIB_OBJ)
$(LINK) -o src/$(TPL_PO2LMO) \
$(TPL_LMO_OBJ) $(TPL_PO2LMO_OBJ)
mkdir -p dist$(LUCI_LIBRARYDIR)/template
cp src/$(TPL_SO) dist$(LUCI_LIBRARYDIR)/template/$(TPL_SO)
@ -24,3 +33,12 @@ clean: build-clean
build-clean:
rm -f src/*.o src/$(TPL_SO)
host-compile: build-clean host-clean $(TPL_LMO_OBJ) $(TPL_PO2LMO_OBJ)
$(LINK) -o src/$(TPL_PO2LMO) $(TPL_LMO_OBJ) $(TPL_PO2LMO_OBJ)
host-install: host-compile
cp src/$(TPL_PO2LMO) ../../build/$(TPL_PO2LMO)
host-clean:
rm -f ../../build/$(TPL_PO2LMO)

44
libs/web/luasrc/i18n.lua
View file

@ -27,7 +27,8 @@ limitations under the License.
--- LuCI translation library.
module("luci.i18n", package.seeall)
require("luci.util")
require("lmo")
local tparser = require "luci.template.parser"
table = {}
i18ndir = luci.util.libpath() .. "/i18n/"
@ -37,7 +38,6 @@ default = "en"
--- Clear the translation table.
function clear()
table = {}
end
--- Load a translation and copy its data into the translation table.
@ -46,33 +46,6 @@ end
-- @param force Force reload even if already loaded (optional)
-- @return Success status
function load(file, lang, force)
lang = lang and lang:gsub("_", "-") or ""
if force or not loaded[lang] or not loaded[lang][file] then
local f = lmo.open(i18ndir .. file .. "." .. lang .. ".lmo")
if f then
if not table[lang] then
table[lang] = { f }
setmetatable(table[lang], {
__index = function(tbl, key)
for i = 1, #tbl do
local s = rawget(tbl, i):lookup(key)
if s then return s end
end
end
})
else
table[lang][#table[lang]+1] = f
end
loaded[lang] = loaded[lang] or {}
loaded[lang][file] = true
return true
else
return false
end
else
return true
end
end
--- Load a translation file using the default translation language.
@ -80,9 +53,6 @@ end
-- @param file Language file
-- @param force Force reload even if already loaded (optional)
function loadc(file, force)
load(file, default, force)
if context.parent then load(file, context.parent, force) end
return load(file, context.lang, force)
end
--- Set the context default translation language.
@ -90,16 +60,18 @@ end
function setlanguage(lang)
context.lang = lang:gsub("_", "-")
context.parent = (context.lang:match("^([a-z][a-z])_"))
if not tparser.load_catalog(context.lang, i18ndir) then
if context.parent then
tparser.load_catalog(context.parent, i18ndir)
end
end
end
--- Return the translated value for a specific translation key.
-- @param key Default translation text
-- @return Translated string
function translate(key)
return (table[context.lang] and table[context.lang][key])
or (table[context.parent] and table[context.parent][key])
or (table[default] and table[default][key])
or key
return tparser.translate(key) or key
end
--- Return the translated value for a specific translation key and use it as sprintf pattern.

5
libs/web/luasrc/template.lua
View file

@ -79,9 +79,8 @@ function Template.__init__(self, name)
-- If we have no valid template throw error, otherwise cache the template
if not self.template then
error("Failed to load template '" .. name .. "'.\n" ..
"Error while parsing template '" .. sourcefile .. "'.\n" ..
"A syntax error occured near '" ..
(err or "(nil)"):gsub("\t", "\\t"):gsub("\n", "\\n") .. "'.")
"Error while parsing template '" .. sourcefile .. "':\n" ..
(err or "Unknown syntax error"))
else
self.cache[name] = self.template
end

80
libs/lmo/src/lmo_po2lmo.c → libs/web/src/po2lmo.c
View file

@ -1,7 +1,7 @@
/*
* lmo - Lua Machine Objects - PO to LMO conversion tool
*
* Copyright (C) 2009-2011 Jo-Philipp Wich <xm@subsignal.org>
* Copyright (C) 2009-2012 Jo-Philipp Wich <xm@subsignal.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,7 +16,7 @@
* limitations under the License.
*/
#include "lmo.h"
#include "template_lmo.h"
static void die(const char *msg)
{
@ -82,6 +82,34 @@ static int extract_string(const char *src, char *dest, int len)
return (off > -1) ? strlen(dest) : -1;
}
static int cmp_index(const void *a, const void *b)
{
uint32_t x = ntohl(((const lmo_entry_t *)a)->key_id);
uint32_t y = ntohl(((const lmo_entry_t *)b)->key_id);
if (x < y)
return -1;
else if (x > y)
return 1;
return 0;
}
static void print_index(void *array, int n, FILE *out)
{
lmo_entry_t *e;
qsort(array, n, sizeof(*e), cmp_index);
for (e = array; n > 0; n--, e++)
{
print(&e->key_id, sizeof(uint32_t), 1, out);
print(&e->val_id, sizeof(uint32_t), 1, out);
print(&e->offset, sizeof(uint32_t), 1, out);
print(&e->length, sizeof(uint32_t), 1, out);
}
}
int main(int argc, char *argv[])
{
char line[4096];
@ -91,14 +119,14 @@ int main(int argc, char *argv[])
int state = 0;
int offset = 0;
int length = 0;
int n_entries = 0;
void *array = NULL;
lmo_entry_t *entry = NULL;
uint32_t key_id, val_id;
FILE *in;
FILE *out;
lmo_entry_t *head = NULL;
lmo_entry_t *entry = NULL;
if( (argc != 3) || ((in = fopen(argv[1], "r")) == NULL) || ((out = fopen(argv[2], "w")) == NULL) )
usage(argv[0]);
@ -167,26 +195,22 @@ int main(int argc, char *argv[])
if( key_id != val_id )
{
if( (entry = (lmo_entry_t *) malloc(sizeof(lmo_entry_t))) != NULL )
{
memset(entry, 0, sizeof(entry));
length = strlen(val) + ((4 - (strlen(val) % 4)) % 4);
n_entries++;
array = realloc(array, n_entries * sizeof(lmo_entry_t));
entry = (lmo_entry_t *)array + n_entries - 1;
entry->key_id = htonl(key_id);
entry->val_id = htonl(val_id);
entry->offset = htonl(offset);
entry->length = htonl(strlen(val));
print(val, length, 1, out);
offset += length;
entry->next = head;
head = entry;
}
else
{
if (!array)
die("Out of memory");
}
entry->key_id = htonl(key_id);
entry->val_id = htonl(val_id);
entry->offset = htonl(offset);
entry->length = htonl(strlen(val));
length = strlen(val) + ((4 - (strlen(val) % 4)) % 4);
print(val, length, 1, out);
offset += length;
}
}
@ -198,15 +222,7 @@ int main(int argc, char *argv[])
memset(line, 0, sizeof(line));
}
entry = head;
while( entry != NULL )
{
print(&entry->key_id, sizeof(uint32_t), 1, out);
print(&entry->val_id, sizeof(uint32_t), 1, out);
print(&entry->offset, sizeof(uint32_t), 1, out);
print(&entry->length, sizeof(uint32_t), 1, out);
entry = entry->next;
}
print_index(array, n_entries, out);
if( offset > 0 )
{

325
libs/web/src/template_lmo.c Normal file
View file

@ -0,0 +1,325 @@
/*
* lmo - Lua Machine Objects - Base functions
*
* Copyright (C) 2009-2010 Jo-Philipp Wich <xm@subsignal.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "template_lmo.h"
/*
* Hash function from http://www.azillionmonkeys.com/qed/hash.html
* Copyright (C) 2004-2008 by Paul Hsieh
*/
uint32_t sfh_hash(const char *data, int len)
{
uint32_t hash = len, tmp;
int rem;
if (len <= 0 || data == NULL) return 0;
rem = len & 3;
len >>= 2;
/* Main loop */
for (;len > 0; len--) {
hash += sfh_get16(data);
tmp = (sfh_get16(data+2) << 11) ^ hash;
hash = (hash << 16) ^ tmp;
data += 2*sizeof(uint16_t);
hash += hash >> 11;
}
/* Handle end cases */
switch (rem) {
case 3: hash += sfh_get16(data);
hash ^= hash << 16;
hash ^= data[sizeof(uint16_t)] << 18;
hash += hash >> 11;
break;
case 2: hash += sfh_get16(data);
hash ^= hash << 11;
hash += hash >> 17;
break;
case 1: hash += *data;
hash ^= hash << 10;
hash += hash >> 1;
}
/* Force "avalanching" of final 127 bits */
hash ^= hash << 3;
hash += hash >> 5;
hash ^= hash << 4;
hash += hash >> 17;
hash ^= hash << 25;
hash += hash >> 6;
return hash;
}
uint32_t lmo_canon_hash(const char *str, int len)
{
char res[4096];
char *ptr, prev;
int off;
if (!str || len >= sizeof(res))
return 0;
for (prev = ' ', ptr = res, off = 0; off < len; prev = *str, off++, str++)
{
if (isspace(*str))
{
if (!isspace(prev))
*ptr++ = ' ';
}
else
{
*ptr++ = *str;
}
}
if ((ptr > res) && isspace(*(ptr-1)))
ptr--;
return sfh_hash(res, ptr - res);
}
lmo_archive_t * lmo_open(const char *file)
{
int in = -1;
uint32_t idx_offset = 0;
struct stat s;
lmo_archive_t *ar = NULL;
if (stat(file, &s) == -1)
goto err;
if ((in = open(file, O_RDONLY)) == -1)
goto err;
if ((ar = (lmo_archive_t *)malloc(sizeof(*ar))) != NULL)
{
memset(ar, 0, sizeof(*ar));
ar->fd = in;
ar->size = s.st_size;
fcntl(ar->fd, F_SETFD, fcntl(ar->fd, F_GETFD) | FD_CLOEXEC);
if ((ar->mmap = mmap(NULL, ar->size, PROT_READ, MAP_SHARED, ar->fd, 0)) == MAP_FAILED)
goto err;
idx_offset = *((const uint32_t *)
(ar->mmap + ar->size - sizeof(uint32_t)));
if (idx_offset >= ar->size)
goto err;
ar->index = (lmo_entry_t *)(ar->mmap + idx_offset);
ar->length = (ar->size - idx_offset - sizeof(uint32_t)) / sizeof(lmo_entry_t);
ar->end = ar->mmap + ar->size;
return ar;
}
err:
if (in > -1)
close(in);
if (ar != NULL)
{
if ((ar->mmap != NULL) && (ar->mmap != MAP_FAILED))
munmap(ar->mmap, ar->size);
free(ar);
}
return NULL;
}
void lmo_close(lmo_archive_t *ar)
{
if (ar != NULL)
{
if ((ar->mmap != NULL) && (ar->mmap != MAP_FAILED))
munmap(ar->mmap, ar->size);
close(ar->fd);
free(ar);
ar = NULL;
}
}
lmo_catalog_t *_lmo_catalogs = NULL;
lmo_catalog_t *_lmo_active_catalog = NULL;
int lmo_load_catalog(const char *lang, const char *dir)
{
DIR *dh = NULL;
char pattern[16];
char path[PATH_MAX];
struct dirent *de = NULL;
lmo_archive_t *ar = NULL;
lmo_catalog_t *cat = NULL;
if (!lmo_change_catalog(lang))
return 0;
if (!dir || !(dh = opendir(dir)))
goto err;
if (!(cat = malloc(sizeof(*cat))))
goto err;
memset(cat, 0, sizeof(*cat));
snprintf(cat->lang, sizeof(cat->lang), "%s", lang);
snprintf(pattern, sizeof(pattern), "*.%s.lmo", lang);
while ((de = readdir(dh)) != NULL)
{
if (!fnmatch(pattern, de->d_name, 0))
{
snprintf(path, sizeof(path), "%s/%s", dir, de->d_name);
ar = lmo_open(path);
if (ar)
{
ar->next = cat->archives;
cat->archives = ar;
}
}
}
closedir(dh);
cat->next = _lmo_catalogs;
_lmo_catalogs = cat;
if (!_lmo_active_catalog)
_lmo_active_catalog = cat;
return 0;
err:
if (dh) closedir(dh);
if (cat) free(cat);
return -1;
}
int lmo_change_catalog(const char *lang)
{
lmo_catalog_t *cat;
for (cat = _lmo_catalogs; cat; cat = cat->next)
{
if (!strncmp(cat->lang, lang, sizeof(cat->lang)))
{
_lmo_active_catalog = cat;
return 0;
}
}
return -1;
}
static lmo_entry_t * lmo_find_entry(lmo_archive_t *ar, uint32_t hash)
{
unsigned int m, l, r;
l = 0;
r = ar->length - 1;
while (1)
{
m = l + ((r - l) / 2);
if (r < l)
break;
if (ar->index[m].key_id == hash)
return &ar->index[m];
if (ar->index[m].key_id > hash)
{
if (!m)
break;
r = m - 1;
}
else
{
l = m + 1;
}
}
return NULL;
}
int lmo_translate(const char *key, int keylen, char **out, int *outlen)
{
uint32_t hash;
lmo_entry_t *e;
lmo_archive_t *ar;
if (!key || !_lmo_active_catalog)
return -2;
hash = htonl(lmo_canon_hash(key, keylen));
for (ar = _lmo_active_catalog->archives; ar; ar = ar->next)
{
if ((e = lmo_find_entry(ar, hash)) != NULL)
{
*out = ar->mmap + e->offset;
*outlen = e->length;
return 0;
}
}
return -1;
}
void lmo_close_catalog(const char *lang)
{
lmo_archive_t *ar, *next;
lmo_catalog_t *cat, *prev;
for (prev = NULL, cat = _lmo_catalogs; cat; prev = cat, cat = cat->next)
{
if (!strncmp(cat->lang, lang, sizeof(cat->lang)))
{
if (prev)
prev->next = cat->next;
else
_lmo_catalogs = cat->next;
for (ar = cat->archives; ar; ar = next)
{
next = ar->next;
lmo_close(ar);
}
free(cat);
break;
}
}
}

39
libs/lmo/src/lmo.h → libs/web/src/template_lmo.h
View file

@ -1,7 +1,7 @@
/*
* lmo - Lua Machine Objects - General header
*
* Copyright (C) 2009 Jo-Philipp Wich <xm@subsignal.org>
* Copyright (C) 2009-2012 Jo-Philipp Wich <xm@subsignal.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,8 +16,8 @@
* limitations under the License.
*/
#ifndef _LMO_H_
#define _LMO_H_
#ifndef _TEMPLATE_LMO_H_
#define _TEMPLATE_LMO_H_
#include <stdlib.h>
#include <stdio.h>
@ -29,7 +29,9 @@
#include <arpa/inet.h>
#include <unistd.h>
#include <errno.h>
#include <fnmatch.h>
#include <dirent.h>
#include <ctype.h>
#if (defined(__GNUC__) && defined(__i386__))
#define sfh_get16(d) (*((const uint16_t *) (d)))
@ -44,7 +46,6 @@ struct lmo_entry {
uint32_t val_id;
uint32_t offset;
uint32_t length;
struct lmo_entry *next;
} __attribute__((packed));
typedef struct lmo_entry lmo_entry_t;
@ -52,21 +53,39 @@ typedef struct lmo_entry lmo_entry_t;
struct lmo_archive {
int fd;
uint32_t length;
int length;
uint32_t size;
lmo_entry_t *index;
char *mmap;
char *end;
struct lmo_archive *next;
};
typedef struct lmo_archive lmo_archive_t;
uint32_t sfh_hash(const char * data, int len);
struct lmo_catalog {
char lang[6];
struct lmo_archive *archives;
struct lmo_catalog *next;
};
char _lmo_error[1024];
const char * lmo_error(void);
typedef struct lmo_catalog lmo_catalog_t;
uint32_t sfh_hash(const char *data, int len);
uint32_t lmo_canon_hash(const char *data, int len);
lmo_archive_t * lmo_open(const char *file);
int lmo_lookup(lmo_archive_t *ar, const char *key, char *dest, int len);
void lmo_close(lmo_archive_t *ar);
extern lmo_catalog_t *_lmo_catalogs;
extern lmo_catalog_t *_lmo_active_catalog;
int lmo_load_catalog(const char *lang, const char *dir);
int lmo_change_catalog(const char *lang);
int lmo_translate(const char *key, int keylen, char **out, int *outlen);
void lmo_close_catalog(const char *lang);
#endif

96
libs/web/src/template_lualib.c
View file

@ -21,37 +21,27 @@
int template_L_parse(lua_State *L)
{
const char *file = luaL_checkstring(L, 1);
struct template_parser parser;
int lua_status;
struct template_parser *parser = template_open(file);
int lua_status, rv;
if( (parser.fd = open(file, O_RDONLY)) > 0 )
if (!parser)
{
parser.flags = 0;
parser.bufsize = 0;
parser.state = T_STATE_TEXT_NEXT;
lua_status = lua_load(L, template_reader, &parser, file);
(void) close(parser.fd);
if( lua_status == 0 )
{
return 1;
}
else
{
lua_pushnil(L);
lua_pushinteger(L, lua_status);
lua_pushlstring(L, parser.out, parser.outsize);
return 3;
}
lua_pushnil(L);
lua_pushinteger(L, errno);
lua_pushstring(L, strerror(errno));
return 3;
}
lua_pushnil(L);
lua_pushinteger(L, 255);
lua_pushstring(L, "No such file or directory");
return 3;
lua_status = lua_load(L, template_reader, parser, file);
if (lua_status == 0)
rv = 1;
else
rv = template_error(L, parser);
template_close(parser);
return rv;
}
int template_L_sanitize_utf8(lua_State *L)
@ -88,12 +78,64 @@ int template_L_sanitize_pcdata(lua_State *L)
return 0;
}
static int template_L_load_catalog(lua_State *L) {
const char *lang = luaL_optstring(L, 1, "en");
const char *dir = luaL_optstring(L, 2, NULL);
lua_pushboolean(L, !lmo_load_catalog(lang, dir));
return 1;
}
static int template_L_close_catalog(lua_State *L) {
const char *lang = luaL_optstring(L, 1, "en");
lmo_close_catalog(lang);
return 0;
}
static int template_L_change_catalog(lua_State *L) {
const char *lang = luaL_optstring(L, 1, "en");
lua_pushboolean(L, !lmo_change_catalog(lang));
return 1;
}
static int template_L_translate(lua_State *L) {
size_t len;
char *tr;
int trlen;
const char *key = luaL_checklstring(L, 1, &len);
switch (lmo_translate(key, len, &tr, &trlen))
{
case 0:
lua_pushlstring(L, tr, trlen);
return 1;
case -1:
return 0;
}
lua_pushnil(L);
lua_pushstring(L, "no catalog loaded");
return 2;
}
static int template_L_hash(lua_State *L) {
size_t len;
const char *key = luaL_checklstring(L, 1, &len);
lua_pushinteger(L, sfh_hash(key, len));
return 1;
}
/* module table */
static const luaL_reg R[] = {
{ "parse", template_L_parse },
{ "sanitize_utf8", template_L_sanitize_utf8 },
{ "sanitize_pcdata", template_L_sanitize_pcdata },
{ "load_catalog", template_L_load_catalog },
{ "close_catalog", template_L_close_catalog },
{ "change_catalog", template_L_change_catalog },
{ "translate", template_L_translate },
{ "hash", template_L_hash },
{ NULL, NULL }
};

1
libs/web/src/template_lualib.h
View file

@ -21,6 +21,7 @@
#include "template_parser.h"
#include "template_utils.h"
#include "template_lmo.h"
#define TEMPLATE_LUALIB_META "template.parser"

723
libs/web/src/template_parser.c
View file

@ -1,7 +1,7 @@
/*
* LuCI Template - Parser implementation
*
* Copyright (C) 2009 Jo-Philipp Wich <xm@subsignal.org>
* Copyright (C) 2009-2012 Jo-Philipp Wich <xm@subsignal.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -17,17 +17,21 @@
*/
#include "template_parser.h"
#include "template_utils.h"
#include "template_lmo.h"
/* leading and trailing code for different types */
const char * gen_code[7][2] = {
const char *gen_code[9][2] = {
{ NULL, NULL },
{ "write(\"", "\")" },
{ NULL, NULL },
{ "write(tostring(", " or \"\"))" },
{ "include(\"", "\")" },
{ "write(pcdata(translate(\"", "\")))" },
{ "write(translate(\"", "\"))" },
{ NULL, " " }
{ "write(\"", "\")" },
{ "write(\"", "\")" },
{ NULL, " " },
{ NULL, NULL },
};
/* Simple strstr() like function that takes len arguments for both haystack and needle. */
@ -59,407 +63,326 @@ static char *strfind(char *haystack, int hslen, const char *needle, int ndlen)
return NULL;
}
/*
* Inspect current read buffer and find the number of "vague" characters at the end
* which could indicate an opening token. Returns the number of "vague" chars.
* The last continuous sequence of whitespace, optionally followed by a "<" is
* treated as "vague" because whitespace may be discarded if the upcoming opening
* token indicates pre-whitespace-removal ("<%-"). A single remaining "<" char
* can't be differentiated from an opening token ("<%"), so it's kept to be processed
* in the next cycle.
*/
static int stokscan(struct template_parser *data, int off, int no_whitespace)
struct template_parser * template_open(const char *file)
{
int i;
int skip = 0;
int tokoff = data->bufsize - 1;
struct stat s;
static struct template_parser *parser;
for( i = tokoff; i >= off; i-- )
if (!(parser = malloc(sizeof(*parser))))
goto err;
memset(parser, 0, sizeof(*parser));
parser->fd = -1;
parser->file = file;
if (stat(file, &s))
goto err;
if ((parser->fd = open(file, O_RDONLY)) < 0)
goto err;
parser->size = s.st_size;
parser->mmap = mmap(NULL, parser->size, PROT_READ, MAP_PRIVATE,
parser->fd, 0);
if (parser->mmap != MAP_FAILED)
{
if( data->buf[i] == T_TOK_START[0] )
{
skip = tokoff - i + 1;
tokoff = i - 1;
break;
}
parser->off = parser->mmap;
parser->cur_chunk.type = T_TYPE_INIT;
parser->cur_chunk.s = parser->mmap;
parser->cur_chunk.e = parser->mmap;
return parser;
}
if( !no_whitespace )
{
for( i = tokoff; i >= off; i-- )
{
if( isspace(data->buf[i]) )
skip++;
else
break;
}
}
return skip;
}
/*
* Similar to stokscan() but looking for closing token indicators.
* Matches "-", optionally followed by a "%" char.
*/
static int etokscan(struct template_parser *data)
{
int skip = 0;
if( (data->bufsize > 0) && (data->buf[data->bufsize-1] == T_TOK_END[0]) )
skip++;
if( (data->bufsize > skip) && (data->buf[data->bufsize-skip-1] == T_TOK_SKIPWS[0]) )
skip++;
return skip;
}
/*
* Generate Lua expressions from the given raw code, write it into the
* output buffer and set the lua_Reader specific size pointer.
* Takes parser-state, lua_Reader's size pointer and generator flags
* as parameter. The given flags indicate whether leading or trailing
* code should be added. Returns a pointer to the output buffer.
*/
static const char * generate_expression(struct template_parser *data, size_t *sz, int what)
{
char tmp[T_OUTBUFSZ];
int i;
int size = 0;
int start = 0;
int whitespace = 0;
memset(tmp, 0, T_OUTBUFSZ);
/* Inject leading expression code (if any) */
if( (what & T_GEN_START) && (gen_code[data->type][0] != NULL) )
{
memcpy(tmp, gen_code[data->type][0], strlen(gen_code[data->type][0]));
size += strlen(gen_code[data->type][0]);
}
/* Parse source buffer */
for( i = 0; i < data->outsize; i++ )
{
/* Skip leading whitespace for non-raw and non-expr chunks */
if( !start && isspace(data->out[i]) && (data->type == T_TYPE_I18N ||
data->type == T_TYPE_I18N_RAW || data->type == T_TYPE_INCLUDE) )
continue;
else if( !start )
start = 1;
/* Found whitespace after i18n key */
if( data->type == T_TYPE_I18N || data->type == T_TYPE_I18N_RAW )
{
/* Is initial whitespace, insert space */
if( !whitespace && isspace(data->out[i]) )
{
tmp[size++] = ' ';
whitespace = 1;
}
/* Suppress subsequent whitespace, escape special chars */
else if( !isspace(data->out[i]) )
{
if( data->out[i] == '\\' || data->out[i] == '"' )
tmp[size++] = '\\';
tmp[size++] = data->out[i];
whitespace = 0;
}
}
/* Escape quotes, backslashes and newlines for plain and include expressions */
else if( (data->type == T_TYPE_TEXT || data->type == T_TYPE_INCLUDE) &&
(data->out[i] == '\\' || data->out[i] == '"' || data->out[i] == '\n' || data->out[i] == '\t') )
{
tmp[size++] = '\\';
switch(data->out[i])
{
case '\n':
tmp[size++] = 'n';
break;
case '\t':
tmp[size++] = 't';
break;
default:
tmp[size++] = data->out[i];
}
}
/* Normal char */
else
{
tmp[size++] = data->out[i];
}
}
/* Inject trailing expression code (if any) */
if( (what & T_GEN_END) && (gen_code[data->type][1] != NULL) )
{
/* Strip trailing space for i18n expressions */
if( data->type == T_TYPE_I18N || data->type == T_TYPE_I18N_RAW )
if( (size > 0) && (tmp[size-1] == ' ') )
size--;
memcpy(&tmp[size], gen_code[data->type][1], strlen(gen_code[data->type][1]));
size += strlen(gen_code[data->type][1]);
}
*sz = data->outsize = size;
memset(data->out, 0, T_OUTBUFSZ);
memcpy(data->out, tmp, size);
//printf("<<<%i|%i|%i|%s>>>\n", what, data->type, *sz, data->out);
return data->out;
}
/*
* Move the number of bytes specified in data->bufsize from the
* given source pointer to the beginning of the read buffer.
*/
static void bufmove(struct template_parser *data, const char *src)
{
if( data->bufsize > 0 )
memmove(data->buf, src, data->bufsize);
else if( data->bufsize < 0 )
data->bufsize = 0;
data->buf[data->bufsize] = 0;
}
/*
* Move the given amount of bytes from the given source pointer
* to the output buffer and set data->outputsize.
*/
static void bufout(struct template_parser *data, const char *src, int len)
{
if( len >= 0 )
{
memset(data->out, 0, T_OUTBUFSZ);
memcpy(data->out, src, len);
data->outsize = len;
}
else
{
data->outsize = 0;
}
}
/*
* lua_Reader compatible function that parses template code on demand from
* the given file handle.
*/
const char *template_reader(lua_State *L, void *ud, size_t *sz)
{
struct template_parser *data = ud;
char *match = NULL;
int off = 0;
int ignore = 0;
int genflags = 0;
int readlen = 0;
int vague = 0;
while( !(data->flags & T_FLAG_EOF) || (data->bufsize > 0) )
{
/* Fill buffer */
if( !(data->flags & T_FLAG_EOF) && (data->bufsize < T_READBUFSZ) )
{
if( (readlen = read(data->fd, &data->buf[data->bufsize], T_READBUFSZ - data->bufsize)) > 0 )
data->bufsize += readlen;
else if( readlen == 0 )
data->flags |= T_FLAG_EOF;
else
return NULL;
}
/* Evaluate state */
switch(data->state)
{
/* Plain text chunk (before "<%") */
case T_STATE_TEXT_INIT:
case T_STATE_TEXT_NEXT:
off = 0; ignore = 0; *sz = 0;
data->type = T_TYPE_TEXT;
/* Skip leading whitespace if requested */
if( data->flags & T_FLAG_SKIPWS )
{
data->flags &= ~T_FLAG_SKIPWS;
while( (off < data->bufsize) && isspace(data->buf[off]) )
off++;
}
/* Found "<%" */
if( (match = strfind(&data->buf[off], data->bufsize - off - 1, T_TOK_START, strlen(T_TOK_START))) != NULL )
{
readlen = (int)(match - &data->buf[off]);
data->bufsize -= (readlen + strlen(T_TOK_START) + off);
match += strlen(T_TOK_START);
/* Check for leading '-' */
if( match[0] == T_TOK_SKIPWS[0] )
{
data->bufsize--;
match++;
while( (readlen > 1) && isspace(data->buf[off+readlen-1]) )
{
readlen--;
}
}
bufout(data, &data->buf[off], readlen);
bufmove(data, match);
data->state = T_STATE_CODE_INIT;
}
/* Maybe plain chunk */
else
{
/* Preserve trailing "<" or white space, maybe a start token */
vague = stokscan(data, off, 0);
/* We can process some bytes ... */
if( vague < data->bufsize )
{
readlen = data->bufsize - vague - off;
}
/* No bytes to process, so try to remove at least whitespace ... */
else
{
/* ... but try to preserve trailing "<" ... */
vague = stokscan(data, off, 1);
if( vague < data->bufsize )
{
readlen = data->bufsize - vague - off;
}
/* ... no chance, push out buffer */
else
{
readlen = vague - off;
vague = 0;
}
}
bufout(data, &data->buf[off], readlen);
data->state = T_STATE_TEXT_NEXT;
data->bufsize = vague;
bufmove(data, &data->buf[off+readlen]);
}
if( ignore || data->outsize == 0 )
continue;
else
return generate_expression(data, sz, T_GEN_START | T_GEN_END);
break;
/* Ignored chunk (inside "<%# ... %>") */
case T_STATE_SKIP:
ignore = 1;
/* Initial code chunk ("<% ...") */
case T_STATE_CODE_INIT:
off = 0;
/* Check for leading '-' */
if( data->buf[off] == T_TOK_SKIPWS[0] )
off++;
/* Determine code type */
switch(data->buf[off])
{
case '#':
ignore = 1;
off++;
data->type = T_TYPE_COMMENT;
break;
case '=':
off++;
data->type = T_TYPE_EXPR;
break;
case '+':
off++;
data->type = T_TYPE_INCLUDE;
break;
case ':':
off++;
data->type = T_TYPE_I18N;
break;
case '_':
off++;
data->type = T_TYPE_I18N_RAW;
break;
default:
data->type = T_TYPE_CODE;
break;
}
/* Subsequent code chunk ("..." or "... %>") */
case T_STATE_CODE_NEXT:
/* Found "%>" */
if( (match = strfind(&data->buf[off], data->bufsize - off, T_TOK_END, strlen(T_TOK_END))) != NULL )
{
genflags = ( data->state == T_STATE_CODE_INIT )
? (T_GEN_START | T_GEN_END) : T_GEN_END;
readlen = (int)(match - &data->buf[off]);
/* Check for trailing '-' */
if( (match > data->buf) && (*(match-1) == T_TOK_SKIPWS[0]) )
{
readlen--;
data->flags |= T_FLAG_SKIPWS;
}
bufout(data, &data->buf[off], readlen);
data->state = T_STATE_TEXT_INIT;
data->bufsize -= ((int)(match - &data->buf[off]) + strlen(T_TOK_END) + off);
bufmove(data, &match[strlen(T_TOK_END)]);
}
/* Code chunk */
else
{
genflags = ( data->state == T_STATE_CODE_INIT ) ? T_GEN_START : 0;
/* Preserve trailing "%" and "-", maybe an end token */
vague = etokscan(data);
readlen = data->bufsize - off - vague;
bufout(data, &data->buf[off], readlen);
data->state = T_STATE_CODE_NEXT;
data->bufsize = vague;
bufmove(data, &data->buf[readlen+off]);
}
if( ignore || (data->outsize == 0 && !genflags) )
continue;
else
return generate_expression(data, sz, genflags);
break;
}
}
*sz = 0;
err:
template_close(parser);
return NULL;
}
void template_close(struct template_parser *parser)
{
if (!parser)
return;
if (parser->gc != NULL)
free(parser->gc);
if ((parser->mmap != NULL) && (parser->mmap != MAP_FAILED))
munmap(parser->mmap, parser->size);
if (parser->fd >= 0)
close(parser->fd);
free(parser);
}
void template_text(struct template_parser *parser, const char *e)
{
const char *s = parser->off;
if (s < (parser->mmap + parser->size))
{
if (parser->strip_after)
{
while ((s <= e) && isspace(*s))
s++;
}
parser->cur_chunk.type = T_TYPE_TEXT;
}
else
{
parser->cur_chunk.type = T_TYPE_EOF;
}
parser->cur_chunk.line = parser->line;
parser->cur_chunk.s = s;
parser->cur_chunk.e = e;
}
void template_code(struct template_parser *parser, const char *e)
{
const char *s = parser->off;
parser->strip_before = 0;
parser->strip_after = 0;
if (*s == '-')
{
parser->strip_before = 1;
for (s++; (s <= e) && (*s == ' ' || *s == '\t'); s++);
}
if (*(e-1) == '-')
{
parser->strip_after = 1;
for (e--; (e >= s) && (*e == ' ' || *e == '\t'); e--);
}
switch (*s)
{
/* comment */
case '#':
s++;
parser->cur_chunk.type = T_TYPE_COMMENT;
break;
/* include */
case '+':
s++;
parser->cur_chunk.type = T_TYPE_INCLUDE;
break;
/* translate */
case ':':
s++;
parser->cur_chunk.type = T_TYPE_I18N;
break;
/* translate raw */
case '_':
s++;
parser->cur_chunk.type = T_TYPE_I18N_RAW;
break;
/* expr */
case '=':
s++;
parser->cur_chunk.type = T_TYPE_EXPR;
break;
/* code */
default:
parser->cur_chunk.type = T_TYPE_CODE;
break;
}
parser->cur_chunk.line = parser->line;
parser->cur_chunk.s = s;
parser->cur_chunk.e = e;
}
static const char *
template_format_chunk(struct template_parser *parser, size_t *sz)
{
const char *s, *p;
const char *head, *tail;
struct template_chunk *c = &parser->prv_chunk;
struct template_buffer *buf;
*sz = 0;
s = parser->gc = NULL;
if (parser->strip_before && c->type == T_TYPE_TEXT)
{
while ((c->e > c->s) && isspace(*(c->e - 1)))
c->e--;
}
/* empty chunk */
if (c->s == c->e)
{
if (c->type == T_TYPE_EOF)
{
*sz = 0;
s = NULL;
}
else
{
*sz = 1;
s = " ";
}
}
/* format chunk */
else if ((buf = buf_init(c->e - c->s)) != NULL)
{
if ((head = gen_code[c->type][0]) != NULL)
buf_append(buf, head, strlen(head));
switch (c->type)
{
case T_TYPE_TEXT:
escape_luastr(buf, c->s, c->e - c->s, 0);
break;
case T_TYPE_EXPR:
buf_append(buf, c->s, c->e - c->s);
for (p = c->s; p < c->e; p++)
parser->line += (*p == '\n');
break;
case T_TYPE_INCLUDE:
escape_luastr(buf, c->s, c->e - c->s, 0);
break;
case T_TYPE_I18N:
translate_luastr(buf, c->s, c->e - c->s, 1);
break;
case T_TYPE_I18N_RAW:
translate_luastr(buf, c->s, c->e - c->s, 0);
break;
case T_TYPE_CODE:
buf_append(buf, c->s, c->e - c->s);
for (p = c->s; p < c->e; p++)
parser->line += (*p == '\n');
break;
}
if ((tail = gen_code[c->type][1]) != NULL)
buf_append(buf, tail, strlen(tail));
*sz = buf_length(buf);
s = parser->gc = buf_destroy(buf);
if (!*sz)
{
*sz = 1;
s = " ";
}
}
return s;
}
const char *template_reader(lua_State *L, void *ud, size_t *sz)
{
struct template_parser *parser = ud;
int rem = parser->size - (parser->off - parser->mmap);
char *tag;
parser->prv_chunk = parser->cur_chunk;
/* free previous string */
if (parser->gc)
{
free(parser->gc);
parser->gc = NULL;
}
/* before tag */
if (!parser->in_expr)
{
if ((tag = strfind(parser->off, rem, "<%", 2)) != NULL)
{
template_text(parser, tag);
parser->off = tag + 2;
parser->in_expr = 1;
}
else
{
template_text(parser, parser->mmap + parser->size);
parser->off = parser->mmap + parser->size;
}
}
/* inside tag */
else
{
if ((tag = strfind(parser->off, rem, "%>", 2)) != NULL)
{
template_code(parser, tag);
parser->off = tag + 2;
parser->in_expr = 0;
}
else
{
/* unexpected EOF */
template_code(parser, parser->mmap + parser->size);
*sz = 1;
return "\033";
}
}
return template_format_chunk(parser, sz);
}
int template_error(lua_State *L, struct template_parser *parser)
{
const char *err = luaL_checkstring(L, -1);
const char *off = parser->prv_chunk.s;
const char *ptr;
char msg[1024];
int line = 0;
int chunkline = 0;
fprintf(stderr, "E[%s]\n", err);
if ((ptr = strfind((char *)err, strlen(err), "]:", 2)) != NULL)
{
chunkline = atoi(ptr + 2) - parser->prv_chunk.line;
while (*ptr)
{
if (*ptr++ == ' ')
{
err = ptr;
break;
}
}
}
if (strfind((char *)err, strlen(err), "'char(27)'", 10) != NULL)
{
off = parser->mmap + parser->size;
err = "'%>' expected before end of file";
chunkline = 0;
}
for (ptr = parser->mmap; ptr < off; ptr++)
if (*ptr == '\n')
line++;
snprintf(msg, sizeof(msg), "Syntax error in %s:%d: %s",
parser->file, line + chunkline, err ? err : "(unknown error)");
lua_pushnil(L);
lua_pushinteger(L, line + chunkline);
lua_pushstring(L, msg);
return 3;
}

72
libs/web/src/template_parser.h
View file

@ -21,61 +21,59 @@
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
#define T_READBUFSZ 1024
#define T_OUTBUFSZ T_READBUFSZ * 3
/* parser states */
#define T_STATE_TEXT_INIT 0
#define T_STATE_TEXT_NEXT 1
#define T_STATE_CODE_INIT 2
#define T_STATE_CODE_NEXT 3
#define T_STATE_SKIP 4
/* parser flags */
#define T_FLAG_EOF 0x01
#define T_FLAG_SKIPWS 0x02
/* tokens used in matching and expression generation */
#define T_TOK_START "<%"
#define T_TOK_END "%>"
#define T_TOK_SKIPWS "-"
/* generator flags */
#define T_GEN_START 0x01
#define T_GEN_END 0x02
/* code types */
#define T_TYPE_TEXT 0
#define T_TYPE_COMMENT 1
#define T_TYPE_EXPR 2
#define T_TYPE_INCLUDE 3
#define T_TYPE_I18N 4
#define T_TYPE_I18N_RAW 5
#define T_TYPE_CODE 6
#define T_TYPE_INIT 0
#define T_TYPE_TEXT 1
#define T_TYPE_COMMENT 2
#define T_TYPE_EXPR 3
#define T_TYPE_INCLUDE 4
#define T_TYPE_I18N 5
#define T_TYPE_I18N_RAW 6
#define T_TYPE_CODE 7
#define T_TYPE_EOF 8
struct template_chunk {
const char *s;
const char *e;
int type;
int line;
};
/* parser state */
struct template_parser {
int fd;
int bufsize;
int outsize;
int state;
int flags;
int type;
char buf[T_READBUFSZ];
char out[T_OUTBUFSZ];
uint32_t size;
char *mmap;
char *off;
char *gc;
int line;
int in_expr;
int strip_before;
int strip_after;
struct template_chunk prv_chunk;
struct template_chunk cur_chunk;
const char *file;
};
struct template_parser * template_open(const char *file);
void template_close(struct template_parser *parser);
const char *template_reader(lua_State *L, void *ud, size_t *sz);
int template_error(lua_State *L, struct template_parser *parser);
#endif

123
libs/web/src/template_utils.c
View file

@ -17,19 +17,23 @@
*/
#include "template_utils.h"
#include "template_lmo.h"
/* initialize a buffer object */
static struct template_buffer * buf_init(void)
struct template_buffer * buf_init(int size)
{
struct template_buffer *buf;
if (size <= 0)
size = 1024;
buf = (struct template_buffer *)malloc(sizeof(struct template_buffer));
if (buf != NULL)
{
buf->fill = 0;
buf->size = 1024;
buf->data = (unsigned char *)malloc(buf->size);
buf->size = size;
buf->data = malloc(buf->size);
if (buf->data != NULL)
{
@ -46,17 +50,21 @@ static struct template_buffer * buf_init(void)
}
/* grow buffer */
static int buf_grow(struct template_buffer *buf)
int buf_grow(struct template_buffer *buf, int size)
{
unsigned int off = (buf->dptr - buf->data);
unsigned char *data =
(unsigned char *)realloc(buf->data, buf->size + 1024);
char *data;
if (size <= 0)
size = 1024;
data = realloc(buf->data, buf->size + size);
if (data != NULL)
{
buf->data = data;
buf->dptr = data + off;
buf->size += 1024;
buf->size += size;
return buf->size;
}
@ -65,9 +73,9 @@ static int buf_grow(struct template_buffer *buf)
}
/* put one char into buffer object */
static int buf_putchar(struct template_buffer *buf, unsigned char c)
int buf_putchar(struct template_buffer *buf, char c)
{
if( ((buf->fill + 1) >= buf->size) && !buf_grow(buf) )
if( ((buf->fill + 1) >= buf->size) && !buf_grow(buf, 0) )
return 0;
*(buf->dptr++) = c;
@ -78,11 +86,11 @@ static int buf_putchar(struct template_buffer *buf, unsigned char c)
}
/* append data to buffer */
static int buf_append(struct template_buffer *buf, unsigned char *s, int len)
int buf_append(struct template_buffer *buf, const char *s, int len)
{
while ((buf->fill + len + 1) >= buf->size)
if ((buf->fill + len + 1) >= buf->size)
{
if (!buf_grow(buf))
if (!buf_grow(buf, len + 1))
return 0;
}
@ -95,13 +103,19 @@ static int buf_append(struct template_buffer *buf, unsigned char *s, int len)
return len;
}
/* destroy buffer object and return pointer to data */
static char * buf_destroy(struct template_buffer *buf)
/* read buffer length */
int buf_length(struct template_buffer *buf)
{
unsigned char *data = buf->data;
return buf->fill;
}
/* destroy buffer object and return pointer to data */
char * buf_destroy(struct template_buffer *buf)
{
char *data = buf->data;
free(buf);
return (char *)data;
return data;
}
@ -229,7 +243,7 @@ static int _validate_utf8(unsigned char **s, int l, struct template_buffer *buf)
!mb_is_surrogate(ptr, n) && !mb_is_illegal(ptr, n))
{
/* copy sequence */
if (!buf_append(buf, ptr, n))
if (!buf_append(buf, (char *)ptr, n))
return 0;
}
@ -266,7 +280,7 @@ static int _validate_utf8(unsigned char **s, int l, struct template_buffer *buf)
/* sanitize given string and replace all invalid UTF-8 sequences with "?" */
char * sanitize_utf8(const char *s, unsigned int l)
{
struct template_buffer *buf = buf_init();
struct template_buffer *buf = buf_init(l);
unsigned char *ptr = (unsigned char *)s;
unsigned int v, o;
@ -278,7 +292,7 @@ char * sanitize_utf8(const char *s, unsigned int l)
/* ascii char */
if ((*ptr >= 0x01) && (*ptr <= 0x7F))
{
if (!buf_putchar(buf, *ptr++))
if (!buf_putchar(buf, (char)*ptr++))
break;
}
@ -300,7 +314,7 @@ char * sanitize_utf8(const char *s, unsigned int l)
* Escape XML control chars */
char * sanitize_pcdata(const char *s, unsigned int l)
{
struct template_buffer *buf = buf_init();
struct template_buffer *buf = buf_init(l);
unsigned char *ptr = (unsigned char *)s;
unsigned int o, v;
char esq[8];
@ -329,7 +343,7 @@ char * sanitize_pcdata(const char *s, unsigned int l)
{
esl = snprintf(esq, sizeof(esq), "&#%i;", *ptr);
if (!buf_append(buf, (unsigned char *)esq, esl))
if (!buf_append(buf, esq, esl))
break;
ptr++;
@ -338,7 +352,7 @@ char * sanitize_pcdata(const char *s, unsigned int l)
/* ascii char */
else if (*ptr <= 0x7F)
{
buf_putchar(buf, *ptr++);
buf_putchar(buf, (char)*ptr++);
}
/* multi byte sequence */
@ -353,3 +367,68 @@ char * sanitize_pcdata(const char *s, unsigned int l)
return buf_destroy(buf);
}
void escape_luastr(struct template_buffer *out, const char *s, unsigned int l,
int escape_xml)
{
int esl;
char esq[8];
char *ptr;
for (ptr = (char *)s; ptr < (s + l); ptr++)
{
switch (*ptr)
{
case '\\':
buf_append(out, "\\\\", 2);
break;
case '"':
if (escape_xml)
buf_append(out, "&#34;", 5);
else
buf_append(out, "\\\"", 2);
break;
case '\n':
buf_append(out, "\\n", 2);
break;
case '\'':
case '&':
case '<':
case '>':
if (escape_xml)
{
esl = snprintf(esq, sizeof(esq), "&#%i;", *ptr);
buf_append(out, esq, esl);
break;
}
default:
buf_putchar(out, *ptr);
}
}
}
void translate_luastr(struct template_buffer *out, const char *s, unsigned int l,
int escape_xml)
{
char *tr;
int trlen;
switch (lmo_translate(s, l, &tr, &trlen))
{
case 0:
escape_luastr(out, tr, trlen, escape_xml);
break;
case -1:
escape_luastr(out, s, l, escape_xml);
break;
default:
/* no catalog loaded */
break;
}
}

17
libs/web/src/template_utils.h
View file

@ -1,7 +1,7 @@
/*
* LuCI Template - Utility header
*
* Copyright (C) 2010 Jo-Philipp Wich <xm@subsignal.org>
* Copyright (C) 2010-2012 Jo-Philipp Wich <xm@subsignal.org>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -22,17 +22,28 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <dlfcn.h>
/* buffer object */
struct template_buffer {
unsigned char *data;
unsigned char *dptr;
char *data;
char *dptr;
unsigned int size;
unsigned int fill;
};
struct template_buffer * buf_init(int size);
int buf_grow(struct template_buffer *buf, int size);
int buf_putchar(struct template_buffer *buf, char c);
int buf_append(struct template_buffer *buf, const char *s, int len);
int buf_length(struct template_buffer *buf);
char * buf_destroy(struct template_buffer *buf);
char * sanitize_utf8(const char *s, unsigned int l);
char * sanitize_pcdata(const char *s, unsigned int l);
void escape_luastr(struct template_buffer *out, const char *s, unsigned int l, int escape_xml);
void translate_luastr(struct template_buffer *out, const char *s, unsigned int l, int escape_xml);
#endif

0
libs/lmo/standalone.mk → libs/web/standalone.mk
View file