lib_utf8.c (6839B)
1 /* 2 ** Standard library for UTF-8 manipulation 3 ** 4 ** Mostly taken verbatim from the Lua interpreter. 5 ** See Copyright Notice in lua.h 6 */ 7 #define lib_utf8_c 8 #define LUA_LIB 9 10 #include <assert.h> 11 #include <limits.h> 12 #include <stdlib.h> 13 #include <string.h> 14 15 #include "lua.h" 16 #include "lauxlib.h" 17 #include "lualib.h" 18 #include "lj_obj.h" 19 #include "lj_lib.h" 20 21 #define LJLIB_MODULE_utf8 22 23 #define MAXUNICODE 0x10FFFF 24 25 #define iscont(p) ((*(p) & 0xC0) == 0x80) 26 27 #if LJ_53 28 29 /* from strlib */ 30 /* translate a relative string position: negative means back from end */ 31 static lua_Integer u_posrelat (lua_Integer pos, size_t len) { 32 if (pos >= 0) return pos; 33 else if (0u - (size_t)pos > len) return 0; 34 else return (lua_Integer)len + pos + 1; 35 } 36 37 38 /* 39 ** Decode one UTF-8 sequence, returning NULL if byte sequence is invalid. 40 */ 41 static const char *utf8_decode (const char *o, int *val) { 42 static const unsigned int limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFF}; 43 const unsigned char *s = (const unsigned char *)o; 44 unsigned int c = s[0]; 45 unsigned int res = 0; /* final result */ 46 if (c < 0x80) /* ascii? */ 47 res = c; 48 else { 49 int count = 0; /* to count number of continuation bytes */ 50 while (c & 0x40) { /* still have continuation bytes? */ 51 int cc = s[++count]; /* read next byte */ 52 if ((cc & 0xC0) != 0x80) /* not a continuation byte? */ 53 return NULL; /* invalid byte sequence */ 54 res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */ 55 c <<= 1; /* to test next bit */ 56 } 57 res |= ((c & 0x7F) << (count * 5)); /* add first byte */ 58 if (count > 3 || res > MAXUNICODE || res <= limits[count]) 59 return NULL; /* invalid byte sequence */ 60 s += count; /* skip continuation bytes read */ 61 } 62 if (val) *val = res; 63 return (const char *)s + 1; /* +1 to include first byte */ 64 } 65 66 67 /* 68 ** utf8len(s [, i [, j]]) --> number of characters that start in the 69 ** range [i,j], or nil + current position if 's' is not well formed in 70 ** that interval 71 */ 72 LJLIB_CF(utf8_len) 73 { 74 int n = 0; 75 size_t len; 76 const char *s = luaL_checklstring(L, 1, &len); 77 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); 78 lua_Integer posj = u_posrelat(luaL_optinteger(L, 3, -1), len); 79 luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 2, 80 "initial position out of string"); 81 luaL_argcheck(L, --posj < (lua_Integer)len, 3, 82 "final position out of string"); 83 while (posi <= posj) { 84 const char *s1 = utf8_decode(s + posi, NULL); 85 if (s1 == NULL) { /* conversion error? */ 86 lua_pushnil(L); /* return nil ... */ 87 lua_pushinteger(L, posi + 1); /* ... and current position */ 88 return 2; 89 } 90 posi = s1 - s; 91 n++; 92 } 93 lua_pushinteger(L, n); 94 return 1; 95 } 96 97 98 /* 99 ** codepoint(s, [i, [j]]) -> returns codepoints for all characters 100 ** that start in the range [i,j] 101 */ 102 LJLIB_CF(utf8_codepoint) 103 { 104 size_t len; 105 const char *s = luaL_checklstring(L, 1, &len); 106 lua_Integer posi = u_posrelat(luaL_optinteger(L, 2, 1), len); 107 lua_Integer pose = u_posrelat(luaL_optinteger(L, 3, posi), len); 108 int n; 109 const char *se; 110 luaL_argcheck(L, posi >= 1, 2, "out of range"); 111 luaL_argcheck(L, pose <= (lua_Integer)len, 3, "out of range"); 112 if (posi > pose) return 0; /* empty interval; return no values */ 113 if (pose - posi >= INT_MAX) /* (lua_Integer -> int) overflow? */ 114 return luaL_error(L, "string slice too long"); 115 n = (int)(pose - posi) + 1; 116 luaL_checkstack(L, n, "string slice too long"); 117 n = 0; 118 se = s + pose; 119 for (s += posi - 1; s < se;) { 120 int code; 121 s = utf8_decode(s, &code); 122 if (s == NULL) 123 return luaL_error(L, "invalid UTF-8 code"); 124 lua_pushinteger(L, code); 125 n++; 126 } 127 return n; 128 } 129 130 131 static void pushutfchar (lua_State *L, int arg) { 132 lua_Integer code = luaL_checkinteger(L, arg); 133 luaL_argcheck(L, 0 <= code && code <= MAXUNICODE, arg, "value out of range"); 134 lua_pushfstring(L, "%U", (long)code); 135 } 136 137 138 /* 139 ** utfchar(n1, n2, ...) -> char(n1)..char(n2)... 140 */ 141 LJLIB_CF(utf8_char) 142 { 143 int n = lua_gettop(L); /* number of arguments */ 144 if (n == 1) /* optimize common case of single char */ 145 pushutfchar(L, 1); 146 else { 147 int i; 148 luaL_Buffer b; 149 luaL_buffinit(L, &b); 150 for (i = 1; i <= n; i++) { 151 pushutfchar(L, i); 152 luaL_addvalue(&b); 153 } 154 luaL_pushresult(&b); 155 } 156 return 1; 157 } 158 159 160 /* 161 ** offset(s, n, [i]) -> index where n-th character counting from 162 ** position 'i' starts; 0 means character at 'i'. 163 */ 164 LJLIB_CF(utf8_offset) 165 { 166 size_t len; 167 const char *s = luaL_checklstring(L, 1, &len); 168 lua_Integer n = luaL_checkinteger(L, 2); 169 lua_Integer posi = (n >= 0) ? 1 : len + 1; 170 posi = u_posrelat(luaL_optinteger(L, 3, posi), len); 171 luaL_argcheck(L, 1 <= posi && --posi <= (lua_Integer)len, 3, 172 "position out of range"); 173 if (n == 0) { 174 /* find beginning of current byte sequence */ 175 while (posi > 0 && iscont(s + posi)) posi--; 176 } 177 else { 178 if (iscont(s + posi)) 179 luaL_error(L, "initial position is a continuation byte"); 180 if (n < 0) { 181 while (n < 0 && posi > 0) { /* move back */ 182 do { /* find beginning of previous character */ 183 posi--; 184 } while (posi > 0 && iscont(s + posi)); 185 n++; 186 } 187 } 188 else { 189 n--; /* do not move for 1st character */ 190 while (n > 0 && posi < (lua_Integer)len) { 191 do { /* find beginning of next character */ 192 posi++; 193 } while (iscont(s + posi)); /* (cannot pass final '\0') */ 194 n--; 195 } 196 } 197 } 198 if (n == 0) /* did it find given character? */ 199 lua_pushinteger(L, posi + 1); 200 else /* no such character */ 201 lua_pushnil(L); 202 return 1; 203 } 204 205 206 static int iter_aux (lua_State *L) { 207 size_t len; 208 const char *s = luaL_checklstring(L, 1, &len); 209 lua_Integer n = lua_tointeger(L, 2) - 1; 210 if (n < 0) /* first iteration? */ 211 n = 0; /* start from here */ 212 else if (n < (lua_Integer)len) { 213 n++; /* skip current byte */ 214 while (iscont(s + n)) n++; /* and its continuations */ 215 } 216 if (n >= (lua_Integer)len) 217 return 0; /* no more codepoints */ 218 else { 219 int code; 220 const char *next = utf8_decode(s + n, &code); 221 if (next == NULL || iscont(next)) 222 return luaL_error(L, "invalid UTF-8 code"); 223 lua_pushinteger(L, n + 1); 224 lua_pushinteger(L, code); 225 return 2; 226 } 227 } 228 229 230 LJLIB_CF(utf8_codes) 231 { 232 luaL_checkstring(L, 1); 233 lua_pushcfunction(L, iter_aux); 234 lua_pushvalue(L, 1); 235 lua_pushinteger(L, 0); 236 return 3; 237 } 238 239 /* pattern to match a single UTF-8 character */ 240 #define CHARPATTERN "[\0-\x7F\xC2-\xF4][\x80-\xBF]*" 241 242 #include "lj_libdef.h" 243 244 LUALIB_API int luaopen_utf8 (lua_State *L) { 245 LJ_LIB_REG(L, LUA_UTF8LIBNAME, utf8); 246 lua_pushliteral(L, CHARPATTERN); 247 lua_setfield(L, -2, "charpattern"); 248 return 1; 249 } 250 251 #endif