diff --git a/lslopt/lslbasefuncs.py b/lslopt/lslbasefuncs.py index 8519287..f0c5d38 100644 --- a/lslopt/lslbasefuncs.py +++ b/lslopt/lslbasefuncs.py @@ -897,10 +897,61 @@ def llBase64ToInteger(s): i = ord(s[0]) if s[0] < b'\x80' else ord(s[0])-256 return (i<<24)+(ord(s[1])<<16)+(ord(s[2])<<8)+ord(s[3]) +b64tos_re = re.compile( + b'(' + # Those pass through and are caught by InternalUTF8toString: + b'\x00$' # NUL at last position (zstr removes it) + b'|[\x01-\x7F\xFE\xFF]|[\xC2-\xDF][\x80-\xBF]' + b'|(?:\xE0[\xA0-\xBF]|[\xE1-\xEF][\x80-\xBF])[\x80-\xBF]' + b'|(?:\xF0[\x90-\xBF]|[\xF1-\xF7][\x80-\xBF])[\x80-\xBF]{2}' + b'|(?:\xF8[\x88-\xBF]|[\xF9-\xFB][\x80-\xBF])[\x80-\xBF]{3}' + b'|(?:\xFC[\x84-\xBF]|\xFD[\x80-\xBF])[\x80-\xBF]{4}' + b')|(' + # Those are caught here and substituted by a single "?" + # (greediness is important here): + b'[\x00\x80-\xBF]' + b'|[\xC0-\xDF][\x80-\xBF]?' + b'|[\xE0-\xEF][\x80-\xBF]{0,2}' + b'|[\xF0-\xF7][\x80-\xBF]{0,3}' + b'|[\xF8-\xFB][\x80-\xBF]{0,4}' + b'|[\xFC-\xFD][\x80-\xBF]{0,5}' + b')|(.)' # should never be reached +) + def llBase64ToString(s): assert isstring(s) s = b64_re.search(s).group(0) - return InternalUTF8toString(b64decode(s + u'='*(-len(s)&3))) + + # llUnescapeURL and llBase64ToString behave differently. + # llBase64ToString does a first check on the UTF-8 before the standard + # conversion, unlike llUnescapeURL. That makes it have a much more similar + # behaviour to LSO's than llUnescapeURL does. But LL being LL, the check + # is, of course, flawed, and some illegal sequences pass as good (but in + # Mono they are fortunately stopped on the conversion to UTF-8 instead). + # The check that llBase64ToString does has the quirk that the invalid + # sequences that it catches are treated as 1 single bad character instead + # of as many as the sequence has. The latter is what normal conversion to + # UTF-8 does. This causes inconsistencies in the number of ?'s returned. + + # In llBase64ToString, trailing NUL is stripped, and embedded NULs are + # converted to "?". + + byteseq = bytearray(b64decode(s + u'='*(-len(s)&3))) + + pos = 0 + match = b64tos_re.search(byteseq, pos) + while match is not None: + assert match.group(3) is None, 'Fail in b64tos_re: ' + match.group(3) + L = len(match.group(2) or '') + if L: + byteseq[pos:pos+L] = b'?' + pos = match.end(2) - L + 1 + else: + pos = match.end(1) + + match = b64tos_re.search(byteseq, pos) + + return InternalUTF8toString(bytes(byteseq)) def llCSV2List(s): assert isstring(s) diff --git a/testfuncs.py b/testfuncs.py index cfea5f6..9d291f9 100644 --- a/testfuncs.py +++ b/testfuncs.py @@ -22,6 +22,7 @@ from lslopt.lslfuncs import * from lslopt import lslcommon import sys import math +from base64 import b64encode StopAtFirstError = False @@ -151,6 +152,14 @@ def testXB64(s1, s2, expect, Bug3763=False): test('llXorBase64(' + repr(s1) + ',' + repr(s2) + ')', expect) Bugs.discard(3763) +def h2b64(h): + return unicode(b64encode(bytes(bytearray.fromhex(h)))) + +def testB642S(b, expected): + assert type(b) == str + assert type(expected) == str + test('llEscapeURL(llBase64ToString(h2b64(' + repr(b) + ')))', unicode(expected)) + # Begin JSON tests from http://wiki.secondlife.com/wiki/Json_usage_in_LSL/TestScript def verify(msg, result, expected): global tests @@ -1126,7 +1135,90 @@ def do_tests(): test('llBase64ToString(u"")', u'') test('llBase64ToString(u"1")', u'') test('llBase64ToString(u"12")', u'?') - test('llBase64ToString(u"14A")', u'\u05C0') # Defer thorough testing of UTF-8 to llUnescapeURL tests + test('llBase64ToString(u"14A")', u'\u05C0') + # llUnescapeURL and llBase64ToString behave differently. We need to test + # both thoroughly. + + # Embedded and trailing NUL tests: + test('llBase64ToString(u"QUJDAERFRg")', u'ABC?DEF') # 'ABC\x00DEF' + test('llBase64ToString(u"AEEAQgBD")', u'?A?B?C') # '\x00A\x00B\x00C' + test('llBase64ToString(u"AEEAQgBDAA")', u'?A?B?C') # '\x00A\x00B\x00C\x00' + test('llBase64ToString(u"AEEAQgBDAA")', u'?A?B?C') # '\x00A\x00B\x00C\x00' + testB642S("0041004200430000", "%3FA%3FB%3FC%3F") + + # Some assorted tests: + test('llBase64ToString(u"gIAA")', u'??') + test('llBase64ToString(u"gAA")', u'?') + test('llBase64ToString(u"44AA")', u'?') + test('llBase64ToString(u"4IAh")', u'?!') + test('llBase64ToString(u"gICAgGE")', u'????a') + test('llBase64ToString(u"QQA")', u'A') + test('llBase64ToString(u"AEE=")', u'?A') + test('llBase64ToString(u"wKE")', u'?') # C080 + test('llBase64ToString(u"9ICA")', u'?') # F48080 + test('llBase64ToString(u"94CAgICA")', u'??????') + test('llBase64ToString(u"4ICA")', u'?') # E08080 + test('llBase64ToString(u"4IA")', u'?') # E080 + test('llUnescapeURL(u"%E0%80")', u'??') # Compare with the above + + testB642S("C38180E381C380414243D3", "%C3%81%3F%3F%C3%80ABC%3F") + testB642S("C38180E381C38041424300D3", "%C3%81%3F%3F%C3%80ABC%3F%3F") + testB642S("E0808080808080E381C38041424300D3", "%3F%3F%3F%3F%3F%3F%C3%80ABC%3F%3F") + # test UTF-8 valid ranges + testB642S("7F78", "%7Fx") # range up to U+007F + testB642S("808078", "%3F%3Fx") # invalid range begin + testB642S("BFBF78", "%3F%3Fx") # invalid range end + testB642S("C08078", "%3Fx") # aliased range begin (U+0000) + testB642S("C1BF78", "%3Fx") # aliased range end (U+007F) + testB642S("C28078", "%C2%80x") # U+0080 (2-byte range start) + testB642S("DFBF78", "%DF%BFx") # U+07FF (2-byte range end) + testB642S("E0808078", "%3Fx") # aliased range begin (U+0000) + testB642S("E09FBF78", "%3Fx") # aliased range end (U+07FF) + testB642S("E0A08078", "%E0%A0%80x") # U+0800 (3-byte range start) + testB642S("ED9FBF78", "%ED%9F%BFx") # U+D7FF (right before first UTF-16 high surrogate) + testB642S("EE808078", "%EE%80%80x") # U+E000 (right after last UTF-16 low surrogate) + testB642S("EFBFBF78", "%EF%BF%BFx") # U+FFFF (3-byte range end) + testB642S("F080808078", "%3Fx") # aliased range begin (U+0000) + testB642S("F08FBFBF78", "%3Fx") # aliased range end (U+FFFF) + testB642S("F090808078", "%F0%90%80%80x") # U+10000 (4-byte range start) + testB642S("F48FBFBF78", "%F4%8F%BF%BFx") # U+10FFFF (valid 4-byte range end) + # excluded because they are used for UTF-16 surrogates, not valid characters + testB642S("EDA080", "%3F%3F%3F") # D800 - first high surrogate + testB642S("EDAFBF", "%3F%3F%3F") # DBFF - last high surrogate + testB642S("EDB080", "%3F%3F%3F") # DC00 - first low surrogate + testB642S("EDBFBF", "%3F%3F%3F") # DFFF - last low surrogate + # excluded because of truncation to U+10FFFF + testB642S("F490808078", "%3F%3F%3F%3Fx") # U+110000 (invalid 4-byte range start) + testB642S("F7BFBFBF78", "%3F%3F%3F%3Fx") # U+1FFFFF (invalid 4-byte range end) + testB642S("F88080808078", "%3Fx") # aliased range begin (U+0000) + testB642S("F887BFBFBF78", "%3Fx") # aliased range end (U+1FFFFF) + testB642S("F88880808078", "%3F%3F%3F%3F%3Fx") # U+200000 (5-byte range start) + testB642S("FBBFBFBFBF78", "%3F%3F%3F%3F%3Fx") # U+3FFFFFF (5-byte range end) + testB642S("FC808080808078", "%3Fx") # aliased range begin (U+0000) + testB642S("FC83BFBFBFBF78", "%3Fx") # aliased range end (U+3FFFFFF) + testB642S("FC848080808078", "%3F%3F%3F%3F%3F%3Fx") # U+4000000 (6-byte range start) + testB642S("FDBFBFBFBFBF78", "%3F%3F%3F%3F%3F%3Fx") # U+7FFFFFFF (6-byte range end) + # not actually valid either way (these are actually used to distinguish the input as UTF-16 BOM) + testB642S("FEB080808080808078", "%3F%3F%3F%3F%3F%3F%3F%3Fx") + testB642S("FFBFBFBFBFBFBFBF78", "%3F%3F%3F%3F%3F%3F%3F%3Fx") + # short or invalid sequences + testB642S("80", "%3F") + testB642S("BF", "%3F") + testB642S("C2", "%3F") + testB642S("E1", "%3F") + testB642S("E180", "%3F") + testB642S("F1", "%3F") + testB642S("F180", "%3F") + testB642S("F18080", "%3F") + testB642S("F8808080", "%3F") + testB642S("F8888080", "%3F") + testB642S("FC80808080", "%3F") + testB642S("FC84808080", "%3F") + + # Test that U+FFFD is preserved even with invalid characters + testB642S("EFBFBD90", "%EF%BF%BD%3F") + + test('typecast([1,F32(3.14),Key(u"blah"),Quaternion((1.,0.,0.,0.))], unicode)', u'13.140000blah<1.000000, 0.000000, 0.000000, 0.000000>') @@ -1199,6 +1291,15 @@ def do_tests(): # not actually valid either way (these are actually used to distinguish the input as UTF-16 BOM) test('llEscapeURL(llUnescapeURL(u"%FE%B0%80%80%80%80%80%80x"))', u'%3F%3F%3F%3F%3F%3F%3F%3Fx') test('llEscapeURL(llUnescapeURL(u"%FF%BF%BF%BF%BF%BF%BF%BFx"))', u'%3F%3F%3F%3F%3F%3F%3F%3Fx') + # short or invalid sequences + test('llEscapeURL(llUnescapeURL(u"%80"))', u'%3F') + test('llEscapeURL(llUnescapeURL(u"%BF"))', u'%3F') + test('llEscapeURL(llUnescapeURL(u"%C2"))', u'%3F') + test('llEscapeURL(llUnescapeURL(u"%E1"))', u'%3F') + test('llEscapeURL(llUnescapeURL(u"%E1%80"))', u'%3F%3F') + test('llEscapeURL(llUnescapeURL(u"%F1"))', u'%3F') + test('llEscapeURL(llUnescapeURL(u"%F1%80"))', u'%3F%3F') + test('llEscapeURL(llUnescapeURL(u"%F1%80%80"))', u'%3F%3F%3F') # Test that U+FFFD is preserved even with invalid characters test('llEscapeURL(llUnescapeURL(u"%EF%BF%BD%90"))', u'%EF%BF%BD%3F') @@ -1221,11 +1322,11 @@ def do_tests(): test('llParseStringKeepNulls(u"",[],[""])', [u""]) test('llParseStringKeepNulls(u"",[""],[])', [u""]) test('llParseStringKeepNulls(u"",[""],[""])', [u""]) - test('llParseString2List(u"",[],[])', []); - test('llParseString2List(u"",[],[u""])', []); - test('llParseString2List(u"",[u""],[])', []); - test('llParseString2List(u"",[u""],[u""])', []); - test('llParseStringKeepNulls(u"a",[u""],[])', [u"a"]); + test('llParseString2List(u"",[],[])', []) + test('llParseString2List(u"",[],[u""])', []) + test('llParseString2List(u"",[u""],[])', []) + test('llParseString2List(u"",[u""],[u""])', []) + test('llParseStringKeepNulls(u"a",[u""],[])', [u"a"]) test(r'llToUpper(u"\u03c4\u03ac\u03c7\u03b9\u03c3\u03c4\u03b7 \u03b1\u03bb' @@ -1424,7 +1525,7 @@ def do_tests(): test(r'llJsonGetValue(u" [[[]]]]]]][] ", [])', u'[[[]]]]]]][]') test(r'llJsonGetValue(u"{\"a\":[1],\"a\":[2],\"a\":[3]}", [u"a",0])', u'3') - test(r'llJsonGetValue(u"{\"a\":[2,3,[4]],\"a\":1}", [u"a", 2, 0])', JSON_INVALID); + test(r'llJsonGetValue(u"{\"a\":[2,3,[4]],\"a\":1}", [u"a", 2, 0])', JSON_INVALID) test(r'llJsonGetValue(u"{\"a\":1,}",["a"])', JSON_INVALID)