From 091f06cccc6871bdfcfa88875f6e683e0a524d62 Mon Sep 17 00:00:00 2001 From: Sei Lisa Date: Fri, 27 Apr 2018 23:44:27 +0200 Subject: [PATCH] Fix threshold value for denormal to be taken as 0 (float)"1.1754944e-38" is != 0 (float)"1.1754943e-38" is == 0 Yet, 1.1754944e-38 == 1.1754943e-38. The fix is to perform the operations as doubles, and convert to F32 *after* comparing the denormal range. --- lslopt/lslbasefuncs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lslopt/lslbasefuncs.py b/lslopt/lslbasefuncs.py index c3f6074..adf848b 100644 --- a/lslopt/lslbasefuncs.py +++ b/lslopt/lslbasefuncs.py @@ -456,18 +456,18 @@ def InternalTypecast(val, out, InList, f32): if match is None: return 0.0 if match.group(1): - ret = F32(float.fromhex(match.group(0)), f32) + ret = float.fromhex(match.group(0)) elif match.group(2): # (float)"-nan" produces NaN instead of Indet, even though # (vector)"<-nan,0,0>" produces . Go figure. ret = NaN else: - ret = F32(float(match.group(0)), f32) + ret = float(match.group(0)) if not lslcommon.LSO and abs(ret) < 1.1754943508222875e-38: # Mono doesn't return denormals when using (float)"val" # (but it returns them when using (vector)"") ret = 0.0 - return ret + return F32(ret, f32) if out == int: match = int_re.search(val) if match is None: