promote to float (for internal representation) before scaling by format multiplier (#175)

This commit is contained in:
georgehines
2016-07-10 11:13:18 +02:00
committed by Lorenz Meier
parent a2d866520a
commit 4f8ffb3420
+2 -2
View File
@@ -39,7 +39,7 @@ FORMAT_TO_STRUCT = {
'e', {'int32', 0.01};
'E', {'uint32', 0.01};
'L', {'int32', 0.0000001};
'M', {'int8', 1};
'M', {'uint8', 1};
'q', {'int64', 1};
'Q', {'uint64', 1}};
@@ -185,7 +185,7 @@ else
elseif strcmp('single',format_type)
dataLen = 4;
end
data = typecast(uint8(byte_array(thisPtr:(thisPtr+dataLen-1))),format_type)*mult;
data = double(typecast(uint8(byte_array(thisPtr:(thisPtr+dataLen-1))),format_type))*mult;
end
end