promote to float (for internal representation) before scaling by format multiplier (#175)

This commit is contained in:
georgehines 2016-07-10 02:13:18 -07:00 committed by Lorenz Meier
parent a2d866520a
commit 4f8ffb3420

View File

@ -39,7 +39,7 @@ FORMAT_TO_STRUCT = {
'e', {'int32', 0.01};
'E', {'uint32', 0.01};
'L', {'int32', 0.0000001};
'M', {'int8', 1};
'M', {'uint8', 1};
'q', {'int64', 1};
'Q', {'uint64', 1}};
@ -185,7 +185,7 @@ else
elseif strcmp('single',format_type)
dataLen = 4;
end
data = typecast(uint8(byte_array(thisPtr:(thisPtr+dataLen-1))),format_type)*mult;
data = double(typecast(uint8(byte_array(thisPtr:(thisPtr+dataLen-1))),format_type))*mult;
end
end