summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDean Rasheed <dean.a.rasheed@gmail.com>2021-10-06 13:20:23 +0100
committerDean Rasheed <dean.a.rasheed@gmail.com>2021-10-06 13:20:23 +0100
commit9ab94ccb15af288f9ff295b4f04b28d86115b803 (patch)
tree4f3afc1877a2b0e52cd0ade6cf142ee4633a3d95
parentd6d68e223379985661df45c728c32c3b6d462629 (diff)
Fix corner-case loss of precision in numeric_power().
This fixes a loss of precision that occurs when the first input is very close to 1, so that its logarithm is very small. Formerly, during the initial low-precision calculation to estimate the result weight, the logarithm was computed to a local rscale that was capped to NUMERIC_MAX_DISPLAY_SCALE (1000). However, the base may be as close as 1e-16383 to 1, hence its logarithm may be as small as 1e-16383, and so the local rscale needs to be allowed to exceed 16383, otherwise all precision is lost, leading to a poor choice of rscale for the full-precision calculation. Fix this by removing the cap on the local rscale during the initial low-precision calculation, as we already do in the full-precision calculation. This doesn't change the fact that the initial calculation is a low-precision approximation, computing the logarithm to around 8 significant digits, which is very fast, especially when the base is very close to 1. Patch by me, reviewed by Alvaro Herrera. Discussion: https://postgr.es/m/CAEZATCV-Ceu%2BHpRMf416yUe4KKFv%3DtdgXQAe5-7S9tD%3D5E-T1g%40mail.gmail.com
-rw-r--r--src/backend/utils/adt/numeric.c6
-rw-r--r--src/test/regress/expected/numeric.out6
-rw-r--r--src/test/regress/sql/numeric.sql1
3 files changed, 12 insertions, 1 deletions
diff --git a/src/backend/utils/adt/numeric.c b/src/backend/utils/adt/numeric.c
index 9f16f09c110..885d50f5fc3 100644
--- a/src/backend/utils/adt/numeric.c
+++ b/src/backend/utils/adt/numeric.c
@@ -9245,9 +9245,13 @@ power_var(const NumericVar *base, const NumericVar *exp, NumericVar *result)
*/
ln_dweight = estimate_ln_dweight(base);
+ /*
+ * Set the scale for the low-precision calculation, computing ln(base) to
+ * around 8 significant digits. Note that ln_dweight may be as small as
+ * -SHRT_MAX, so the scale may exceed NUMERIC_MAX_DISPLAY_SCALE here.
+ */
local_rscale = 8 - ln_dweight;
local_rscale = Max(local_rscale, NUMERIC_MIN_DISPLAY_SCALE);
- local_rscale = Min(local_rscale, NUMERIC_MAX_DISPLAY_SCALE);
ln_var(base, &ln_base, local_rscale);
diff --git a/src/test/regress/expected/numeric.out b/src/test/regress/expected/numeric.out
index 5c54d7bc48d..4ca9fd64088 100644
--- a/src/test/regress/expected/numeric.out
+++ b/src/test/regress/expected/numeric.out
@@ -1774,6 +1774,12 @@ select coalesce(nullif(0.9999999999 ^ 23300000000000, 0), 0) as rounds_to_zero;
0
(1 row)
+select round(((1 - 1.500012345678e-1000) ^ 1.45e1003) * 1e1000);
+ round
+----------------------------------------------------------
+ 25218976308958387188077465658068501556514992509509282366
+(1 row)
+
-- cases that used to error out
select 0.12 ^ (-25);
?column?
diff --git a/src/test/regress/sql/numeric.sql b/src/test/regress/sql/numeric.sql
index c0388691e56..34f96d1aab3 100644
--- a/src/test/regress/sql/numeric.sql
+++ b/src/test/regress/sql/numeric.sql
@@ -935,6 +935,7 @@ select 1.2 ^ 345;
select 0.12 ^ (-20);
select 1.000000000123 ^ (-2147483648);
select coalesce(nullif(0.9999999999 ^ 23300000000000, 0), 0) as rounds_to_zero;
+select round(((1 - 1.500012345678e-1000) ^ 1.45e1003) * 1e1000);
-- cases that used to error out
select 0.12 ^ (-25);