From: Mark Wooding <mdw@distorted.org.uk>
Date: Thu, 14 Nov 2019 20:17:58 +0000 (+0000)
Subject: math/limlee.c: Don't leak the factor vector on overall failure.
X-Git-Tag: 2.4.5~15
X-Git-Url: https://git.distorted.org.uk/~mdw/catacomb/commitdiff_plain/3d86dc1eec364744b12daf0230d072a4317d78bc

math/limlee.c: Don't leak the factor vector on overall failure.

The `done' function doesn't know whether we succeeded or failed, so it
prepares the factor vector for output regardless.  In `limlee', if we
don't have a result, then release the factors.
---

diff --git a/math/limlee.c b/math/limlee.c
index 28975d0c..c99ba7bd 100644
--- a/math/limlee.c
+++ b/math/limlee.c
@@ -390,6 +390,8 @@ mp *limlee(const char *name, mp *d, mp *newp,
 {
   limlee_stepctx l;
   rabin rr;
+  mp **v;
+  size_t i;
 
   l.f = 0; if (f) l.f |= LIMLEE_KEEPFACTORS;
   l.newp = newp;
@@ -402,15 +404,18 @@ mp *limlee(const char *name, mp *d, mp *newp,
   d = pgen(name, d, 0, oev, oec, on, limlee_step, &l,
 	   rabin_iters(pl), pgen_test, &rr);
 
-  if (d && f) {
-    mp **v;
-    size_t i;
-    v = xmalloc(l.nf * sizeof(mp *));
-    for (i = 0; i < l.nf; i++)
-      v[i] = l.v[i].p;
-    xfree(l.v);
-    *f = v;
-    *nf = l.nf;
+  if (f) {
+    if (!d) {
+      for (i = 0; i < l.nf; i++)
+	if (l.v[i].p) llfree(&l.v[i], &l);
+    } else {
+      v = xmalloc(l.nf * sizeof(mp *));
+      for (i = 0; i < l.nf; i++)
+	v[i] = l.v[i].p;
+      xfree(l.v);
+      *f = v;
+      *nf = l.nf;
+    }
   }
 
   return (d);