math/ec-raw.[ch] (ec_ec2osp): Check that the requested flags are sane.

Also tests, just to make sure.

diff --git a/math/ec-raw.c b/math/ec-raw.c
index acee1b6..497e191 100644 (file)
--- a/math/ec-raw.c
+++ b/math/ec-raw.c
@@ -54,6 +54,9 @@
  *             @EC_EXPLY@ is set, then an explicit %$y$%-coordinate is
  *             output in full.  Otherwise the %$y$%-coordinate is
  *             suppressed.
+ *
+ *             Returns failure (@-1@) if the flags are invalid, or if there
+ *             isn't enough space in the output buffer.
  */
 
 int ec_ec2osp(ec_curve *c, unsigned f, buf *b, const ec *p)
@@ -62,15 +65,22 @@ int ec_ec2osp(ec_curve *c, unsigned f, buf *b, const ec *p)
   size_t n;
   ec t = EC_INIT;
 
+  /* --- Check the requested flags for sanity --- */
+
+  if (!f) f = EC_XONLY;
+  if (f & ~((f & EC_XONLY) ? EC_XONLY :
+           (f & EC_CMPR) ? (EC_CMPR | EC_EXPLY | EC_SORT) :
+           (f & EC_EXPLY) ? EC_EXPLY :
+           0u))
+    return (-1);
+
   /* --- Point at infinity --- */
 
   if (EC_ATINF(p)) return (buf_putbyte(b, 0));
 
   /* --- Fix up the format byte, compressing the %$y$%-coordinate --- */
 
-  if (!f)
-    f = EC_XONLY;
-  else if (f & EC_CMPR) {
+  if (f & EC_CMPR) {
     if (!(f & EC_SORT))
       f |= EC_COMPR(c, p) ? EC_YBIT : 0;
     else {

diff --git a/math/ec-raw.h b/math/ec-raw.h
index 0ceadd6..df8d1ab 100644 (file)
--- a/math/ec-raw.h
+++ b/math/ec-raw.h
@@ -74,6 +74,9 @@
  *             @EC_EXPLY@ is set, then an explicit %$y$%-coordinate is
  *             output in full.  Otherwise the %$y$%-coordinate is
  *             suppressed.
+ *
+ *             Returns failure (@-1@) if the flags are invalid, or if there
+ *             isn't enough space in the output buffer.
  */
 
 extern int ec_ec2osp(ec_curve */*c*/, unsigned /*f*/,

diff --git a/math/t/ec b/math/t/ec
index 9b57d6c..9af65c8 100644 (file)
--- a/math/t/ec
+++ b/math/t/ec
@@ -462,6 +462,32 @@ mul {
 }
 
 ec2osp {
+  ## Make sure some things which I expect to fail actually do.
+  "prime: 6277101735386680763835789423207666416102355444464034512659; prime: -3, 6"
+    3 "0x4a4edd749b4e809ddf0ecdb0cddd64b64f4558816ef243f5, 0xb3cef6f7f891a23d823aa01e96a74a3890cf71f4032d49d"
+    FAIL;
+  "prime: 6277101735386680763835789423207666416102355444464034512659; prime: -3, 6"
+    5 "0x4a4edd749b4e809ddf0ecdb0cddd64b64f4558816ef243f5, 0xb3cef6f7f891a23d823aa01e96a74a3890cf71f4032d49d"
+    FAIL;
+  "prime: 6277101735386680763835789423207666416102355444464034512659; prime: -3, 6"
+    7 "0x4a4edd749b4e809ddf0ecdb0cddd64b64f4558816ef243f5, 0xb3cef6f7f891a23d823aa01e96a74a3890cf71f4032d49d"
+    FAIL;
+  "prime: 6277101735386680763835789423207666416102355444464034512659; prime: -3, 6"
+    8 "0x4a4edd749b4e809ddf0ecdb0cddd64b64f4558816ef243f5, 0xb3cef6f7f891a23d823aa01e96a74a3890cf71f4032d49d"
+    FAIL;
+  "prime: 6277101735386680763835789423207666416102355444464034512659; prime: -3, 6"
+    9 "0x4a4edd749b4e809ddf0ecdb0cddd64b64f4558816ef243f5, 0xb3cef6f7f891a23d823aa01e96a74a3890cf71f4032d49d"
+    FAIL;
+  "prime: 6277101735386680763835789423207666416102355444464034512659; prime: -3, 6"
+    11 "0x4a4edd749b4e809ddf0ecdb0cddd64b64f4558816ef243f5, 0xb3cef6f7f891a23d823aa01e96a74a3890cf71f4032d49d"
+    FAIL;
+  "prime: 6277101735386680763835789423207666416102355444464034512659; prime: -3, 6"
+    13 "0x4a4edd749b4e809ddf0ecdb0cddd64b64f4558816ef243f5, 0xb3cef6f7f891a23d823aa01e96a74a3890cf71f4032d49d"
+    FAIL;
+  "prime: 6277101735386680763835789423207666416102355444464034512659; prime: -3, 6"
+    15 "0x4a4edd749b4e809ddf0ecdb0cddd64b64f4558816ef243f5, 0xb3cef6f7f891a23d823aa01e96a74a3890cf71f4032d49d"
+    FAIL;
+
   ## Some automated tests, from `ec-compr-test.sage'.
   "prime: 6277101735386680763835789423207666416102355444464034512659; prime: -3, 6"
     1 "0x4a4edd749b4e809ddf0ecdb0cddd64b64f4558816ef243f5, 0xb3cef6f7f891a23d823aa01e96a74a3890cf71f4032d49d"