lib: wrap crypt API to hide errno usage

In order to prevent using the global errno, replace it with a static
version and create a wrapper function which returns the error value.

Signed-off-by: Steffen Jaeckel <jaeckel-floss@eyet-services.de>
Reviewed-by: Simon Glass <sjg@chromium.org>
Reviewed-by: Heiko Schocher <hs@denx.de>
diff --git a/lib/crypt/alg-sha256.h b/lib/crypt/alg-sha256.h
index e4b29c9..62e7b9d 100644
--- a/lib/crypt/alg-sha256.h
+++ b/lib/crypt/alg-sha256.h
@@ -1,12 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ */
 /* Copyright (C) 2020 Steffen Jaeckel <jaeckel-floss@eyet-services.de> */
 
-#ifndef USE_HOSTCC
-#include "common.h"
-#else
-#include <string.h>
-#endif
-
 #include "u-boot/sha256.h"
 
 #define INCLUDE_sha256crypt 1
diff --git a/lib/crypt/alg-sha512.h b/lib/crypt/alg-sha512.h
index 93b6109..47e4573 100644
--- a/lib/crypt/alg-sha512.h
+++ b/lib/crypt/alg-sha512.h
@@ -1,12 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ */
 /* Copyright (C) 2020 Steffen Jaeckel <jaeckel-floss@eyet-services.de> */
 
-#ifndef USE_HOSTCC
-#include "common.h"
-#else
-#include <string.h>
-#endif
-
 #include "u-boot/sha512.h"
 
 #define INCLUDE_sha512crypt 1
diff --git a/lib/crypt/crypt-port.h b/lib/crypt/crypt-port.h
index 680ffe9..6b9542d 100644
--- a/lib/crypt/crypt-port.h
+++ b/lib/crypt/crypt-port.h
@@ -18,11 +18,13 @@
 
 #define b64t ((const char *)ascii64)
 
-void crypt_sha256crypt_rn(const char *phrase, size_t phr_size,
-			  const char *setting, size_t ARG_UNUSED(set_size),
-			  uint8_t *output, size_t out_size, void *scratch,
-			  size_t scr_size);
-void crypt_sha512crypt_rn(const char *phrase, size_t phr_size,
-			  const char *setting, size_t ARG_UNUSED(set_size),
-			  uint8_t *output, size_t out_size, void *scratch,
-			  size_t scr_size);
+int crypt_sha256crypt_rn_wrapped(const char *phrase, size_t phr_size,
+				 const char *setting,
+				 size_t ARG_UNUSED(set_size), uint8_t *output,
+				 size_t out_size, void *scratch,
+				 size_t scr_size);
+int crypt_sha512crypt_rn_wrapped(const char *phrase, size_t phr_size,
+				 const char *setting,
+				 size_t ARG_UNUSED(set_size), uint8_t *output,
+				 size_t out_size, void *scratch,
+				 size_t scr_size);
diff --git a/lib/crypt/crypt-sha256.c b/lib/crypt/crypt-sha256.c
index 37127d4..335c888 100644
--- a/lib/crypt/crypt-sha256.c
+++ b/lib/crypt/crypt-sha256.c
@@ -1,10 +1,14 @@
+// SPDX-License-Identifier: CC0-1.0
+/* Based on libxcrypt v4.4.17-0-g6b110bc */
 /* One way encryption based on the SHA256-based Unix crypt implementation.
  *
  * Written by Ulrich Drepper <drepper at redhat.com> in 2007 [1].
  * Modified by Zack Weinberg <zackw at panix.com> in 2017, 2018.
  * Composed by Björn Esser <besser82 at fedoraproject.org> in 2018.
  * Modified by Björn Esser <besser82 at fedoraproject.org> in 2020.
- * Modified by Steffen Jaeckel <jaeckel-floss at eyet-services.de> in 2020.
+ * Modified by Steffen Jaeckel <jaeckel-floss at eyet-services.de> in 2021
+ * for U-Boot, instead of using the global errno to use a static one
+ * inside this file.
  * To the extent possible under law, the named authors have waived all
  * copyright and related or neighboring rights to this work.
  *
@@ -20,7 +24,7 @@
 #include "crypt-port.h"
 #include "alg-sha256.h"
 
-#include <errno.h>
+#include <linux/errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -69,6 +73,25 @@
                "ALG_SPECIFIC_SIZE is too small for SHA256");
 
 
+/* Use this instead of including errno.h */
+static int errno;
+
+void crypt_sha256crypt_rn(const char *phrase, size_t phr_size,
+			  const char *setting, size_t ARG_UNUSED(set_size),
+			  uint8_t *output, size_t out_size, void *scratch,
+			  size_t scr_size);
+
+int crypt_sha256crypt_rn_wrapped(const char *phrase, size_t phr_size,
+				 const char *setting, size_t set_size,
+				 u8 *output, size_t out_size, void *scratch,
+				 size_t scr_size)
+{
+	errno = 0;
+	crypt_sha256crypt_rn(phrase, phr_size, setting, set_size, output,
+			     out_size, scratch, scr_size);
+	return -errno;
+}
+
 /* Feed CTX with LEN bytes of a virtual byte sequence consisting of
    BLOCK repeated over and over indefinitely.  */
 static void
diff --git a/lib/crypt/crypt-sha512.c b/lib/crypt/crypt-sha512.c
index 3616019..8c8e6dd 100644
--- a/lib/crypt/crypt-sha512.c
+++ b/lib/crypt/crypt-sha512.c
@@ -1,10 +1,14 @@
+// SPDX-License-Identifier: CC0-1.0
+/* Based on libxcrypt v4.4.17-0-g6b110bc */
 /* One way encryption based on the SHA512-based Unix crypt implementation.
  *
  * Written by Ulrich Drepper <drepper at redhat.com> in 2007 [1].
  * Modified by Zack Weinberg <zackw at panix.com> in 2017, 2018.
  * Composed by Björn Esser <besser82 at fedoraproject.org> in 2018.
  * Modified by Björn Esser <besser82 at fedoraproject.org> in 2020.
- * Modified by Steffen Jaeckel <jaeckel-floss at eyet-services.de> in 2020.
+ * Modified by Steffen Jaeckel <jaeckel-floss at eyet-services.de> in 2021
+ * for U-Boot, instead of using the global errno to use a static one
+ * inside this file.
  * To the extent possible under law, the named authors have waived all
  * copyright and related or neighboring rights to this work.
  *
@@ -20,7 +24,7 @@
 #include "crypt-port.h"
 #include "alg-sha512.h"
 
-#include <errno.h>
+#include <linux/errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -69,6 +73,25 @@
                "ALG_SPECIFIC_SIZE is too small for SHA512");
 
 
+/* Use this instead of including errno.h */
+static int errno;
+
+void crypt_sha512crypt_rn(const char *phrase, size_t phr_size,
+			  const char *setting, size_t ARG_UNUSED(set_size),
+			  uint8_t *output, size_t out_size, void *scratch,
+			  size_t scr_size);
+
+int crypt_sha512crypt_rn_wrapped(const char *phrase, size_t phr_size,
+				 const char *setting, size_t set_size,
+				 u8 *output, size_t out_size, void *scratch,
+				 size_t scr_size)
+{
+	errno = 0;
+	crypt_sha512crypt_rn(phrase, phr_size, setting, set_size, output,
+			     out_size, scratch, scr_size);
+	return -errno;
+}
+
 /* Subroutine of _xcrypt_crypt_sha512crypt_rn: Feed CTX with LEN bytes of a
    virtual byte sequence consisting of BLOCK repeated over and over
    indefinitely.  */
diff --git a/lib/crypt/crypt.c b/lib/crypt/crypt.c
index 4ec6079..247c34b 100644
--- a/lib/crypt/crypt.c
+++ b/lib/crypt/crypt.c
@@ -5,8 +5,8 @@
 #include <crypt.h>
 #include "crypt-port.h"
 
-typedef void (*crypt_fn)(const char *, size_t, const char *, size_t, uint8_t *,
-			 size_t, void *, size_t);
+typedef int (*crypt_fn)(const char *, size_t, const char *, size_t, uint8_t *,
+			size_t, void *, size_t);
 
 const unsigned char ascii64[65] =
 	"./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
@@ -29,19 +29,20 @@
 	*equal = ret ^ 1;
 }
 
-void crypt_compare(const char *should, const char *passphrase, int *equal)
+int crypt_compare(const char *should, const char *passphrase, int *equal)
 {
 	u8 output[CRYPT_OUTPUT_SIZE], scratch[ALG_SPECIFIC_SIZE];
 	size_t n;
+	int err;
 	struct {
 		const char *prefix;
 		crypt_fn crypt;
 	} crypt_algos[] = {
 #if defined(CONFIG_CRYPT_PW_SHA256)
-		{ "$5$", crypt_sha256crypt_rn },
+		{ "$5$", crypt_sha256crypt_rn_wrapped },
 #endif
 #if defined(CONFIG_CRYPT_PW_SHA512)
-		{ "$6$", crypt_sha512crypt_rn },
+		{ "$6$", crypt_sha512crypt_rn_wrapped },
 #endif
 		{ NULL, NULL }
 	};
@@ -56,18 +57,20 @@
 	}
 
 	if (n >= ARRAY_SIZE(crypt_algos))
-		return;
+		return -EINVAL;
 
-	crypt_algos[n].crypt(passphrase, strlen(passphrase), should, 0, output,
-			     sizeof(output), scratch, sizeof(scratch));
-
+	err = crypt_algos[n].crypt(passphrase, strlen(passphrase), should, 0,
+				   output, sizeof(output), scratch,
+				   sizeof(scratch));
 	/* early return on error, nothing really happened inside the crypt() function */
-	if (errno == ERANGE || errno == EINVAL)
-		return;
+	if (err)
+		return err;
 
 	equals_constant_time(should, output, strlen((const char *)output),
 			     equal);
 
 	memset(scratch, 0, sizeof(scratch));
 	memset(output, 0, sizeof(output));
+
+	return 0;
 }