Make hexchar() return -1 on invalid input instead of calling fatalx().

Callers used to check that the string was hex before calling hexchar().
Now callers must check for a -1 return value instead.

plugins/sudoers/hexchar.c

@@ -21,8 +21,11 @@
 
 #include "missing.h"
 #include "sudo_debug.h"
-#include "fatal.h"
 
+/*
+ * Converts a two-byte hex string to decimal.
+ * Returns the decimal value or -1 for invalid input.
+ */
 int
 hexchar(const char *s)
 {
@@ -87,8 +90,8 @@ hexchar(const char *s)
 	    result[i] = 15;
 	    break;
 	default:
-	    /* Should not happen. */
-	    fatalx("internal error, \\x%s not in proper hex format", s);
+	    /* Invalid input. */
+	    debug_return_int(-1);
 	}
     }
     debug_return_int((result[0] << 4) | result[1]);

plugins/sudoers/match.c

@@ -81,7 +81,6 @@
 #  include <ndir.h>
 # endif
 #endif
-#include <ctype.h>
 #include <pwd.h>
 #include <grp.h>
 #include <errno.h>
@@ -605,6 +604,7 @@ digest_matches(const char *file, const struct sudo_digest *sd)
     SHA2_CTX ctx;
     FILE *fp;
     unsigned int i;
+    int h;
     debug_decl(digest_matches, SUDO_DEBUG_MATCH)
 
     for (i = 0; digest_functions[i].digest_name != NULL; i++) {
@@ -620,11 +620,10 @@ digest_matches(const char *file, const struct sudo_digest *sd)
     if (strlen(sd->digest_str) == func->digest_len * 2) {
 	/* Convert the command digest from ascii hex to binary. */
 	for (i = 0; i < func->digest_len; i++) {
-	    if (!isxdigit((unsigned char)sd->digest_str[i + i]) ||
-		!isxdigit((unsigned char)sd->digest_str[i + i + 1])) {
+	    h = hexchar(&sd->digest_str[i + i]);
+	    if (h == -1)
 		goto bad_format;
-	    }
-	    sudoers_digest[i] = hexchar(&sd->digest_str[i + i]);
+	    sudoers_digest[i] = (unsigned char)h;
 	}
     } else {
 	size_t len = base64_decode(sd->digest_str, sudoers_digest,

plugins/sudoers/toke_util.c

@@ -46,7 +46,6 @@
 #if defined(HAVE_MALLOC_H) && !defined(STDC_HEADERS)
 # include <malloc.h>
 #endif /* HAVE_MALLOC_H && !STDC_HEADERS */
-#include <ctype.h>
 #include <errno.h>
 
 #include "sudoers.h"
@@ -61,6 +60,7 @@ bool
 fill_txt(const char *src, int len, int olen)
 {
     char *dst;
+    int h;
     debug_decl(fill_txt, SUDO_DEBUG_PARSER)
 
     dst = olen ? realloc(sudoerslval.string, olen + len + 1) : malloc(len + 1);
@@ -75,10 +75,8 @@ fill_txt(const char *src, int len, int olen)
     dst += olen;
     while (len--) {
 	if (*src == '\\' && len) {
-	    if (src[1] == 'x' && len >= 3 && 
-		isxdigit((unsigned char) src[2]) &&
-		isxdigit((unsigned char) src[3])) {
-		*dst++ = hexchar(src + 2);
+	    if (src[1] == 'x' && len >= 3 && (h = hexchar(src + 2)) != -1) {
+		*dst++ = h;
 		src += 4;
 		len -= 3;
 	    } else {