Add SUDO_DEBUG_INSTANCE_ERROR return value for sudo_debug_register()

and check for it in places where we check the return value of
sudo_debug_register().

plugins/sudoers/sudoers_debug.c

@@ -113,7 +113,7 @@ oom:
  * debug subsystem, freeing the debug list when done.
  * Sets the active debug instance as a side effect.
  */
-void
+bool
 sudoers_debug_register(const char *program,
     struct sudo_conf_debug_file_list *debug_files)
 {
@@ -129,6 +129,8 @@ sudoers_debug_register(const char *program,
 	if (program != NULL) {
 	    sudoers_debug_instance = sudo_debug_register(program,
 		sudoers_subsystem_names, sudoers_subsystem_ids, debug_files);
+	    if (sudoers_debug_instance == SUDO_DEBUG_INSTANCE_ERROR)
+		return false;
 	}
 	TAILQ_FOREACH_SAFE(debug_file, debug_files, entries, debug_next) {
 	    TAILQ_REMOVE(debug_files, debug_file, entries);
@@ -137,6 +139,7 @@ sudoers_debug_register(const char *program,
 	    free(debug_file);
 	}
     }
+    return true;
 }
 
 /*