diff --git a/log.c b/log.c
index 468c730..6dc6673 100644
--- a/log.c
+++ b/log.c
@@ -66,8 +66,8 @@ void name(const char *format, ...) {					\
 		va_end(args);						\
 	}								\
 									\
-	if ((setlogmask(0) & LOG_MASK(LOG_DEBUG) ||			\
-	     setlogmask(0) == LOG_MASK(LOG_EMERG)) && log_file == -1) {	\
+	if ((setlogmask(0) & LOG_MASK(LOG_DEBUG) && log_file == -1) ||	\
+	     setlogmask(0) == LOG_MASK(LOG_EMERG)) {			\
 		va_start(args, format);					\
 		(void)vfprintf(stderr, format, args); 			\
 		va_end(args);						\
diff --git a/passt.c b/passt.c
index d957e14..c48c2d5 100644
--- a/passt.c
+++ b/passt.c
@@ -246,13 +246,6 @@ int main(int argc, char **argv)
 	if (c.stderr || isatty(fileno(stdout)))
 		__openlog(log_name, LOG_PERROR, LOG_DAEMON);
 
-	if (c.debug)
-		__setlogmask(LOG_UPTO(LOG_DEBUG));
-	else if (c.quiet)
-		__setlogmask(LOG_UPTO(LOG_ERR));
-	else
-		__setlogmask(LOG_UPTO(LOG_INFO));
-
 	quit_fd = pasta_netns_quit_init(&c);
 
 	tap_sock_init(&c);
@@ -296,6 +289,16 @@ int main(int argc, char **argv)
 		exit(EXIT_FAILURE);
 	}
 
+	/* Once the log mask is not LOG_EMERG, we will no longer
+	 * log to stderr if there was a log file specified.
+	 */
+	if (c.debug)
+		__setlogmask(LOG_UPTO(LOG_DEBUG));
+	else if (c.quiet)
+		__setlogmask(LOG_UPTO(LOG_ERR));
+	else
+		__setlogmask(LOG_UPTO(LOG_INFO));
+
 	if (!c.foreground)
 		__daemon(pidfile_fd, devnull_fd);
 	else