diff --git a/conf.c b/conf.c
index 9c99531..17cf279 100644
--- a/conf.c
+++ b/conf.c
@@ -1646,6 +1646,16 @@ void conf(struct ctx *c, int argc, char **argv)
 			     logfile, logsize);
 	}
 
+	/* Once the log mask is not LOG_EARLY, we will no longer log to stderr
+	 * if there was a log file specified.
+	 */
+	if (c->debug)
+		__setlogmask(LOG_UPTO(LOG_DEBUG));
+	else if (c->quiet)
+		__setlogmask(LOG_UPTO(LOG_WARNING));
+	else
+		__setlogmask(LOG_UPTO(LOG_INFO));
+
 	nl_sock_init(c, false);
 	if (!v6_only)
 		c->ifi4 = conf_ip4(ifi4, &c->ip4, c->mac);
diff --git a/passt.c b/passt.c
index 5d7e7c4..a061f2b 100644
--- a/passt.c
+++ b/passt.c
@@ -320,16 +320,6 @@ int main(int argc, char **argv)
 	if (isolate_prefork(&c))
 		die("Failed to sandbox process, exiting");
 
-	/* Once the log mask is not LOG_EARLY, we will no longer log to stderr
-	 * if there was a log file specified.
-	 */
-	if (c.debug)
-		__setlogmask(LOG_UPTO(LOG_DEBUG));
-	else if (c.quiet)
-		__setlogmask(LOG_UPTO(LOG_WARNING));
-	else
-		__setlogmask(LOG_UPTO(LOG_INFO));
-
 	if (!c.foreground)
 		__daemon(pidfile_fd, devnull_fd);
 	else