diff --git a/include/ns16550.h b/include/ns16550.h
index 243226fc3d..e7e68663d0 100644
--- a/include/ns16550.h
+++ b/include/ns16550.h
@@ -26,15 +26,13 @@
 
 #include <linux/types.h>
 
-#if CONFIG_IS_ENABLED(DM_SERIAL) && !defined(CONFIG_SYS_NS16550_REG_SIZE)
+#if CONFIG_IS_ENABLED(DM_SERIAL) ||  defined(CONFIG_NS16550_DYNAMIC) || \
+	defined(CONFIG_DEBUG_UART)
 /*
  * For driver model we always use one byte per register, and sort out the
- * differences in the driver
+ * differences in the driver. In the case of CONFIG_NS16550_DYNAMIC we do
+ * similar, and CONFIG_DEBUG_UART is responsible for shifts in its own manner.
  */
-#define CONFIG_SYS_NS16550_REG_SIZE (-1)
-#endif
-
-#if defined(CONFIG_NS16550_DYNAMIC) || defined(CONFIG_DEBUG_UART)
 #define UART_REG(x)	unsigned char x
 #else
 #if !defined(CONFIG_SYS_NS16550_REG_SIZE) || (CONFIG_SYS_NS16550_REG_SIZE == 0)