diff --git a/drivers/tty/serdev/core.c b/drivers/tty/serdev/core.c
index ec113e39993f..dde2ddc5967d 100644
--- a/drivers/tty/serdev/core.c
+++ b/drivers/tty/serdev/core.c
@@ -75,21 +75,32 @@ static int serdev_uevent(struct device *dev, struct kobj_uevent_env *env)
  */
 int serdev_device_add(struct serdev_device *serdev)
 {
+	struct serdev_controller *ctrl = serdev->ctrl;
 	struct device *parent = serdev->dev.parent;
 	int err;
 
 	dev_set_name(&serdev->dev, "%s-%d", dev_name(parent), serdev->nr);
 
+	/* Only a single slave device is currently supported. */
+	if (ctrl->serdev) {
+		dev_err(&serdev->dev, "controller busy\n");
+		return -EBUSY;
+	}
+	ctrl->serdev = serdev;
+
 	err = device_add(&serdev->dev);
 	if (err < 0) {
 		dev_err(&serdev->dev, "Can't add %s, status %d\n",
 			dev_name(&serdev->dev), err);
-		goto err_device_add;
+		goto err_clear_serdev;
 	}
 
 	dev_dbg(&serdev->dev, "device %s registered\n", dev_name(&serdev->dev));
 
-err_device_add:
+	return 0;
+
+err_clear_serdev:
+	ctrl->serdev = NULL;
 	return err;
 }
 EXPORT_SYMBOL_GPL(serdev_device_add);
@@ -100,7 +111,10 @@ EXPORT_SYMBOL_GPL(serdev_device_add);
  */
 void serdev_device_remove(struct serdev_device *serdev)
 {
+	struct serdev_controller *ctrl = serdev->ctrl;
+
 	device_unregister(&serdev->dev);
+	ctrl->serdev = NULL;
 }
 EXPORT_SYMBOL_GPL(serdev_device_remove);
 
@@ -311,7 +325,6 @@ struct serdev_device *serdev_device_alloc(struct serdev_controller *ctrl)
 		return NULL;
 
 	serdev->ctrl = ctrl;
-	ctrl->serdev = serdev;
 	device_initialize(&serdev->dev);
 	serdev->dev.parent = &ctrl->dev;
 	serdev->dev.bus = &serdev_bus_type;