diff --git a/sound/soc/codecs/es8328.c b/sound/soc/codecs/es8328.c
index 3f84fbd071e2..51dca8662942 100644
--- a/sound/soc/codecs/es8328.c
+++ b/sound/soc/codecs/es8328.c
@@ -91,6 +91,7 @@ struct es8328_priv {
 	int mclkdiv2;
 	const struct snd_pcm_hw_constraint_list *sysclk_constraints;
 	const int *mclk_ratios;
+	bool master;
 	struct regulator_bulk_data supplies[ES8328_SUPPLY_NUM];
 };
 
@@ -469,7 +470,7 @@ static int es8328_startup(struct snd_pcm_substream *substream,
 	struct snd_soc_codec *codec = dai->codec;
 	struct es8328_priv *es8328 = snd_soc_codec_get_drvdata(codec);
 
-	if (es8328->sysclk_constraints)
+	if (es8328->master && es8328->sysclk_constraints)
 		snd_pcm_hw_constraint_list(substream->runtime, 0,
 				SNDRV_PCM_HW_PARAM_RATE,
 				es8328->sysclk_constraints);
@@ -488,27 +489,34 @@ static int es8328_hw_params(struct snd_pcm_substream *substream,
 	int wl;
 	int ratio;
 
-	if (!es8328->sysclk_constraints) {
-		dev_err(codec->dev, "No MCLK configured\n");
-		return -EINVAL;
-	}
-
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
 		reg = ES8328_DACCONTROL2;
 	else
 		reg = ES8328_ADCCONTROL5;
 
-	for (i = 0; i < es8328->sysclk_constraints->count; i++)
-		if (es8328->sysclk_constraints->list[i] == params_rate(params))
-			break;
+	if (es8328->master) {
+		if (!es8328->sysclk_constraints) {
+			dev_err(codec->dev, "No MCLK configured\n");
+			return -EINVAL;
+		}
 
-	if (i == es8328->sysclk_constraints->count) {
-		dev_err(codec->dev, "LRCLK %d unsupported with current clock\n",
-			params_rate(params));
-		return -EINVAL;
+		for (i = 0; i < es8328->sysclk_constraints->count; i++)
+			if (es8328->sysclk_constraints->list[i] ==
+			    params_rate(params))
+				break;
+
+		if (i == es8328->sysclk_constraints->count) {
+			dev_err(codec->dev,
+				"LRCLK %d unsupported with current clock\n",
+				params_rate(params));
+			return -EINVAL;
+		}
+		ratio = es8328->mclk_ratios[i];
+	} else {
+		ratio = 0;
+		es8328->mclkdiv2 = 0;
 	}
 
-	ratio = es8328->mclk_ratios[i];
 	snd_soc_update_bits(codec, ES8328_MASTERMODE,
 			ES8328_MASTERMODE_MCLKDIV2,
 			es8328->mclkdiv2 ? ES8328_MASTERMODE_MCLKDIV2 : 0);
@@ -586,6 +594,7 @@ static int es8328_set_dai_fmt(struct snd_soc_dai *codec_dai,
 		unsigned int fmt)
 {
 	struct snd_soc_codec *codec = codec_dai->codec;
+	struct es8328_priv *es8328 = snd_soc_codec_get_drvdata(codec);
 	u8 dac_mode = 0;
 	u8 adc_mode = 0;
 
@@ -595,11 +604,13 @@ static int es8328_set_dai_fmt(struct snd_soc_dai *codec_dai,
 		snd_soc_update_bits(codec, ES8328_MASTERMODE,
 				    ES8328_MASTERMODE_MSC,
 				    ES8328_MASTERMODE_MSC);
+		es8328->master = true;
 		break;
 	case SND_SOC_DAIFMT_CBS_CFS:
 		/* Slave serial port mode */
 		snd_soc_update_bits(codec, ES8328_MASTERMODE,
 				    ES8328_MASTERMODE_MSC, 0);
+		es8328->master = false;
 		break;
 	default:
 		return -EINVAL;