diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile
index cc5d98ace49..f162c1428c8 100644
--- a/arch/arm/lib/Makefile
+++ b/arch/arm/lib/Makefile
@@ -44,7 +44,7 @@ obj-y	+= stack.o
 ifdef CONFIG_CPU_V7M
 obj-y	+= interrupts_m.o
 else ifdef CONFIG_ARM64
-obj-y	+= ccn504.o
+obj-$(CONFIG_FSL_LAYERSCAPE) += ccn504.o
 ifneq ($(CONFIG_GICV2)$(CONFIG_GICV3),)
 obj-y	+= gic_64.o
 endif