lib: descend into sub directories only when it is necessary

Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
diff --git a/lib/rsa/Makefile b/lib/rsa/Makefile
index 693c745..164ab39 100644
--- a/lib/rsa/Makefile
+++ b/lib/rsa/Makefile
@@ -7,6 +7,4 @@
 # SPDX-License-Identifier:	GPL-2.0+
 #
 
-ifdef CONFIG_FIT_SIGNATURE
-obj-$(CONFIG_RSA) += rsa-verify.o
-endif
+obj-$(CONFIG_FIT_SIGNATURE) += rsa-verify.o