binman: Support obtaining section contents immediately

Generally the content of sections is not built until the final assembly
of the image. This is partly to avoid wasting time, since the entries
within sections may change multiple times as binman works through its
various stages. This works quite well since sections exist in a strict
hierarchy, so they can be processed in a depth-first manner.

However the 'collection' entry type does not have this luxury. If it
contains a section within its 'content' list, then it must produce the
section contents, if available. That section is typically a sibling
node, i.e. not part oc the collection's hierarchy.

Add a new 'required' argument to section.GetData() to support this. When
required is True, any referenced sections are immediately built. If this
is not possible (because one of the subentries does not have its data yet)
then an error is produced.

The test for this uses a 'collection' entry type, referencing a section as
its first member. This forces a call to _BuildSectionData() with required
set to False, at first, then True later, when the image is assembled.

Signed-off-by: Simon Glass <sjg@chromium.org>

tools/binman/entry.py

@@ -482,9 +482,13 @@ class Entry(object):
         """
         return self._node.path
 
-    def GetData(self):
+    def GetData(self, required=True):
         """Get the contents of an entry
 
+        Args:
+            required: True if the data must be present, False if it is OK to
+                return None
+
         Returns:
             bytes content of the entry, excluding any padding. If the entry is
                 compressed, the compressed data is returned

tools/binman/etype/collection.py

@@ -28,18 +28,24 @@ class Entry_collection(Entry):
         if not self.content:
             self.Raise("Collection must have a 'content' property")
 
-    def GetContents(self):
+    def GetContents(self, required):
         """Get the contents of this entry
 
+        Args:
+            required: True if the data must be present, False if it is OK to
+                return None
+
         Returns:
             bytes content of the entry
         """
         # Join up all the data
-        self.Info('Getting content')
+        self.Info('Getting contents, required=%s' % required)
         data = b''
         for entry_phandle in self.content:
-            entry_data = self.section.GetContentsByPhandle(entry_phandle, self)
-            if entry_data is None:
+            entry_data = self.section.GetContentsByPhandle(entry_phandle, self,
+                                                           required)
+            if not required and entry_data is None:
+                self.Info('Contents not available yet')
                 # Data not available yet
                 return None
             data += entry_data
@@ -49,7 +55,7 @@ class Entry_collection(Entry):
         return data
 
     def ObtainContents(self):
-        data = self.GetContents()
+        data = self.GetContents(False)
         if data is None:
             return False
         self.SetContents(data)
@@ -57,5 +63,5 @@ class Entry_collection(Entry):
 
     def ProcessContents(self):
         # The blob may have changed due to WriteSymbols()
-        data = self.GetContents()
+        data = self.GetContents(True)
         return self.ProcessContentsUpdate(data)

tools/binman/etype/section.py

@@ -180,7 +180,7 @@ class Entry_section(Entry):
 
         return data
 
-    def _BuildSectionData(self):
+    def _BuildSectionData(self, required):
         """Build the contents of a section
 
         This places all entries at the right place, dealing with padding before
@@ -188,13 +188,20 @@ class Entry_section(Entry):
         pad-before and pad-after properties in the section items) since that is
         handled by the parent section.
 
+        Args:
+            required: True if the data must be present, False if it is OK to
+                return None
+
         Returns:
             Contents of the section (bytes)
         """
         section_data = b''
 
         for entry in self._entries.values():
-            data = self.GetPaddedDataForEntry(entry, entry.GetData())
+            entry_data = entry.GetData(required)
+            if not required and entry_data is None:
+                return None
+            data = self.GetPaddedDataForEntry(entry, entry_data)
             # Handle empty space before the entry
             pad = (entry.offset or 0) - self._skip_at_start - len(section_data)
             if pad > 0:
@@ -226,18 +233,24 @@ class Entry_section(Entry):
             data = self.GetData()
         return section.GetPaddedDataForEntry(self, data)
 
-    def GetData(self):
+    def GetData(self, required=True):
         """Get the contents of an entry
 
         This builds the contents of the section, stores this as the contents of
         the section and returns it
 
+        Args:
+            required: True if the data must be present, False if it is OK to
+                return None
+
         Returns:
             bytes content of the section, made up for all all of its subentries.
             This excludes any padding. If the section is compressed, the
             compressed data is returned
         """
-        data = self._BuildSectionData()
+        data = self._BuildSectionData(required)
+        if data is None:
+            return None
         self.SetContents(data)
         return data
 
@@ -263,7 +276,7 @@ class Entry_section(Entry):
             self._SortEntries()
         self._ExpandEntries()
 
-        data = self._BuildSectionData()
+        data = self._BuildSectionData(True)
         self.SetContents(data)
 
         self.CheckSize()
@@ -360,16 +373,20 @@ class Entry_section(Entry):
     def GetEntries(self):
         return self._entries
 
-    def GetContentsByPhandle(self, phandle, source_entry):
+    def GetContentsByPhandle(self, phandle, source_entry, required):
         """Get the data contents of an entry specified by a phandle
 
         This uses a phandle to look up a node and and find the entry
-        associated with it. Then it returnst he contents of that entry.
+        associated with it. Then it returns the contents of that entry.
+
+        The node must be a direct subnode of this section.
 
         Args:
             phandle: Phandle to look up (integer)
             source_entry: Entry containing that phandle (used for error
                 reporting)
+            required: True if the data must be present, False if it is OK to
+                return None
 
         Returns:
             data from associated entry (as a string), or None if not found
@@ -379,7 +396,7 @@ class Entry_section(Entry):
             source_entry.Raise("Cannot find node for phandle %d" % phandle)
         for entry in self._entries.values():
             if entry._node == node:
-                return entry.GetData()
+                return entry.GetData(required)
         source_entry.Raise("Cannot find entry for node '%s'" % node.name)
 
     def LookupSymbol(self, sym_name, optional, msg, base_addr, entries=None):

tools/binman/etype/vblock.py

@@ -47,15 +47,19 @@ class Entry_vblock(Entry_collection):
             EntryArg('kernelkey', str),
             EntryArg('preamble-flags', int)])
 
-    def GetVblock(self):
+    def GetVblock(self, required):
         """Get the contents of this entry
 
+        Args:
+            required: True if the data must be present, False if it is OK to
+                return None
+
         Returns:
             bytes content of the entry, which is the signed vblock for the
                 provided data
         """
         # Join up the data files to be signed
-        input_data = self.GetContents()
+        input_data = self.GetContents(required)
         if input_data is None:
             return None
 
@@ -79,7 +83,7 @@ class Entry_vblock(Entry_collection):
         return tools.ReadFile(output_fname)
 
     def ObtainContents(self):
-        data = self.GetVblock()
+        data = self.GetVblock(False)
         if data is None:
             return False
         self.SetContents(data)
@@ -87,5 +91,5 @@ class Entry_vblock(Entry_collection):
 
     def ProcessContents(self):
         # The blob may have changed due to WriteSymbols()
-        data = self.GetVblock()
+        data = self.GetVblock(True)
         return self.ProcessContentsUpdate(data)

tools/binman/ftest.py

@@ -4484,5 +4484,18 @@ class TestFunctional(unittest.TestCase):
                          tools.GetBytes(0xfe, 3) + U_BOOT_DTB_DATA,
                          data)
 
+    def testCollectionSection(self):
+        """Test a collection where a section must be built first"""
+        # Sections never have their contents when GetData() is called, but when
+        # _BuildSectionData() is called with required=True, a section will force
+        # building the contents, producing an error is anything is still
+        # missing.
+        data = self._DoReadFile('199_collection_section.dts')
+        section = U_BOOT_NODTB_DATA + U_BOOT_DTB_DATA
+        self.assertEqual(section + U_BOOT_DATA + tools.GetBytes(0xff, 2) +
+                         section + tools.GetBytes(0xfe, 3) + U_BOOT_DATA,
+                         data)
+
+
 if __name__ == "__main__":
     unittest.main()

tools/binman/test/199_collection_section.dts

@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	binman {
+		collection {
+			content = <&section &u_boot>;
+		};
+		fill {
+			size = <2>;
+			fill-byte = [ff];
+		};
+		section: section {
+			u-boot-nodtb {
+			};
+			u-boot-dtb {
+			};
+		};
+		fill2 {
+			type = "fill";
+			size = <3>;
+			fill-byte = [fe];
+		};
+		u_boot: u-boot {
+			no-expanded;
+		};
+	};
+};