For dynamic partitions devices. Like poco X2(phoenix) and Redmi Note 9s(Curtana)

tools/releasetools/common.py

@@ -2372,10 +2372,11 @@ def __init__(self, src_size=None, tgt_size=None):
 
 class DynamicPartitionsDifference(object):
   def __init__(self, info_dict, block_diffs, progress_dict=None,
-               source_info_dict=None):
+               source_info_dict=None, build_without_vendor=False):
     if progress_dict is None:
       progress_dict = dict()
 
+    self._build_without_vendor = build_without_vendor
     self._remove_all_before_apply = False
     if source_info_dict is None:
       self._remove_all_before_apply = True
@@ -2498,6 +2499,17 @@ def append(line):
     def comment(line):
       self._op_list.append("# %s" % line)
 
+    if self._build_without_vendor:
+      comment('System-only build, keep original vendor partition')
+      # When building without vendor, we do not want to override
+      # any partition already existing. In this case, we can only
+      # resize, but not remove / create / re-create any other
+      # partition.
+      for p, u in self._partition_updates.items():
+        comment('Resize partition %s to %s' % (p, u.tgt_size))
+        append('resize %s %s' % (p, u.tgt_size))
+      return
+
     if self._remove_all_before_apply:
       comment('Remove all existing dynamic partitions and groups before '
               'applying full OTA')

tools/releasetools/ota_from_target_files.py

@@ -192,6 +192,7 @@
 
 from __future__ import print_function
 
+import collections
 import logging
 import multiprocessing
 import os.path
@@ -869,6 +870,100 @@ def FingerprintChanged(source_fp, target_fp):
   AddCompatibilityArchive(system_updated or product_updated,
                           vendor_updated or odm_updated)
 
+def CopyInstallTools(output_zip):
+  install_path = os.path.join(OPTIONS.input_tmp, "INSTALL")
+  for root, subdirs, files in os.walk(install_path):
+     for f in files:
+      install_source = os.path.join(root, f)
+      install_target = os.path.join("install", os.path.relpath(root, install_path), f)
+      output_zip.write(install_source, install_target)
+
+def GetBlockDifferences(target_zip, source_zip, target_info, source_info,
+                        device_specific):
+  """Returns a ordered dict of block differences with partition name as key."""
+
+  def GetIncrementalBlockDifferenceForPartition(name):
+    if not HasPartition(source_zip, name):
+      raise RuntimeError("can't generate incremental that adds {}".format(name))
+
+    partition_src = common.GetUserImage(name, OPTIONS.source_tmp, source_zip,
+                                        info_dict=source_info,
+                                        allow_shared_blocks=allow_shared_blocks)
+
+    hashtree_info_generator = verity_utils.CreateHashtreeInfoGenerator(
+        name, 4096, target_info)
+    partition_tgt = common.GetUserImage(name, OPTIONS.target_tmp, target_zip,
+                                        info_dict=target_info,
+                                        allow_shared_blocks=allow_shared_blocks,
+                                        hashtree_info_generator=
+                                        hashtree_info_generator)
+
+    # Check the first block of the source system partition for remount R/W only
+    # if the filesystem is ext4.
+    partition_source_info = source_info["fstab"]["/" + name]
+    check_first_block = partition_source_info.fs_type == "ext4"
+    # Disable using imgdiff for squashfs. 'imgdiff -z' expects input files to be
+    # in zip formats. However with squashfs, a) all files are compressed in LZ4;
+    # b) the blocks listed in block map may not contain all the bytes for a
+    # given file (because they're rounded to be 4K-aligned).
+    partition_target_info = target_info["fstab"]["/" + name]
+    disable_imgdiff = (partition_source_info.fs_type == "squashfs" or
+                       partition_target_info.fs_type == "squashfs")
+    return common.BlockDifference(name, partition_src, partition_tgt,
+                                  check_first_block,
+                                  version=blockimgdiff_version,
+                                  disable_imgdiff=disable_imgdiff)
+
+  if source_zip:
+    # See notes in common.GetUserImage()
+    allow_shared_blocks = (source_info.get('ext4_share_dup_blocks') == "true" or
+                           target_info.get('ext4_share_dup_blocks') == "true")
+    blockimgdiff_version = max(
+        int(i) for i in target_info.get(
+            "blockimgdiff_versions", "1").split(","))
+    assert blockimgdiff_version >= 3
+
+  block_diff_dict = collections.OrderedDict()
+  partition_names = ["system", "vendor", "product", "odm", "system_ext"]
+  for partition in partition_names:
+    if not HasPartition(target_zip, partition):
+      continue
+    # Full OTA update.
+    if not source_zip:
+      tgt = common.GetUserImage(partition, OPTIONS.input_tmp, target_zip,
+                                info_dict=target_info,
+                                reset_file_map=True)
+      block_diff_dict[partition] = common.BlockDifference(partition, tgt,
+                                                          src=None)
+    # Incremental OTA update.
+    else:
+      block_diff_dict[partition] = GetIncrementalBlockDifferenceForPartition(
+          partition)
+  assert "system" in block_diff_dict
+
+  # Get the block diffs from the device specific script. If there is a
+  # duplicate block diff for a partition, ignore the diff in the generic script
+  # and use the one in the device specific script instead.
+  if source_zip:
+    device_specific_diffs = device_specific.IncrementalOTA_GetBlockDifferences()
+    function_name = "IncrementalOTA_GetBlockDifferences"
+  else:
+    device_specific_diffs = device_specific.FullOTA_GetBlockDifferences()
+    function_name = "FullOTA_GetBlockDifferences"
+
+  if device_specific_diffs:
+    assert all(isinstance(diff, common.BlockDifference)
+               for diff in device_specific_diffs), \
+        "{} is not returning a list of BlockDifference objects".format(
+            function_name)
+    for diff in device_specific_diffs:
+      if diff.partition in block_diff_dict:
+        logger.warning("Duplicate block difference found. Device specific block"
+                       " diff for partition '%s' overrides the one in generic"
+                       " script.", diff.partition)
+      block_diff_dict[diff.partition] = diff
+
+  return block_diff_dict
 
 def WriteFullOTAPackage(input_zip, output_file):
   target_info = BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts)
@@ -911,6 +1006,11 @@ def WriteFullOTAPackage(input_zip, output_file):
   target_info.WriteDeviceAssertions(script, OPTIONS.oem_no_mount)
   device_specific.FullOTA_Assertions()
 
+  block_diff_dict = GetBlockDifferences(target_zip=input_zip, source_zip=None,
+                                        target_info=target_info,
+                                        source_info=None,
+                                        device_specific=device_specific)
+
   # Two-step package strategy (in chronological order, which is *not*
   # the order in which the generated script has things):
   #
@@ -990,52 +1090,27 @@ def WriteFullOTAPackage(input_zip, output_file):
     script.RunBackup("backup", sysmount)
     script.Unmount(sysmount);
 
-  system_progress = 0.75
-
+  # All other partitions as well as the data wipe use 10% of the progress, and
+  # the update of the system partition takes the remaining progress.
+  system_progress = 0.9 - (len(block_diff_dict) - 1) * 0.1
   if OPTIONS.wipe_user_data:
     system_progress -= 0.1
-  if HasVendorPartition(input_zip):
-    system_progress -= 0.1
-
-  script.ShowProgress(system_progress, 0)
 
-  def GetBlockDifference(partition):
-    # Full OTA is done as an "incremental" against an empty source image. This
-    # has the effect of writing new data from the package to the entire
-    # partition, but lets us reuse the updater code that writes incrementals to
-    # do it.
-    tgt = common.GetUserImage(partition, OPTIONS.input_tmp, input_zip,
-                              info_dict=target_info,
-                              reset_file_map=True)
-    diff = common.BlockDifference(partition, tgt, src=None, brotli=OPTIONS.brotli)
-    return diff
-
-  device_specific_diffs = device_specific.FullOTA_GetBlockDifferences()
-  if device_specific_diffs:
-    assert all(isinstance(diff, common.BlockDifference)
-               for diff in device_specific_diffs), \
-        "FullOTA_GetBlockDifferences is not returning a list of " \
-        "BlockDifference objects"
-
-  progress_dict = dict()
-  block_diffs = [GetBlockDifference("system")]
-  if HasVendorPartition(input_zip):
-    block_diffs.append(GetBlockDifference("vendor"))
-    progress_dict["vendor"] = 0.1
-  if device_specific_diffs:
-    block_diffs += device_specific_diffs
+  progress_dict = {partition: 0.1 for partition in block_diff_dict}
+  progress_dict["system"] = system_progress
 
   if target_info.get('use_dynamic_partitions') == "true":
     # Use empty source_info_dict to indicate that all partitions / groups must
     # be re-added.
     dynamic_partitions_diff = common.DynamicPartitionsDifference(
         info_dict=OPTIONS.info_dict,
-        block_diffs=block_diffs,
-        progress_dict=progress_dict)
+        block_diffs=block_diff_dict.values(),
+        progress_dict=progress_dict,
+        build_without_vendor=(not HasVendorPartition(input_zip)))
     dynamic_partitions_diff.WriteScript(script, output_zip,
                                         write_verify_script=OPTIONS.verify)
   else:
-    for block_diff in block_diffs:
+    for block_diff in block_diff_dict.values():
       block_diff.WriteScript(script, output_zip,
                              progress=progress_dict.get(block_diff.partition),
                              write_verify_script=OPTIONS.verify)
@@ -1054,10 +1129,9 @@ def GetBlockDifference(partition):
     script.RunBackup("restore", sysmount)
     script.Unmount(sysmount);
 
-  script.ShowProgress(0.05, 5)
   script.WriteRawImage("/boot", "boot.img")
 
-  script.ShowProgress(0.2, 10)
+  script.ShowProgress(0.1, 10)
   device_specific.FullOTA_InstallEnd()
 
   if OPTIONS.extra_script is not None:
@@ -1617,6 +1691,7 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
   target_recovery = common.GetBootableImage(
       "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY")
 
+<<<<<<< HEAD
   # See notes in common.GetUserImage()
   allow_shared_blocks = (source_info.get('ext4_share_dup_blocks') == "true" or
                          target_info.get('ext4_share_dup_blocks') == "true")
@@ -1679,6 +1754,13 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
                                          brotli=OPTIONS.brotli)
   else:
     vendor_diff = None
+=======
+  block_diff_dict = GetBlockDifferences(target_zip=target_zip,
+                                        source_zip=source_zip,
+                                        target_info=target_info,
+                                        source_info=source_info,
+                                        device_specific=device_specific)
+>>>>>>> 06e025ba3... Generate block based OTA for product partitions
 
   AddCompatibilityArchiveIfTrebleEnabled(
       target_zip, output_zip, target_info, source_info)
@@ -1745,12 +1827,8 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
   WriteFingerprintAssertion(script, target_info, source_info)
 
   # Check the required cache size (i.e. stashed blocks).
-  size = []
-  if system_diff:
-    size.append(system_diff.required_cache)
-  if vendor_diff:
-    size.append(vendor_diff.required_cache)
-
+  required_cache_sizes = [diff.required_cache for diff in
+                          block_diff_dict.values()]
   if updating_boot:
     boot_type, boot_device = common.GetTypeAndDevice("/boot", source_info)
     d = common.Difference(target_boot, source_boot)
@@ -1773,10 +1851,14 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
           "{}:{}:{}:{}".format(
               boot_type, boot_device, source_boot.size, source_boot.sha1))
 
-      size.append(target_boot.size)
+      required_cache_sizes.append(target_boot.size)
+
+  if required_cache_sizes:
+    script.CacheFreeSpaceCheck(max(required_cache_sizes))
 
-  if size:
-    script.CacheFreeSpaceCheck(max(size))
+  # Verify the existing partitions.
+  for diff in block_diff_dict.values():
+    diff.WriteVerifyScript(script, touched_blocks_only=True)
 
   device_specific.IncrementalOTA_VerifyEnd()
 
@@ -1793,30 +1875,12 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
     # Stage 3/3: Make changes.
     script.Comment("Stage 3/3")
 
-  # Verify the existing partitions.
-  system_diff.WriteVerifyScript(script, touched_blocks_only=True)
-  if vendor_diff:
-    vendor_diff.WriteVerifyScript(script, touched_blocks_only=True)
-  device_specific_diffs = device_specific.IncrementalOTA_GetBlockDifferences()
-  if device_specific_diffs:
-    assert all(isinstance(diff, common.BlockDifference)
-               for diff in device_specific_diffs), \
-        "IncrementalOTA_GetBlockDifferences is not returning a list of " \
-        "BlockDifference objects"
-    for diff in device_specific_diffs:
-      diff.WriteVerifyScript(script, touched_blocks_only=True)
-
   script.Comment("---- start making changes here ----")
 
   device_specific.IncrementalOTA_InstallBegin()
 
-  block_diffs = [system_diff]
-  progress_dict = {"system": 0.8 if vendor_diff else 0.9}
-  if vendor_diff:
-    block_diffs.append(vendor_diff)
-    progress_dict["vendor"] = 0.1
-  if device_specific_diffs:
-    block_diffs += device_specific_diffs
+  progress_dict = {partition: 0.1 for partition in block_diff_dict}
+  progress_dict["system"] = 1 - len(block_diff_dict) * 0.1
 
   if OPTIONS.source_info_dict.get("use_dynamic_partitions") == "true":
     if OPTIONS.target_info_dict.get("use_dynamic_partitions") != "true":
@@ -1825,12 +1889,12 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
     dynamic_partitions_diff = common.DynamicPartitionsDifference(
         info_dict=OPTIONS.target_info_dict,
         source_info_dict=OPTIONS.source_info_dict,
-        block_diffs=block_diffs,
+        block_diffs=block_diff_dict.values(),
         progress_dict=progress_dict)
     dynamic_partitions_diff.WriteScript(
         script, output_zip, write_verify_script=OPTIONS.verify)
   else:
-    for block_diff in block_diffs:
+    for block_diff in block_diff_dict.values():
       block_diff.WriteScript(script, output_zip,
                              progress=progress_dict.get(block_diff.partition),
                              write_verify_script=OPTIONS.verify)