From 2e8eeecad030ee8799d5f6d94520934fca373e68 Mon Sep 17 00:00:00 2001 From: Peter Cai Date: Sun, 1 Mar 2020 14:43:57 +0800 Subject: [PATCH 1/2] releasetools: do not remove dynamic partitions in system-only builds * Before this commit, the generated `dynamic_partitions_op_list` in FullOTA packages always tries to remove all partitions and recreate them upon flashing. This makes it impossible to have a system-only "FullOTA" because vendor partition(s) are always removed. * This commit detects if a build is vendor-less and disables every dynamic partition operation except `resize`, in order to keep the original content around after the flash. The change should not affect non-dynamic-partition or builds with vendor image included. Change-Id: I0cded7f3b2958f35103d73d19b7fb5f292f6c17f Signed-off-by: Jesse Chan --- tools/releasetools/common.py | 14 +++++++++++++- tools/releasetools/ota_from_target_files.py | 3 ++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index 3dcd7bd59f0..b2a310c6ba8 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -2372,10 +2372,11 @@ def __init__(self, src_size=None, tgt_size=None): class DynamicPartitionsDifference(object): def __init__(self, info_dict, block_diffs, progress_dict=None, - source_info_dict=None): + source_info_dict=None, build_without_vendor=False): if progress_dict is None: progress_dict = dict() + self._build_without_vendor = build_without_vendor self._remove_all_before_apply = False if source_info_dict is None: self._remove_all_before_apply = True @@ -2498,6 +2499,17 @@ def append(line): def comment(line): self._op_list.append("# %s" % line) + if self._build_without_vendor: + comment('System-only build, keep original vendor partition') + # When building without vendor, we do not want to override + # any partition already existing. In this case, we can only + # resize, but not remove / create / re-create any other + # partition. + for p, u in self._partition_updates.items(): + comment('Resize partition %s to %s' % (p, u.tgt_size)) + append('resize %s %s' % (p, u.tgt_size)) + return + if self._remove_all_before_apply: comment('Remove all existing dynamic partitions and groups before ' 'applying full OTA') diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py index 2a0583301b2..fa849317d77 100755 --- a/tools/releasetools/ota_from_target_files.py +++ b/tools/releasetools/ota_from_target_files.py @@ -1031,7 +1031,8 @@ def GetBlockDifference(partition): dynamic_partitions_diff = common.DynamicPartitionsDifference( info_dict=OPTIONS.info_dict, block_diffs=block_diffs, - progress_dict=progress_dict) + progress_dict=progress_dict, + build_without_vendor=(not HasVendorPartition(input_zip))) dynamic_partitions_diff.WriteScript(script, output_zip, write_verify_script=OPTIONS.verify) else: From 9235273bcf0654a62145411cfbe1999a932d50a3 Mon Sep 17 00:00:00 2001 From: Tianjie Xu Date: Mon, 20 May 2019 17:50:36 -0700 Subject: [PATCH 2/2] Generate block based OTA for product partitions Currently only the system and vendor partitions are updated in the generic block based OTA generation script. Since the product partition is quite large and consist most of APK files, it would be beneficial to update it similar to system. Handle the odm and system_ext partitions in the same way as well. Bug: 132683080 Test: Run unit tests, generate full and incremental OTA with product partitions. Change-Id: I13478cf9bd32137c6729b8c9cb102080147093f2 --- tools/releasetools/ota_from_target_files.py | 195 +++++++++++++------- 1 file changed, 129 insertions(+), 66 deletions(-) diff --git a/tools/releasetools/ota_from_target_files.py b/tools/releasetools/ota_from_target_files.py index fa849317d77..5fe074a3317 100755 --- a/tools/releasetools/ota_from_target_files.py +++ b/tools/releasetools/ota_from_target_files.py @@ -192,6 +192,7 @@ from __future__ import print_function +import collections import logging import multiprocessing import os.path @@ -869,6 +870,100 @@ def FingerprintChanged(source_fp, target_fp): AddCompatibilityArchive(system_updated or product_updated, vendor_updated or odm_updated) +def CopyInstallTools(output_zip): + install_path = os.path.join(OPTIONS.input_tmp, "INSTALL") + for root, subdirs, files in os.walk(install_path): + for f in files: + install_source = os.path.join(root, f) + install_target = os.path.join("install", os.path.relpath(root, install_path), f) + output_zip.write(install_source, install_target) + +def GetBlockDifferences(target_zip, source_zip, target_info, source_info, + device_specific): + """Returns a ordered dict of block differences with partition name as key.""" + + def GetIncrementalBlockDifferenceForPartition(name): + if not HasPartition(source_zip, name): + raise RuntimeError("can't generate incremental that adds {}".format(name)) + + partition_src = common.GetUserImage(name, OPTIONS.source_tmp, source_zip, + info_dict=source_info, + allow_shared_blocks=allow_shared_blocks) + + hashtree_info_generator = verity_utils.CreateHashtreeInfoGenerator( + name, 4096, target_info) + partition_tgt = common.GetUserImage(name, OPTIONS.target_tmp, target_zip, + info_dict=target_info, + allow_shared_blocks=allow_shared_blocks, + hashtree_info_generator= + hashtree_info_generator) + + # Check the first block of the source system partition for remount R/W only + # if the filesystem is ext4. + partition_source_info = source_info["fstab"]["/" + name] + check_first_block = partition_source_info.fs_type == "ext4" + # Disable using imgdiff for squashfs. 'imgdiff -z' expects input files to be + # in zip formats. However with squashfs, a) all files are compressed in LZ4; + # b) the blocks listed in block map may not contain all the bytes for a + # given file (because they're rounded to be 4K-aligned). + partition_target_info = target_info["fstab"]["/" + name] + disable_imgdiff = (partition_source_info.fs_type == "squashfs" or + partition_target_info.fs_type == "squashfs") + return common.BlockDifference(name, partition_src, partition_tgt, + check_first_block, + version=blockimgdiff_version, + disable_imgdiff=disable_imgdiff) + + if source_zip: + # See notes in common.GetUserImage() + allow_shared_blocks = (source_info.get('ext4_share_dup_blocks') == "true" or + target_info.get('ext4_share_dup_blocks') == "true") + blockimgdiff_version = max( + int(i) for i in target_info.get( + "blockimgdiff_versions", "1").split(",")) + assert blockimgdiff_version >= 3 + + block_diff_dict = collections.OrderedDict() + partition_names = ["system", "vendor", "product", "odm", "system_ext"] + for partition in partition_names: + if not HasPartition(target_zip, partition): + continue + # Full OTA update. + if not source_zip: + tgt = common.GetUserImage(partition, OPTIONS.input_tmp, target_zip, + info_dict=target_info, + reset_file_map=True) + block_diff_dict[partition] = common.BlockDifference(partition, tgt, + src=None) + # Incremental OTA update. + else: + block_diff_dict[partition] = GetIncrementalBlockDifferenceForPartition( + partition) + assert "system" in block_diff_dict + + # Get the block diffs from the device specific script. If there is a + # duplicate block diff for a partition, ignore the diff in the generic script + # and use the one in the device specific script instead. + if source_zip: + device_specific_diffs = device_specific.IncrementalOTA_GetBlockDifferences() + function_name = "IncrementalOTA_GetBlockDifferences" + else: + device_specific_diffs = device_specific.FullOTA_GetBlockDifferences() + function_name = "FullOTA_GetBlockDifferences" + + if device_specific_diffs: + assert all(isinstance(diff, common.BlockDifference) + for diff in device_specific_diffs), \ + "{} is not returning a list of BlockDifference objects".format( + function_name) + for diff in device_specific_diffs: + if diff.partition in block_diff_dict: + logger.warning("Duplicate block difference found. Device specific block" + " diff for partition '%s' overrides the one in generic" + " script.", diff.partition) + block_diff_dict[diff.partition] = diff + + return block_diff_dict def WriteFullOTAPackage(input_zip, output_file): target_info = BuildInfo(OPTIONS.info_dict, OPTIONS.oem_dicts) @@ -911,6 +1006,11 @@ def WriteFullOTAPackage(input_zip, output_file): target_info.WriteDeviceAssertions(script, OPTIONS.oem_no_mount) device_specific.FullOTA_Assertions() + block_diff_dict = GetBlockDifferences(target_zip=input_zip, source_zip=None, + target_info=target_info, + source_info=None, + device_specific=device_specific) + # Two-step package strategy (in chronological order, which is *not* # the order in which the generated script has things): # @@ -990,53 +1090,27 @@ def WriteFullOTAPackage(input_zip, output_file): script.RunBackup("backup", sysmount) script.Unmount(sysmount); - system_progress = 0.75 - + # All other partitions as well as the data wipe use 10% of the progress, and + # the update of the system partition takes the remaining progress. + system_progress = 0.9 - (len(block_diff_dict) - 1) * 0.1 if OPTIONS.wipe_user_data: system_progress -= 0.1 - if HasVendorPartition(input_zip): - system_progress -= 0.1 - script.ShowProgress(system_progress, 0) - - def GetBlockDifference(partition): - # Full OTA is done as an "incremental" against an empty source image. This - # has the effect of writing new data from the package to the entire - # partition, but lets us reuse the updater code that writes incrementals to - # do it. - tgt = common.GetUserImage(partition, OPTIONS.input_tmp, input_zip, - info_dict=target_info, - reset_file_map=True) - diff = common.BlockDifference(partition, tgt, src=None, brotli=OPTIONS.brotli) - return diff - - device_specific_diffs = device_specific.FullOTA_GetBlockDifferences() - if device_specific_diffs: - assert all(isinstance(diff, common.BlockDifference) - for diff in device_specific_diffs), \ - "FullOTA_GetBlockDifferences is not returning a list of " \ - "BlockDifference objects" - - progress_dict = dict() - block_diffs = [GetBlockDifference("system")] - if HasVendorPartition(input_zip): - block_diffs.append(GetBlockDifference("vendor")) - progress_dict["vendor"] = 0.1 - if device_specific_diffs: - block_diffs += device_specific_diffs + progress_dict = {partition: 0.1 for partition in block_diff_dict} + progress_dict["system"] = system_progress if target_info.get('use_dynamic_partitions') == "true": # Use empty source_info_dict to indicate that all partitions / groups must # be re-added. dynamic_partitions_diff = common.DynamicPartitionsDifference( info_dict=OPTIONS.info_dict, - block_diffs=block_diffs, + block_diffs=block_diff_dict.values(), progress_dict=progress_dict, build_without_vendor=(not HasVendorPartition(input_zip))) dynamic_partitions_diff.WriteScript(script, output_zip, write_verify_script=OPTIONS.verify) else: - for block_diff in block_diffs: + for block_diff in block_diff_dict.values(): block_diff.WriteScript(script, output_zip, progress=progress_dict.get(block_diff.partition), write_verify_script=OPTIONS.verify) @@ -1055,10 +1129,9 @@ def GetBlockDifference(partition): script.RunBackup("restore", sysmount) script.Unmount(sysmount); - script.ShowProgress(0.05, 5) script.WriteRawImage("/boot", "boot.img") - script.ShowProgress(0.2, 10) + script.ShowProgress(0.1, 10) device_specific.FullOTA_InstallEnd() if OPTIONS.extra_script is not None: @@ -1618,6 +1691,7 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file): target_recovery = common.GetBootableImage( "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY") +<<<<<<< HEAD # See notes in common.GetUserImage() allow_shared_blocks = (source_info.get('ext4_share_dup_blocks') == "true" or target_info.get('ext4_share_dup_blocks') == "true") @@ -1680,6 +1754,13 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file): brotli=OPTIONS.brotli) else: vendor_diff = None +======= + block_diff_dict = GetBlockDifferences(target_zip=target_zip, + source_zip=source_zip, + target_info=target_info, + source_info=source_info, + device_specific=device_specific) +>>>>>>> 06e025ba3... Generate block based OTA for product partitions AddCompatibilityArchiveIfTrebleEnabled( target_zip, output_zip, target_info, source_info) @@ -1746,12 +1827,8 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file): WriteFingerprintAssertion(script, target_info, source_info) # Check the required cache size (i.e. stashed blocks). - size = [] - if system_diff: - size.append(system_diff.required_cache) - if vendor_diff: - size.append(vendor_diff.required_cache) - + required_cache_sizes = [diff.required_cache for diff in + block_diff_dict.values()] if updating_boot: boot_type, boot_device = common.GetTypeAndDevice("/boot", source_info) d = common.Difference(target_boot, source_boot) @@ -1774,10 +1851,14 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file): "{}:{}:{}:{}".format( boot_type, boot_device, source_boot.size, source_boot.sha1)) - size.append(target_boot.size) + required_cache_sizes.append(target_boot.size) + + if required_cache_sizes: + script.CacheFreeSpaceCheck(max(required_cache_sizes)) - if size: - script.CacheFreeSpaceCheck(max(size)) + # Verify the existing partitions. + for diff in block_diff_dict.values(): + diff.WriteVerifyScript(script, touched_blocks_only=True) device_specific.IncrementalOTA_VerifyEnd() @@ -1794,30 +1875,12 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file): # Stage 3/3: Make changes. script.Comment("Stage 3/3") - # Verify the existing partitions. - system_diff.WriteVerifyScript(script, touched_blocks_only=True) - if vendor_diff: - vendor_diff.WriteVerifyScript(script, touched_blocks_only=True) - device_specific_diffs = device_specific.IncrementalOTA_GetBlockDifferences() - if device_specific_diffs: - assert all(isinstance(diff, common.BlockDifference) - for diff in device_specific_diffs), \ - "IncrementalOTA_GetBlockDifferences is not returning a list of " \ - "BlockDifference objects" - for diff in device_specific_diffs: - diff.WriteVerifyScript(script, touched_blocks_only=True) - script.Comment("---- start making changes here ----") device_specific.IncrementalOTA_InstallBegin() - block_diffs = [system_diff] - progress_dict = {"system": 0.8 if vendor_diff else 0.9} - if vendor_diff: - block_diffs.append(vendor_diff) - progress_dict["vendor"] = 0.1 - if device_specific_diffs: - block_diffs += device_specific_diffs + progress_dict = {partition: 0.1 for partition in block_diff_dict} + progress_dict["system"] = 1 - len(block_diff_dict) * 0.1 if OPTIONS.source_info_dict.get("use_dynamic_partitions") == "true": if OPTIONS.target_info_dict.get("use_dynamic_partitions") != "true": @@ -1826,12 +1889,12 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file): dynamic_partitions_diff = common.DynamicPartitionsDifference( info_dict=OPTIONS.target_info_dict, source_info_dict=OPTIONS.source_info_dict, - block_diffs=block_diffs, + block_diffs=block_diff_dict.values(), progress_dict=progress_dict) dynamic_partitions_diff.WriteScript( script, output_zip, write_verify_script=OPTIONS.verify) else: - for block_diff in block_diffs: + for block_diff in block_diff_dict.values(): block_diff.WriteScript(script, output_zip, progress=progress_dict.get(block_diff.partition), write_verify_script=OPTIONS.verify)