qemu

FORK: QEMU emulator
git clone https://git.neptards.moe/neptards/qemu.git
Log | Files | Refs | Submodules | LICENSE

124 (28993B)


      1 #!/usr/bin/env python3
      2 # group: rw backing
      3 #
      4 # Tests for incremental drive-backup
      5 #
      6 # Copyright (C) 2015 John Snow for Red Hat, Inc.
      7 #
      8 # Based on 056.
      9 #
     10 # This program is free software; you can redistribute it and/or modify
     11 # it under the terms of the GNU General Public License as published by
     12 # the Free Software Foundation; either version 2 of the License, or
     13 # (at your option) any later version.
     14 #
     15 # This program is distributed in the hope that it will be useful,
     16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
     17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     18 # GNU General Public License for more details.
     19 #
     20 # You should have received a copy of the GNU General Public License
     21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
     22 #
     23 
     24 import os
     25 import iotests
     26 from iotests import try_remove
     27 
     28 
     29 def io_write_patterns(img, patterns):
     30     for pattern in patterns:
     31         iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img)
     32 
     33 
     34 def transaction_action(action, **kwargs):
     35     return {
     36         'type': action,
     37         'data': dict((k.replace('_', '-'), v) for k, v in kwargs.items())
     38     }
     39 
     40 
     41 def transaction_bitmap_clear(node, name, **kwargs):
     42     return transaction_action('block-dirty-bitmap-clear',
     43                               node=node, name=name, **kwargs)
     44 
     45 
     46 def transaction_drive_backup(device, target, **kwargs):
     47     return transaction_action('drive-backup', job_id=device, device=device,
     48                               target=target, **kwargs)
     49 
     50 
     51 class Bitmap:
     52     def __init__(self, name, drive):
     53         self.name = name
     54         self.drive = drive
     55         self.num = 0
     56         self.backups = list()
     57 
     58     def base_target(self):
     59         return (self.drive['backup'], None)
     60 
     61     def new_target(self, num=None):
     62         if num is None:
     63             num = self.num
     64         self.num = num + 1
     65         base = os.path.join(iotests.test_dir,
     66                             "%s.%s." % (self.drive['id'], self.name))
     67         suff = "%i.%s" % (num, self.drive['fmt'])
     68         target = base + "inc" + suff
     69         reference = base + "ref" + suff
     70         self.backups.append((target, reference))
     71         return (target, reference)
     72 
     73     def last_target(self):
     74         if self.backups:
     75             return self.backups[-1]
     76         return self.base_target()
     77 
     78     def del_target(self):
     79         for image in self.backups.pop():
     80             try_remove(image)
     81         self.num -= 1
     82 
     83     def cleanup(self):
     84         for backup in self.backups:
     85             for image in backup:
     86                 try_remove(image)
     87 
     88 
     89 class TestIncrementalBackupBase(iotests.QMPTestCase):
     90     def __init__(self, *args):
     91         super(TestIncrementalBackupBase, self).__init__(*args)
     92         self.bitmaps = list()
     93         self.files = list()
     94         self.drives = list()
     95         self.vm = iotests.VM()
     96         self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt)
     97 
     98 
     99     def setUp(self):
    100         # Create a base image with a distinctive patterning
    101         drive0 = self.add_node('drive0')
    102         self.img_create(drive0['file'], drive0['fmt'])
    103         self.vm.add_drive(drive0['file'], opts='node-name=node0')
    104         self.write_default_pattern(drive0['file'])
    105         self.vm.launch()
    106 
    107 
    108     def write_default_pattern(self, target):
    109         io_write_patterns(target, (('0x41', 0, 512),
    110                                    ('0xd5', '1M', '32k'),
    111                                    ('0xdc', '32M', '124k')))
    112 
    113 
    114     def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None):
    115         if path is None:
    116             path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt))
    117         if backup is None:
    118             backup = os.path.join(iotests.test_dir,
    119                                   '%s.full.backup.%s' % (node_id, fmt))
    120 
    121         self.drives.append({
    122             'id': node_id,
    123             'file': path,
    124             'backup': backup,
    125             'fmt': fmt })
    126         return self.drives[-1]
    127 
    128 
    129     def img_create(self, img, fmt=iotests.imgfmt, size='64M',
    130                    parent=None, parentFormat=None, **kwargs):
    131         optargs = []
    132         for k,v in kwargs.items():
    133             optargs = optargs + ['-o', '%s=%s' % (k,v)]
    134         args = ['create', '-f', fmt] + optargs + [img, size]
    135         if parent:
    136             if parentFormat is None:
    137                 parentFormat = fmt
    138             args = args + ['-b', parent, '-F', parentFormat]
    139         iotests.qemu_img(*args)
    140         self.files.append(img)
    141 
    142 
    143     def do_qmp_backup(self, error='Input/output error', **kwargs):
    144         res = self.vm.qmp('drive-backup', **kwargs)
    145         self.assert_qmp(res, 'return', {})
    146         return self.wait_qmp_backup(kwargs['device'], error)
    147 
    148 
    149     def ignore_job_status_change_events(self):
    150         while True:
    151             e = self.vm.event_wait(name="JOB_STATUS_CHANGE")
    152             if e['data']['status'] == 'null':
    153                 break
    154 
    155     def wait_qmp_backup(self, device, error='Input/output error'):
    156         event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED",
    157                                    match={'data': {'device': device}})
    158         self.assertNotEqual(event, None)
    159         self.ignore_job_status_change_events()
    160 
    161         try:
    162             failure = self.dictpath(event, 'data/error')
    163         except AssertionError:
    164             # Backup succeeded.
    165             self.assert_qmp(event, 'data/offset', event['data']['len'])
    166             return True
    167         else:
    168             # Backup failed.
    169             self.assert_qmp(event, 'data/error', error)
    170             return False
    171 
    172 
    173     def wait_qmp_backup_cancelled(self, device):
    174         event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED',
    175                                    match={'data': {'device': device}})
    176         self.assertNotEqual(event, None)
    177         self.ignore_job_status_change_events()
    178 
    179 
    180     def create_anchor_backup(self, drive=None):
    181         if drive is None:
    182             drive = self.drives[-1]
    183         res = self.do_qmp_backup(job_id=drive['id'],
    184                                  device=drive['id'], sync='full',
    185                                  format=drive['fmt'], target=drive['backup'])
    186         self.assertTrue(res)
    187         self.files.append(drive['backup'])
    188         return drive['backup']
    189 
    190 
    191     def make_reference_backup(self, bitmap=None):
    192         if bitmap is None:
    193             bitmap = self.bitmaps[-1]
    194         _, reference = bitmap.last_target()
    195         res = self.do_qmp_backup(job_id=bitmap.drive['id'],
    196                                  device=bitmap.drive['id'], sync='full',
    197                                  format=bitmap.drive['fmt'], target=reference)
    198         self.assertTrue(res)
    199 
    200 
    201     def add_bitmap(self, name, drive, **kwargs):
    202         bitmap = Bitmap(name, drive)
    203         self.bitmaps.append(bitmap)
    204         result = self.vm.qmp('block-dirty-bitmap-add', node=drive['id'],
    205                              name=bitmap.name, **kwargs)
    206         self.assert_qmp(result, 'return', {})
    207         return bitmap
    208 
    209 
    210     def prepare_backup(self, bitmap=None, parent=None, **kwargs):
    211         if bitmap is None:
    212             bitmap = self.bitmaps[-1]
    213         if parent is None:
    214             parent, _ = bitmap.last_target()
    215 
    216         target, _ = bitmap.new_target()
    217         self.img_create(target, bitmap.drive['fmt'], parent=parent,
    218                         **kwargs)
    219         return target
    220 
    221 
    222     def create_incremental(self, bitmap=None, parent=None,
    223                            parentFormat=None, validate=True,
    224                            target=None):
    225         if bitmap is None:
    226             bitmap = self.bitmaps[-1]
    227         if parent is None:
    228             parent, _ = bitmap.last_target()
    229 
    230         if target is None:
    231             target = self.prepare_backup(bitmap, parent)
    232         res = self.do_qmp_backup(job_id=bitmap.drive['id'],
    233                                  device=bitmap.drive['id'],
    234                                  sync='incremental', bitmap=bitmap.name,
    235                                  format=bitmap.drive['fmt'], target=target,
    236                                  mode='existing')
    237         if not res:
    238             bitmap.del_target();
    239             self.assertFalse(validate)
    240         else:
    241             self.make_reference_backup(bitmap)
    242         return res
    243 
    244 
    245     def check_backups(self):
    246         for bitmap in self.bitmaps:
    247             for incremental, reference in bitmap.backups:
    248                 self.assertTrue(iotests.compare_images(incremental, reference))
    249             last = bitmap.last_target()[0]
    250             self.assertTrue(iotests.compare_images(last, bitmap.drive['file']))
    251 
    252 
    253     def hmp_io_writes(self, drive, patterns):
    254         for pattern in patterns:
    255             self.vm.hmp_qemu_io(drive, 'write -P%s %s %s' % pattern)
    256         self.vm.hmp_qemu_io(drive, 'flush')
    257 
    258 
    259     def do_incremental_simple(self, **kwargs):
    260         self.create_anchor_backup()
    261         self.add_bitmap('bitmap0', self.drives[0], **kwargs)
    262 
    263         # Sanity: Create a "hollow" incremental backup
    264         self.create_incremental()
    265         # Three writes: One complete overwrite, one new segment,
    266         # and one partial overlap.
    267         self.hmp_io_writes(self.drives[0]['id'], (('0xab', 0, 512),
    268                                                   ('0xfe', '16M', '256k'),
    269                                                   ('0x64', '32736k', '64k')))
    270         self.create_incremental()
    271         # Three more writes, one of each kind, like above
    272         self.hmp_io_writes(self.drives[0]['id'], (('0x9a', 0, 512),
    273                                                   ('0x55', '8M', '352k'),
    274                                                   ('0x78', '15872k', '1M')))
    275         self.create_incremental()
    276         self.vm.shutdown()
    277         self.check_backups()
    278 
    279 
    280     def tearDown(self):
    281         self.vm.shutdown()
    282         for bitmap in self.bitmaps:
    283             bitmap.cleanup()
    284         for filename in self.files:
    285             try_remove(filename)
    286 
    287 
    288 
    289 class TestIncrementalBackup(TestIncrementalBackupBase):
    290     def test_incremental_simple(self):
    291         '''
    292         Test: Create and verify three incremental backups.
    293 
    294         Create a bitmap and a full backup before VM execution begins,
    295         then create a series of three incremental backups "during execution,"
    296         i.e.; after IO requests begin modifying the drive.
    297         '''
    298         return self.do_incremental_simple()
    299 
    300 
    301     def test_small_granularity(self):
    302         '''
    303         Test: Create and verify backups made with a small granularity bitmap.
    304 
    305         Perform the same test as test_incremental_simple, but with a granularity
    306         of only 32KiB instead of the present default of 64KiB.
    307         '''
    308         return self.do_incremental_simple(granularity=32768)
    309 
    310 
    311     def test_large_granularity(self):
    312         '''
    313         Test: Create and verify backups made with a large granularity bitmap.
    314 
    315         Perform the same test as test_incremental_simple, but with a granularity
    316         of 128KiB instead of the present default of 64KiB.
    317         '''
    318         return self.do_incremental_simple(granularity=131072)
    319 
    320 
    321     def test_larger_cluster_target(self):
    322         '''
    323         Test: Create and verify backups made to a larger cluster size target.
    324 
    325         With a default granularity of 64KiB, verify that backups made to a
    326         larger cluster size target of 128KiB without a backing file works.
    327         '''
    328         drive0 = self.drives[0]
    329 
    330         # Create a cluster_size=128k full backup / "anchor" backup
    331         self.img_create(drive0['backup'], cluster_size='128k')
    332         self.assertTrue(self.do_qmp_backup(device=drive0['id'], sync='full',
    333                                            format=drive0['fmt'],
    334                                            target=drive0['backup'],
    335                                            mode='existing'))
    336 
    337         # Create bitmap and dirty it with some new writes.
    338         # overwrite [32736, 32799] which will dirty bitmap clusters at
    339         # 32M-64K and 32M. 32M+64K will be left undirtied.
    340         bitmap0 = self.add_bitmap('bitmap0', drive0)
    341         self.hmp_io_writes(drive0['id'],
    342                            (('0xab', 0, 512),
    343                             ('0xfe', '16M', '256k'),
    344                             ('0x64', '32736k', '64k')))
    345         # Check the dirty bitmap stats
    346         self.assertTrue(self.vm.check_bitmap_status(
    347             'node0', bitmap0.name, {
    348                 'name': 'bitmap0',
    349                 'count': 458752,
    350                 'granularity': 65536,
    351                 'persistent': False
    352             }))
    353 
    354         # Prepare a cluster_size=128k backup target without a backing file.
    355         (target, _) = bitmap0.new_target()
    356         self.img_create(target, bitmap0.drive['fmt'], cluster_size='128k')
    357 
    358         # Perform Incremental Backup
    359         self.assertTrue(self.do_qmp_backup(device=bitmap0.drive['id'],
    360                                            sync='incremental',
    361                                            bitmap=bitmap0.name,
    362                                            format=bitmap0.drive['fmt'],
    363                                            target=target,
    364                                            mode='existing'))
    365         self.make_reference_backup(bitmap0)
    366 
    367         # Add the backing file, then compare and exit.
    368         iotests.qemu_img('rebase', '-f', drive0['fmt'], '-u', '-b',
    369                          drive0['backup'], '-F', drive0['fmt'], target)
    370         self.vm.shutdown()
    371         self.check_backups()
    372 
    373 
    374     def test_incremental_transaction(self):
    375         '''Test: Verify backups made from transactionally created bitmaps.
    376 
    377         Create a bitmap "before" VM execution begins, then create a second
    378         bitmap AFTER writes have already occurred. Use transactions to create
    379         a full backup and synchronize both bitmaps to this backup.
    380         Create an incremental backup through both bitmaps and verify that
    381         both backups match the current drive0 image.
    382         '''
    383 
    384         drive0 = self.drives[0]
    385         bitmap0 = self.add_bitmap('bitmap0', drive0)
    386         self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
    387                                           ('0xfe', '16M', '256k'),
    388                                           ('0x64', '32736k', '64k')))
    389         bitmap1 = self.add_bitmap('bitmap1', drive0)
    390 
    391         result = self.vm.qmp('transaction', actions=[
    392             transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name),
    393             transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name),
    394             transaction_drive_backup(drive0['id'], drive0['backup'],
    395                                      sync='full', format=drive0['fmt'])
    396         ])
    397         self.assert_qmp(result, 'return', {})
    398         self.wait_until_completed(drive0['id'])
    399         self.files.append(drive0['backup'])
    400 
    401         self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
    402                                           ('0x55', '8M', '352k'),
    403                                           ('0x78', '15872k', '1M')))
    404         # Both bitmaps should be correctly in sync.
    405         self.create_incremental(bitmap0)
    406         self.create_incremental(bitmap1)
    407         self.vm.shutdown()
    408         self.check_backups()
    409 
    410 
    411     def do_transaction_failure_test(self, race=False):
    412         # Create a second drive, with pattern:
    413         drive1 = self.add_node('drive1')
    414         self.img_create(drive1['file'], drive1['fmt'])
    415         io_write_patterns(drive1['file'], (('0x14', 0, 512),
    416                                            ('0x5d', '1M', '32k'),
    417                                            ('0xcd', '32M', '124k')))
    418 
    419         # Create a blkdebug interface to this img as 'drive1'
    420         result = self.vm.qmp('blockdev-add',
    421             node_name=drive1['id'],
    422             driver=drive1['fmt'],
    423             file={
    424                 'driver': 'blkdebug',
    425                 'image': {
    426                     'driver': 'file',
    427                     'filename': drive1['file']
    428                 },
    429                 'set-state': [{
    430                     'event': 'flush_to_disk',
    431                     'state': 1,
    432                     'new_state': 2
    433                 }],
    434                 'inject-error': [{
    435                     'event': 'read_aio',
    436                     'errno': 5,
    437                     'state': 2,
    438                     'immediately': False,
    439                     'once': True
    440                 }],
    441             }
    442         )
    443         self.assert_qmp(result, 'return', {})
    444 
    445         # Create bitmaps and full backups for both drives
    446         drive0 = self.drives[0]
    447         dr0bm0 = self.add_bitmap('bitmap0', drive0)
    448         dr1bm0 = self.add_bitmap('bitmap0', drive1)
    449         self.create_anchor_backup(drive0)
    450         self.create_anchor_backup(drive1)
    451         self.assert_no_active_block_jobs()
    452         self.assertFalse(self.vm.get_qmp_events(wait=False))
    453 
    454         # Emulate some writes
    455         if not race:
    456             self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
    457                                               ('0xfe', '16M', '256k'),
    458                                               ('0x64', '32736k', '64k')))
    459         self.hmp_io_writes(drive1['id'], (('0xba', 0, 512),
    460                                           ('0xef', '16M', '256k'),
    461                                           ('0x46', '32736k', '64k')))
    462 
    463         # Create incremental backup targets
    464         target0 = self.prepare_backup(dr0bm0)
    465         target1 = self.prepare_backup(dr1bm0)
    466 
    467         # Ask for a new incremental backup per-each drive,
    468         # expecting drive1's backup to fail. In the 'race' test,
    469         # we expect drive1 to attempt to cancel the empty drive0 job.
    470         transaction = [
    471             transaction_drive_backup(drive0['id'], target0, sync='incremental',
    472                                      format=drive0['fmt'], mode='existing',
    473                                      bitmap=dr0bm0.name),
    474             transaction_drive_backup(drive1['id'], target1, sync='incremental',
    475                                      format=drive1['fmt'], mode='existing',
    476                                      bitmap=dr1bm0.name)
    477         ]
    478         result = self.vm.qmp('transaction', actions=transaction,
    479                              properties={'completion-mode': 'grouped'} )
    480         self.assert_qmp(result, 'return', {})
    481 
    482         # Observe that drive0's backup is cancelled and drive1 completes with
    483         # an error.
    484         self.wait_qmp_backup_cancelled(drive0['id'])
    485         self.assertFalse(self.wait_qmp_backup(drive1['id']))
    486         error = self.vm.event_wait('BLOCK_JOB_ERROR')
    487         self.assert_qmp(error, 'data', {'device': drive1['id'],
    488                                         'action': 'report',
    489                                         'operation': 'read'})
    490         self.assertFalse(self.vm.get_qmp_events(wait=False))
    491         self.assert_no_active_block_jobs()
    492 
    493         # Delete drive0's successful target and eliminate our record of the
    494         # unsuccessful drive1 target.
    495         dr0bm0.del_target()
    496         dr1bm0.del_target()
    497         if race:
    498             # Don't re-run the transaction, we only wanted to test the race.
    499             self.vm.shutdown()
    500             return
    501 
    502         # Re-run the same transaction:
    503         target0 = self.prepare_backup(dr0bm0)
    504         target1 = self.prepare_backup(dr1bm0)
    505 
    506         # Re-run the exact same transaction.
    507         result = self.vm.qmp('transaction', actions=transaction,
    508                              properties={'completion-mode':'grouped'})
    509         self.assert_qmp(result, 'return', {})
    510 
    511         # Both should complete successfully this time.
    512         self.assertTrue(self.wait_qmp_backup(drive0['id']))
    513         self.assertTrue(self.wait_qmp_backup(drive1['id']))
    514         self.make_reference_backup(dr0bm0)
    515         self.make_reference_backup(dr1bm0)
    516         self.assertFalse(self.vm.get_qmp_events(wait=False))
    517         self.assert_no_active_block_jobs()
    518 
    519         # And the images should of course validate.
    520         self.vm.shutdown()
    521         self.check_backups()
    522 
    523     def test_transaction_failure(self):
    524         '''Test: Verify backups made from a transaction that partially fails.
    525 
    526         Add a second drive with its own unique pattern, and add a bitmap to each
    527         drive. Use blkdebug to interfere with the backup on just one drive and
    528         attempt to create a coherent incremental backup across both drives.
    529 
    530         verify a failure in one but not both, then delete the failed stubs and
    531         re-run the same transaction.
    532 
    533         verify that both incrementals are created successfully.
    534         '''
    535         self.do_transaction_failure_test()
    536 
    537     def test_transaction_failure_race(self):
    538         '''Test: Verify that transactions with jobs that have no data to
    539         transfer do not cause race conditions in the cancellation of the entire
    540         transaction job group.
    541         '''
    542         self.do_transaction_failure_test(race=True)
    543 
    544 
    545     def test_sync_dirty_bitmap_missing(self):
    546         self.assert_no_active_block_jobs()
    547         self.files.append(self.err_img)
    548         result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
    549                              sync='incremental', format=self.drives[0]['fmt'],
    550                              target=self.err_img)
    551         self.assert_qmp(result, 'error/class', 'GenericError')
    552 
    553 
    554     def test_sync_dirty_bitmap_not_found(self):
    555         self.assert_no_active_block_jobs()
    556         self.files.append(self.err_img)
    557         result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
    558                              sync='incremental', bitmap='unknown',
    559                              format=self.drives[0]['fmt'], target=self.err_img)
    560         self.assert_qmp(result, 'error/class', 'GenericError')
    561 
    562 
    563     def test_sync_dirty_bitmap_bad_granularity(self):
    564         '''
    565         Test: Test what happens if we provide an improper granularity.
    566 
    567         The granularity must always be a power of 2.
    568         '''
    569         self.assert_no_active_block_jobs()
    570         self.assertRaises(AssertionError, self.add_bitmap,
    571                           'bitmap0', self.drives[0],
    572                           granularity=64000)
    573 
    574     def test_growing_before_backup(self):
    575         '''
    576         Test: Add a bitmap, truncate the image, write past the old
    577               end, do a backup.
    578 
    579         Incremental backup should not ignore dirty bits past the old
    580         image end.
    581         '''
    582         self.assert_no_active_block_jobs()
    583 
    584         self.create_anchor_backup()
    585 
    586         self.add_bitmap('bitmap0', self.drives[0])
    587 
    588         res = self.vm.qmp('block_resize', device=self.drives[0]['id'],
    589                           size=(65 * 1048576))
    590         self.assert_qmp(res, 'return', {})
    591 
    592         # Dirty the image past the old end
    593         self.vm.hmp_qemu_io(self.drives[0]['id'], 'write 64M 64k')
    594 
    595         target = self.prepare_backup(size='65M')
    596         self.create_incremental(target=target)
    597 
    598         self.vm.shutdown()
    599         self.check_backups()
    600 
    601 
    602 class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase):
    603     '''Incremental backup tests that utilize a BlkDebug filter on drive0.'''
    604 
    605     def setUp(self):
    606         drive0 = self.add_node('drive0')
    607         self.img_create(drive0['file'], drive0['fmt'])
    608         self.write_default_pattern(drive0['file'])
    609         self.vm.launch()
    610 
    611     def test_incremental_failure(self):
    612         '''Test: Verify backups made after a failure are correct.
    613 
    614         Simulate a failure during an incremental backup block job,
    615         emulate additional writes, then create another incremental backup
    616         afterwards and verify that the backup created is correct.
    617         '''
    618 
    619         drive0 = self.drives[0]
    620         result = self.vm.qmp('blockdev-add',
    621             node_name=drive0['id'],
    622             driver=drive0['fmt'],
    623             file={
    624                 'driver': 'blkdebug',
    625                 'image': {
    626                     'driver': 'file',
    627                     'filename': drive0['file']
    628                 },
    629                 'set-state': [{
    630                     'event': 'flush_to_disk',
    631                     'state': 1,
    632                     'new_state': 2
    633                 }],
    634                 'inject-error': [{
    635                     'event': 'read_aio',
    636                     'errno': 5,
    637                     'state': 2,
    638                     'immediately': False,
    639                     'once': True
    640                 }],
    641             }
    642         )
    643         self.assert_qmp(result, 'return', {})
    644 
    645         self.create_anchor_backup(drive0)
    646         self.add_bitmap('bitmap0', drive0)
    647         # Note: at this point, during a normal execution,
    648         # Assume that the VM resumes and begins issuing IO requests here.
    649 
    650         self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
    651                                           ('0xfe', '16M', '256k'),
    652                                           ('0x64', '32736k', '64k')))
    653 
    654         result = self.create_incremental(validate=False)
    655         self.assertFalse(result)
    656         self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
    657                                           ('0x55', '8M', '352k'),
    658                                           ('0x78', '15872k', '1M')))
    659         self.create_incremental()
    660         self.vm.shutdown()
    661         self.check_backups()
    662 
    663     def test_incremental_pause(self):
    664         """
    665         Test an incremental backup that errors into a pause and is resumed.
    666         """
    667 
    668         drive0 = self.drives[0]
    669         # NB: The blkdebug script here looks for a "flush, read" pattern.
    670         # The flush occurs in hmp_io_writes, and the read during the block job.
    671         result = self.vm.qmp('blockdev-add',
    672                              node_name=drive0['id'],
    673                              driver=drive0['fmt'],
    674                              file={
    675                                  'driver': 'blkdebug',
    676                                  'image': {
    677                                      'driver': 'file',
    678                                      'filename': drive0['file']
    679                                  },
    680                                  'set-state': [{
    681                                      'event': 'flush_to_disk',
    682                                      'state': 1,
    683                                      'new_state': 2
    684                                  }],
    685                                  'inject-error': [{
    686                                      'event': 'read_aio',
    687                                      'errno': 5,
    688                                      'state': 2,
    689                                      'immediately': False,
    690                                      'once': True
    691                                  }],
    692                              })
    693         self.assert_qmp(result, 'return', {})
    694         self.create_anchor_backup(drive0)
    695         bitmap = self.add_bitmap('bitmap0', drive0)
    696 
    697         # Emulate guest activity
    698         self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
    699                                           ('0xfe', '16M', '256k'),
    700                                           ('0x64', '32736k', '64k')))
    701 
    702         # Bitmap Status Check
    703         self.assertTrue(self.vm.check_bitmap_status(
    704             drive0['id'], bitmap.name, {
    705                 'count': 458752,
    706                 'granularity': 65536,
    707                 'busy': False,
    708                 'recording': True
    709             }))
    710 
    711         # Start backup
    712         parent, _ = bitmap.last_target()
    713         target = self.prepare_backup(bitmap, parent)
    714         res = self.vm.qmp('drive-backup',
    715                           job_id=bitmap.drive['id'],
    716                           device=bitmap.drive['id'],
    717                           sync='incremental',
    718                           bitmap=bitmap.name,
    719                           format=bitmap.drive['fmt'],
    720                           target=target,
    721                           mode='existing',
    722                           on_source_error='stop')
    723         self.assert_qmp(res, 'return', {})
    724 
    725         # Wait for the error
    726         event = self.vm.event_wait(name="BLOCK_JOB_ERROR",
    727                                    match={"data":{"device":bitmap.drive['id']}})
    728         self.assert_qmp(event, 'data', {'device': bitmap.drive['id'],
    729                                         'action': 'stop',
    730                                         'operation': 'read'})
    731 
    732         # Bitmap Status Check
    733         self.assertTrue(self.vm.check_bitmap_status(
    734             drive0['id'], bitmap.name, {
    735                 'count': 458752,
    736                 'granularity': 65536,
    737                 'busy': True,
    738                 'recording': True
    739             }))
    740 
    741         # Resume and check incremental backup for consistency
    742         res = self.vm.qmp('block-job-resume', device=bitmap.drive['id'])
    743         self.assert_qmp(res, 'return', {})
    744         self.wait_qmp_backup(bitmap.drive['id'])
    745 
    746         # Bitmap Status Check
    747         self.assertTrue(self.vm.check_bitmap_status(
    748             drive0['id'], bitmap.name, {
    749                 'count': 0,
    750                 'granularity': 65536,
    751                 'busy': False,
    752                 'recording': True
    753             }))
    754 
    755         # Finalize / Cleanup
    756         self.make_reference_backup(bitmap)
    757         self.vm.shutdown()
    758         self.check_backups()
    759 
    760 
    761 if __name__ == '__main__':
    762     iotests.main(supported_fmts=['qcow2'],
    763                  supported_protocols=['file'])