aboutsummaryrefslogtreecommitdiff
path: root/docs/interop/bitmaps.rst
blob: c29ac4a85482282a1474418d61b82762f25a205f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
..
   Copyright 2019 John Snow <jsnow@redhat.com> and Red Hat, Inc.
   All rights reserved.

   This file is licensed via The FreeBSD Documentation License, the full
   text of which is included at the end of this document.

====================================
Dirty Bitmaps and Incremental Backup
====================================

Dirty Bitmaps are in-memory objects that track writes to block devices. They
can be used in conjunction with various block job operations to perform
incremental or differential backup regimens.

This document explains the conceptual mechanisms, as well as up-to-date,
complete and comprehensive documentation on the API to manipulate them.
(Hopefully, the "why", "what", and "how".)

The intended audience for this document is developers who are adding QEMU
backup features to management applications, or power users who run and
administer QEMU directly via QMP.

.. contents::

Overview
--------

Bitmaps are bit vectors where each '1' bit in the vector indicates a modified
("dirty") segment of the corresponding block device. The size of the segment
that is tracked is the granularity of the bitmap. If the granularity of a
bitmap is 64K, each '1' bit means that a 64K region as a whole may have
changed in some way, possibly by as little as one byte.

Smaller granularities mean more accurate tracking of modified disk data, but
requires more computational overhead and larger bitmap sizes. Larger
granularities mean smaller bitmap sizes, but less targeted backups.

The size of a bitmap (in bytes) can be computed as such:
    ``size`` = ceil(ceil(``image_size`` / ``granularity``) / 8)

e.g. the size of a 64KiB granularity bitmap on a 2TiB image is:
    ``size`` = ((2147483648K / 64K) / 8)
         = 4194304B = 4MiB.

QEMU uses these bitmaps when making incremental backups to know which sections
of the file to copy out. They are not enabled by default and must be
explicitly added in order to begin tracking writes.

Bitmaps can be created at any time and can be attached to any arbitrary block
node in the storage graph, but are most useful conceptually when attached to
the root node attached to the guest's storage device model.

That is to say: It's likely most useful to track the guest's writes to disk,
but you could theoretically track things like qcow2 metadata changes by
attaching the bitmap elsewhere in the storage graph. This is beyond the scope
of this document.

QEMU supports persisting these bitmaps to disk via the qcow2 image format.
Bitmaps which are stored or loaded in this way are called "persistent",
whereas bitmaps that are not are called "transient".

QEMU also supports the migration of both transient bitmaps (tracking any
arbitrary image format) or persistent bitmaps (qcow2) via live migration.

Supported Image Formats
-----------------------

QEMU supports all documented features below on the qcow2 image format.

However, qcow2 is only strictly necessary for the persistence feature, which
writes bitmap data to disk upon close. If persistence is not required for a
specific use case, all bitmap features excepting persistence are available for
any arbitrary image format.

For example, Dirty Bitmaps can be combined with the 'raw' image format, but
any changes to the bitmap will be discarded upon exit.

.. warning:: Transient bitmaps will not be saved on QEMU exit! Persistent
             bitmaps are available only on qcow2 images.

Dirty Bitmap Names
------------------

Bitmap objects need a method to reference them in the API. All API-created and
managed bitmaps have a human-readable name chosen by the user at creation
time.

- A bitmap's name is unique to the node, but bitmaps attached to different
  nodes can share the same name. Therefore, all bitmaps are addressed via
  their (node, name) pair.

- The name of a user-created bitmap cannot be empty ("").

- Transient bitmaps can have JSON unicode names that are effectively not
  length limited. (QMP protocol may restrict messages to less than 64MiB.)

- Persistent storage formats may impose their own requirements on bitmap names
  and namespaces. Presently, only qcow2 supports persistent bitmaps. See
  docs/interop/qcow2.txt for more details on restrictions. Notably:

   - qcow2 bitmap names are limited to between 1 and 1023 bytes long.

   - No two bitmaps saved to the same qcow2 file may share the same name.

- QEMU occasionally uses bitmaps for internal use which have no name. They are
  hidden from API query calls, cannot be manipulated by the external API, are
  never persistent, nor ever migrated.

Bitmap Status
-------------

Dirty Bitmap objects can be queried with the QMP command `query-block
<qemu-qmp-ref.html#index-query_002dblock>`_, and are visible via the
`BlockDirtyInfo <qemu-qmp-ref.html#index-BlockDirtyInfo>`_ QAPI structure.

This struct shows the name, granularity, and dirty byte count for each bitmap.
Additionally, it shows several boolean status indicators:

- ``recording``: This bitmap is recording writes.
- ``busy``: This bitmap is in-use by an operation.
- ``persistent``: This bitmap is a persistent type.
- ``inconsistent``: This bitmap is corrupted and cannot be used.

The ``+busy`` status prohibits you from deleting, clearing, or otherwise
modifying a bitmap, and happens when the bitmap is being used for a backup
operation or is in the process of being loaded from a migration. Many of the
commands documented below will refuse to work on such bitmaps.

The ``+inconsistent`` status similarly prohibits almost all operations,
notably allowing only the ``block-dirty-bitmap-remove`` operation.

There is also a deprecated ``status`` field of type `DirtyBitmapStatus
<qemu-qmp-ref.html#index-DirtyBitmapStatus>`_. A bitmap historically had
five visible states:

   #. ``Frozen``: This bitmap is currently in-use by an operation and is
      immutable. It can't be deleted, renamed, reset, etc.

      (This is now ``+busy``.)

   #. ``Disabled``: This bitmap is not recording new writes.

      (This is now ``-recording -busy``.)

   #. ``Active``: This bitmap is recording new writes.

      (This is now ``+recording -busy``.)

   #. ``Locked``: This bitmap is in-use by an operation, and is immutable.
      The difference from "Frozen" was primarily implementation details.

      (This is now ``+busy``.)

   #. ``Inconsistent``: This persistent bitmap was not saved to disk
      correctly, and can no longer be used. It remains in memory to serve as
      an indicator of failure.

      (This is now ``+inconsistent``.)

These states are directly replaced by the status indicators and should not be
used. The difference between ``Frozen`` and ``Locked`` is an implementation
detail and should not be relevant to external users.

Basic QMP Usage
---------------

The primary interface to manipulating bitmap objects is via the QMP
interface. If you are not familiar, see docs/interop/qmp-intro.txt for a broad
overview, and `qemu-qmp-ref <qemu-qmp-ref.html>`_ for a full reference of all
QMP commands.

Supported Commands
~~~~~~~~~~~~~~~~~~

There are six primary bitmap-management API commands:

- ``block-dirty-bitmap-add``
- ``block-dirty-bitmap-remove``
- ``block-dirty-bitmap-clear``
- ``block-dirty-bitmap-disable``
- ``block-dirty-bitmap-enable``
- ``block-dirty-bitmap-merge``

And one related query command:

- ``query-block``

Creation: block-dirty-bitmap-add
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

`block-dirty-bitmap-add
<qemu-qmp-ref.html#index-block_002ddirty_002dbitmap_002dadd>`_:

Creates a new bitmap that tracks writes to the specified node. granularity,
persistence, and recording state can be adjusted at creation time.

.. admonition:: Example

 to create a new, actively recording persistent bitmap:

 .. code:: json

  -> { "execute": "block-dirty-bitmap-add",
       "arguments": {
         "node": "drive0",
         "name": "bitmap0",
         "persistent": true,
       }
     }

  <- { "return": {} }

- This bitmap will have a default granularity that matches the cluster size of
  its associated drive, if available, clamped to between [4KiB, 64KiB]. The
  current default for qcow2 is 64KiB.

.. admonition:: Example

 To create a new, disabled (``-recording``), transient bitmap that tracks
 changes in 32KiB segments:

 .. code:: json

  -> { "execute": "block-dirty-bitmap-add",
       "arguments": {
         "node": "drive0",
         "name": "bitmap1",
         "granularity": 32768,
         "disabled": true
       }
     }

  <- { "return": {} }

Deletion: block-dirty-bitmap-remove
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

`block-dirty-bitmap-remove
<qemu-qmp-ref.html#index-block_002ddirty_002dbitmap_002dremove>`_:

Deletes a bitmap. Bitmaps that are ``+busy`` cannot be removed.

- Deleting a bitmap does not impact any other bitmaps attached to the same
  node, nor does it affect any backups already created from this bitmap or
  node.

- Because bitmaps are only unique to the node to which they are attached, you
  must specify the node/drive name here, too.

- Deleting a persistent bitmap will remove it from the qcow2 file.

.. admonition:: Example

 Remove a bitmap named ``bitmap0`` from node ``drive0``:

 .. code:: json

  -> { "execute": "block-dirty-bitmap-remove",
       "arguments": {
         "node": "drive0",
         "name": "bitmap0"
       }
     }

  <- { "return": {} }

Resetting: block-dirty-bitmap-clear
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

`block-dirty-bitmap-clear
<qemu-qmp-ref.html#index-block_002ddirty_002dbitmap_002dclear>`_:

Clears all dirty bits from a bitmap. ``+busy`` bitmaps cannot be cleared.

- An incremental backup created from an empty bitmap will copy no data, as if
  nothing has changed.

.. admonition:: Example

 Clear all dirty bits from bitmap ``bitmap0`` on node ``drive0``:

 .. code:: json

  -> { "execute": "block-dirty-bitmap-clear",
       "arguments": {
         "node": "drive0",
         "name": "bitmap0"
       }
     }

  <- { "return": {} }

Enabling: block-dirty-bitmap-enable
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

`block-dirty-bitmap-enable
<qemu-qmp-ref.html#index-block_002ddirty_002dbitmap_002denable>`_:

"Enables" a bitmap, setting the ``recording`` bit to true, causing writes to
begin being recorded. ``+busy`` bitmaps cannot be enabled.

- Bitmaps default to being enabled when created, unless configured otherwise.

- Persistent enabled bitmaps will remember their ``+recording`` status on
  load.

.. admonition:: Example

 To set ``+recording`` on bitmap ``bitmap0`` on node ``drive0``:

 .. code:: json

  -> { "execute": "block-dirty-bitmap-enable",
       "arguments": {
         "node": "drive0",
         "name": "bitmap0"
       }
     }

  <- { "return": {} }

Enabling: block-dirty-bitmap-disable
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

`block-dirty-bitmap-disable
<qemu-qmp-ref.html#index-block_002ddirty_002dbitmap_002ddisable>`_:

"Disables" a bitmap, setting the ``recording`` bit to false, causing further
writes to begin being ignored. ``+busy`` bitmaps cannot be disabled.

.. warning::

  This is potentially dangerous: QEMU makes no effort to stop any writes if
  there are disabled bitmaps on a node, and will not mark any disabled bitmaps
  as ``+inconsistent`` if any such writes do happen. Backups made from such
  bitmaps will not be able to be used to reconstruct a coherent image.

- Disabling a bitmap may be useful for examining which sectors of a disk
  changed during a specific time period, or for explicit management of
  differential backup windows.

- Persistent disabled bitmaps will remember their ``-recording`` status on
  load.

.. admonition:: Example

 To set ``-recording`` on bitmap ``bitmap0`` on node ``drive0``:

 .. code:: json

  -> { "execute": "block-dirty-bitmap-disable",
       "arguments": {
         "node": "drive0",
         "name": "bitmap0"
       }
     }

  <- { "return": {} }

Merging, Copying: block-dirty-bitmap-merge
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

`block-dirty-bitmap-merge
<qemu-qmp-ref.html#index-block_002ddirty_002dbitmap_002dmerge>`_:

Merges one or more bitmaps into a target bitmap. For any segment that is dirty
in any one source bitmap, the target bitmap will mark that segment dirty.

- Merge takes one or more bitmaps as a source and merges them together into a
  single destination, such that any segment marked as dirty in any source
  bitmap(s) will be marked dirty in the destination bitmap.

- Merge does not create the destination bitmap if it does not exist. A blank
  bitmap can be created beforehand to achieve the same effect.

- The destination is not cleared prior to merge, so subsequent merge
  operations will continue to cumulatively mark more segments as dirty.

- If the merge operation should fail, the destination bitmap is guaranteed to
  be unmodified. The operation may fail if the source or destination bitmaps
  are busy, or have different granularities.

- Bitmaps can only be merged on the same node. There is only one "node"
  argument, so all bitmaps must be attached to that same node.

- Copy can be achieved by merging from a single source to an empty
  destination.

.. admonition:: Example

 Merge the data from ``bitmap0`` into the bitmap ``new_bitmap`` on node
 ``drive0``. If ``new_bitmap`` was empty prior to this command, this achieves
 a copy.

 .. code:: json

  -> { "execute": "block-dirty-bitmap-merge",
       "arguments": {
         "node": "drive0",
         "target": "new_bitmap",
         "bitmaps": [ "bitmap0" ]
       }
     }

  <- { "return": {} }

Querying: query-block
~~~~~~~~~~~~~~~~~~~~~

`query-block
<qemu-qmp-ref.html#index-query_002dblock>`_:

Not strictly a bitmaps command, but will return information about any bitmaps
attached to nodes serving as the root for guest devices.

- The "inconsistent" bit will not appear when it is false, appearing only when
  the value is true to indicate there is a problem.

.. admonition:: Example

 Query the block sub-system of QEMU. The following json has trimmed irrelevant
 keys from the response to highlight only the bitmap-relevant portions of the
 API. This result highlights a bitmap ``bitmap0`` attached to the root node of
 device ``drive0``.

 .. code:: json

  -> {
       "execute": "query-block",
       "arguments": {}
     }

  <- {
       "return": [ {
         "dirty-bitmaps": [ {
           "status": "active",
           "count": 0,
           "busy": false,
           "name": "bitmap0",
           "persistent": false,
           "recording": true,
           "granularity": 65536
         } ],
         "device": "drive0",
       } ]
     }

Bitmap Persistence
------------------

As outlined in `Supported Image Formats`_, QEMU can persist bitmaps to qcow2
files. Demonstrated in `Creation: block-dirty-bitmap-add`_, passing
``persistent: true`` to ``block-dirty-bitmap-add`` will persist that bitmap to
disk.

Persistent bitmaps will be automatically loaded into memory upon load, and
will be written back to disk upon close. Their usage should be mostly
transparent.

However, if QEMU does not get a chance to close the file cleanly, the bitmap
will be marked as ``+inconsistent`` at next load and considered unsafe to use
for any operation. At this point, the only valid operation on such bitmaps is
``block-dirty-bitmap-remove``.

Losing a bitmap in this way does not invalidate any existing backups that have
been made from this bitmap, but no further backups will be able to be issued
for this chain.

Transactions
------------

Transactions are a QMP feature that allows you to submit multiple QMP commands
at once, being guaranteed that they will all succeed or fail atomically,
together. The interaction of bitmaps and transactions are demonstrated below.

See `transaction <qemu-qmp.ref.html#index-transaction>`_ in the QMP reference
for more details.

Justification
~~~~~~~~~~~~~

Bitmaps can generally be modified at any time, but certain operations often
only make sense when paired directly with other commands. When a VM is paused,
it's easy to ensure that no guest writes occur between individual QMP
commands. When a VM is running, this is difficult to accomplish with
individual QMP commands that may allow guest writes to occur inbetween each
command.

For example, using only individual QMP commands, we could:

#. Boot the VM in a paused state.
#. Create a full drive backup of drive0.
#. Create a new bitmap attached to drive0, confident that nothing has been
   written to drive0 in the meantime.
#. Resume execution of the VM.
#. At a later point, issue incremental backups from ``bitmap0``.

At this point, the bitmap and drive backup would be correctly in sync, and
incremental backups made from this point forward would be correctly aligned to
the full drive backup.

This is not particularly useful if we decide we want to start incremental
backups after the VM has been running for a while, for which we would want to
perform actions such as the following:

#. Boot the VM and begin execution.
#. Using a single transaction, perform the following operations:

   -  Create ``bitmap0``.
   -  Create a full drive backup of ``drive0``.

#. At a later point, issue incremental backups from ``bitmap0``.

.. note:: As a consideration, if ``bitmap0`` is created prior to the full
          drive backup, incremental backups can still be authored from this
          bitmap, but they will copy extra segments reflecting writes that
          occurred prior to the backup operation. Transactions allow us to
          narrow critical points in time to reduce waste, or, in the other
          direction, to ensure that no segments are omitted.

Supported Bitmap Transactions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

-  ``block-dirty-bitmap-add``
-  ``block-dirty-bitmap-clear``
-  ``block-dirty-bitmap-enable``
-  ``block-dirty-bitmap-disable``
-  ``block-dirty-bitmap-merge``

The usages for these commands are identical to their respective QMP commands,
but see the sections below for concrete examples.

Incremental Backups - Push Model
--------------------------------

Incremental backups are simply partial disk images that can be combined with
other partial disk images on top of a base image to reconstruct a full backup
from the point in time at which the incremental backup was issued.

The "Push Model" here references the fact that QEMU is "pushing" the modified
blocks out to a destination. We will be using the `drive-backup
<qemu-qmp-ref.html#index-drive_002dbackup>`_ and `blockdev-backup
<qemu-qmp-ref.html#index-blockdev_002dbackup>`_ QMP commands to create both
full and incremental backups.

Both of these commands are jobs, which have their own QMP API for querying and
management documented in `Background jobs
<qemu-qmp-ref.html#Background-jobs>`_.

Example: New Incremental Backup Anchor Point
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

As outlined in the Transactions - `Justification`_ section, perhaps we want to
create a new incremental backup chain attached to a drive.

This example creates a new, full backup of "drive0" and accompanies it with a
new, empty bitmap that records writes from this point in time forward.

.. note:: Any new writes that happen after this command is issued, even while
          the backup job runs, will be written locally and not to the backup
          destination. These writes will be recorded in the bitmap
          accordingly.

.. code:: json

  -> {
       "execute": "transaction",
       "arguments": {
         "actions": [
           {
             "type": "block-dirty-bitmap-add",
             "data": {
               "node": "drive0",
               "name": "bitmap0"
             }
           },
           {
             "type": "drive-backup",
             "data": {
               "device": "drive0",
               "target": "/path/to/drive0.full.qcow2",
               "sync": "full",
               "format": "qcow2"
             }
           }
         ]
       }
     }

  <- { "return": {} }

  <- {
       "timestamp": {
         "seconds": 1555436945,
         "microseconds": 179620
       },
       "data": {
         "status": "created",
         "id": "drive0"
       },
       "event": "JOB_STATUS_CHANGE"
     }

  ...

  <- {
       "timestamp": {...},
       "data": {
         "device": "drive0",
         "type": "backup",
         "speed": 0,
         "len": 68719476736,
         "offset": 68719476736
       },
       "event": "BLOCK_JOB_COMPLETED"
     }

  <- {
       "timestamp": {...},
       "data": {
         "status": "concluded",
         "id": "drive0"
       },
       "event": "JOB_STATUS_CHANGE"
     }

  <- {
       "timestamp": {...},
       "data": {
         "status": "null",
         "id": "drive0"
       },
       "event": "JOB_STATUS_CHANGE"
     }

A full explanation of the job transition semantics and the JOB_STATUS_CHANGE
event are beyond the scope of this document and will be omitted in all
subsequent examples; above, several more events have been omitted for brevity.

.. note:: Subsequent examples will omit all events except BLOCK_JOB_COMPLETED
          except where necessary to illustrate workflow differences.

          Omitted events and json objects will be represented by ellipses:
          ``...``

Example: Resetting an Incremental Backup Anchor Point
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

If we want to start a new backup chain with an existing bitmap, we can also
use a transaction to reset the bitmap while making a new full backup:

.. code:: json

  -> {
       "execute": "transaction",
       "arguments": {
         "actions": [
         {
           "type": "block-dirty-bitmap-clear",
           "data": {
             "node": "drive0",
             "name": "bitmap0"
           }
         },
         {
           "type": "drive-backup",
           "data": {
             "device": "drive0",
             "target": "/path/to/drive0.new_full.qcow2",
             "sync": "full",
             "format": "qcow2"
           }
         }
       ]
     }
   }

  <- { "return": {} }

  ...

  <- {
       "timestamp": {...},
       "data": {
         "device": "drive0",
         "type": "backup",
         "speed": 0,
         "len": 68719476736,
         "offset": 68719476736
       },
       "event": "BLOCK_JOB_COMPLETED"
     }

  ...

The result of this example is identical to the first, but we clear an existing
bitmap instead of adding a new one.

.. tip:: In both of these examples, "bitmap0" is tied conceptually to the
         creation of new, full backups. This relationship is not saved or
         remembered by QEMU; it is up to the operator or management layer to
         remember which bitmaps are associated with which backups.

Example: First Incremental Backup
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

#. Create a full backup and sync it to a dirty bitmap using any method:

   - Either of the two live backup method demonstrated above,
   - Using QMP commands with the VM paused as in the `Justification`_ section,
     or
   - With the VM offline, manually copy the image and start the VM in a paused
     state, careful to add a new bitmap before the VM begins execution.

   Whichever method is chosen, let's assume that at the end of this step:

   - The full backup is named ``drive0.full.qcow2``.
   - The bitmap we created is named ``bitmap0``, attached to ``drive0``.

#. Create a destination image for the incremental backup that utilizes the
   full backup as a backing image.

   - Let's assume the new incremental image is named ``drive0.inc0.qcow2``:

   .. code:: bash

       $ qemu-img create -f qcow2 drive0.inc0.qcow2 \
         -b drive0.full.qcow2 -F qcow2

#. Issue an incremental backup command:

   .. code:: json

    -> {
         "execute": "drive-backup",
         "arguments": {
           "device": "drive0",
           "bitmap": "bitmap0",
           "target": "drive0.inc0.qcow2",
           "format": "qcow2",
           "sync": "incremental",
           "mode": "existing"
         }
       }

    <- { "return": {} }

    ...

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive0",
           "type": "backup",
           "speed": 0,
           "len": 68719476736,
           "offset": 68719476736
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

    ...

This copies any blocks modified since the full backup was created into the
``drive0.inc0.qcow2`` file. During the operation, ``bitmap0`` is marked
``+busy``. If the operation is successful, ``bitmap0`` will be cleared to
reflect the "incremental" backup regimen, which only copies out new changes
from each incremental backup.

.. note:: Any new writes that occur after the backup operation starts do not
          get copied to the destination. The backup's "point in time" is when
          the backup starts, not when it ends. These writes are recorded in a
          special bitmap that gets re-added to bitmap0 when the backup ends so
          that the next incremental backup can copy them out.

Example: Second Incremental Backup
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

#. Create a new destination image for the incremental backup that points to
   the previous one, e.g.: ``drive0.inc1.qcow2``

   .. code:: bash

       $ qemu-img create -f qcow2 drive0.inc1.qcow2 \
         -b drive0.inc0.qcow2 -F qcow2

#. Issue a new incremental backup command. The only difference here is that we
   have changed the target image below.

   .. code:: json

    -> {
         "execute": "drive-backup",
         "arguments": {
           "device": "drive0",
           "bitmap": "bitmap0",
           "target": "drive0.inc1.qcow2",
           "format": "qcow2",
           "sync": "incremental",
           "mode": "existing"
         }
       }

    <- { "return": {} }

    ...

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive0",
           "type": "backup",
           "speed": 0,
           "len": 68719476736,
           "offset": 68719476736
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

    ...

Because the first incremental backup from the previous example completed
successfully, ``bitmap0`` was synchronized with ``drive0.inc0.qcow2``. Here,
we use ``bitmap0`` again to create a new incremental backup that targets the
previous one, creating a chain of three images:

.. admonition:: Diagram

 .. code:: text

   +-------------------+   +-------------------+   +-------------------+
   | drive0.full.qcow2 |<--| drive0.inc0.qcow2 |<--| drive0.inc1.qcow2 |
   +-------------------+   +-------------------+   +-------------------+

Each new incremental backup re-synchronizes the bitmap to the latest backup
authored, allowing a user to continue to "consume" it to create new backups on
top of an existing chain.

In the above diagram, neither drive0.inc1.qcow2 nor drive0.inc0.qcow2 are
complete images by themselves, but rely on their backing chain to reconstruct
a full image. The dependency terminates with each full backup.

Each backup in this chain remains independent, and is unchanged by new entries
made later in the chain. For instance, drive0.inc0.qcow2 remains a perfectly
valid backup of the disk as it was when that backup was issued.

Example: Incremental Push Backups without Backing Files
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Backup images are best kept off-site, so we often will not have the preceding
backups in a chain available to link against. This is not a problem at backup
time; we simply do not set the backing image when creating the destination
image:

#. Create a new destination image with no backing file set. We will need to
   specify the size of the base image, because the backing file isn't
   available for QEMU to use to determine it.

   .. code:: bash

       $ qemu-img create -f qcow2 drive0.inc2.qcow2 64G

   .. note:: Alternatively, you can omit ``mode: "existing"`` from the push
             backup commands to have QEMU create an image without a backing
             file for you, but you lose control over format options like
             compatibility and preallocation presets.

#. Issue a new incremental backup command. Apart from the new destination
   image, there is no difference from the last two examples.

   .. code:: json

    -> {
         "execute": "drive-backup",
         "arguments": {
           "device": "drive0",
           "bitmap": "bitmap0",
           "target": "drive0.inc2.qcow2",
           "format": "qcow2",
           "sync": "incremental",
           "mode": "existing"
         }
       }

    <- { "return": {} }

    ...

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive0",
           "type": "backup",
           "speed": 0,
           "len": 68719476736,
           "offset": 68719476736
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

    ...

The only difference from the perspective of the user is that you will need to
set the backing image when attempting to restore the backup:

.. code:: bash

    $ qemu-img rebase drive0.inc2.qcow2 \
      -u -b drive0.inc1.qcow2

This uses the "unsafe" rebase mode to simply set the backing file to a file
that isn't present.

It is also possible to use ``--image-opts`` to specify the entire backing
chain by hand as an ephemeral property at runtime, but that is beyond the
scope of this document.

Example: Multi-drive Incremental Backup
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Assume we have a VM with two drives, "drive0" and "drive1" and we wish to back
both of them up such that the two backups represent the same crash-consistent
point in time.

#. For each drive, create an empty image:

   .. code:: bash

    $ qemu-img create -f qcow2 drive0.full.qcow2 64G
    $ qemu-img create -f qcow2 drive1.full.qcow2 64G

#. Create a full (anchor) backup for each drive, with accompanying bitmaps:

   .. code:: json

    -> {
         "execute": "transaction",
         "arguments": {
           "actions": [
             {
               "type": "block-dirty-bitmap-add",
               "data": {
                 "node": "drive0",
                 "name": "bitmap0"
               }
             },
             {
               "type": "block-dirty-bitmap-add",
               "data": {
                 "node": "drive1",
                 "name": "bitmap0"
               }
             },
             {
               "type": "drive-backup",
               "data": {
                 "device": "drive0",
                 "target": "/path/to/drive0.full.qcow2",
                 "sync": "full",
                 "format": "qcow2"
               }
             },
             {
               "type": "drive-backup",
               "data": {
                 "device": "drive1",
                 "target": "/path/to/drive1.full.qcow2",
                 "sync": "full",
                 "format": "qcow2"
               }
             }
           ]
         }
       }

    <- { "return": {} }

    ...

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive0",
           "type": "backup",
           "speed": 0,
           "len": 68719476736,
           "offset": 68719476736
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

    ...

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive1",
           "type": "backup",
           "speed": 0,
           "len": 68719476736,
           "offset": 68719476736
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

    ...

#. Later, create new destination images for each of the incremental backups
   that point to their respective full backups:

   .. code:: bash

     $ qemu-img create -f qcow2 drive0.inc0.qcow2 \
       -b drive0.full.qcow2 -F qcow2
     $ qemu-img create -f qcow2 drive1.inc0.qcow2 \
       -b drive1.full.qcow2 -F qcow2

#. Issue a multi-drive incremental push backup transaction:

   .. code:: json

    -> {
         "execute": "transaction",
         "arguments": {
           "actions": [
             {
               "type": "drive-backup",
               "data": {
                 "device": "drive0",
                 "bitmap": "bitmap0",
                 "format": "qcow2",
                 "mode": "existing",
                 "sync": "incremental",
                 "target": "drive0.inc0.qcow2"
               }
             },
             {
               "type": "drive-backup",
               "data": {
                 "device": "drive1",
                 "bitmap": "bitmap0",
                 "format": "qcow2",
                 "mode": "existing",
                 "sync": "incremental",
                 "target": "drive1.inc0.qcow2"
               }
             },
           ]
         }
       }

    <- { "return": {} }

    ...

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive0",
           "type": "backup",
           "speed": 0,
           "len": 68719476736,
           "offset": 68719476736
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

    ...

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive1",
           "type": "backup",
           "speed": 0,
           "len": 68719476736,
           "offset": 68719476736
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

    ...

Push Backup Errors & Recovery
-----------------------------

In the event of an error that occurs after a push backup job is successfully
launched, either by an individual QMP command or a QMP transaction, the user
will receive a ``BLOCK_JOB_COMPLETE`` event with a failure message,
accompanied by a ``BLOCK_JOB_ERROR`` event.

In the case of a job being cancelled, the user will receive a
``BLOCK_JOB_CANCELLED`` event instead of a pair of COMPLETE and ERROR
events.

In either failure case, the bitmap used for the failed operation is not
cleared. It will contain all of the dirty bits it did at the start of the
operation, plus any new bits that got marked during the operation.

Effectively, the "point in time" that a bitmap is recording differences
against is kept at the issuance of the last successful incremental backup,
instead of being moved forward to the start of this now-failed backup.

Once the underlying problem is addressed (e.g. more storage space is allocated
on the destination), the incremental backup command can be retried with the
same bitmap.

Example: Individual Failures
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Incremental Push Backup jobs that fail individually behave simply as
described above. This example demonstrates the single-job failure case:

#. Create a target image:

   .. code:: bash

       $ qemu-img create -f qcow2 drive0.inc0.qcow2 \
         -b drive0.full.qcow2 -F qcow2

#. Attempt to create an incremental backup via QMP:

   .. code:: json

    -> {
         "execute": "drive-backup",
         "arguments": {
           "device": "drive0",
           "bitmap": "bitmap0",
           "target": "drive0.inc0.qcow2",
           "format": "qcow2",
           "sync": "incremental",
           "mode": "existing"
         }
       }

    <- { "return": {} }

#. Receive a pair of events indicating failure:

   .. code:: json

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive0",
           "action": "report",
           "operation": "write"
         },
         "event": "BLOCK_JOB_ERROR"
       }

    <- {
         "timestamp": {...},
         "data": {
           "speed": 0,
           "offset": 0,
           "len": 67108864,
           "error": "No space left on device",
           "device": "drive0",
           "type": "backup"
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

#. Delete the failed image, and re-create it.

   .. code:: bash

       $ rm drive0.inc0.qcow2
       $ qemu-img create -f qcow2 drive0.inc0.qcow2 \
         -b drive0.full.qcow2 -F qcow2

#. Retry the command after fixing the underlying problem, such as
   freeing up space on the backup volume:

   .. code:: json

    -> {
         "execute": "drive-backup",
         "arguments": {
           "device": "drive0",
           "bitmap": "bitmap0",
           "target": "drive0.inc0.qcow2",
           "format": "qcow2",
           "sync": "incremental",
           "mode": "existing"
         }
       }

    <- { "return": {} }

#. Receive confirmation that the job completed successfully:

   .. code:: json

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive0",
           "type": "backup",
           "speed": 0,
           "len": 67108864,
           "offset": 67108864
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

Example: Partial Transactional Failures
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

QMP commands like `drive-backup <qemu-qmp-ref.html#index-drive_002dbackup>`_
conceptually only start a job, and so transactions containing these commands
may succeed even if the job it created later fails. This might have surprising
interactions with notions of how a "transaction" ought to behave.

This distinction means that on occasion, a transaction containing such job
launching commands may appear to succeed and return success, but later
individual jobs associated with the transaction may fail. It is possible that
a management application may have to deal with a partial backup failure after
a "successful" transaction.

If multiple backup jobs are specified in a single transaction, if one of those
jobs fails, it will not interact with the other backup jobs in any way by
default. The job(s) that succeeded will clear the dirty bitmap associated with
the operation, but the job(s) that failed will not. It is therefore not safe
to delete any incremental backups that were created successfully in this
scenario, even though others failed.

This example illustrates a transaction with two backup jobs, where one fails
and one succeeds:

#. Issue the transaction to start a backup of both drives.

   .. code:: json

    -> {
         "execute": "transaction",
         "arguments": {
           "actions": [
           {
             "type": "drive-backup",
             "data": {
               "device": "drive0",
               "bitmap": "bitmap0",
               "format": "qcow2",
               "mode": "existing",
               "sync": "incremental",
               "target": "drive0.inc0.qcow2"
             }
           },
           {
             "type": "drive-backup",
             "data": {
               "device": "drive1",
               "bitmap": "bitmap0",
               "format": "qcow2",
               "mode": "existing",
               "sync": "incremental",
               "target": "drive1.inc0.qcow2"
             }
           }]
         }
       }

#. Receive notice that the Transaction was accepted, and jobs were
   launched:

   .. code:: json

    <- { "return": {} }

#. Receive notice that the first job has completed:

   .. code:: json

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive0",
           "type": "backup",
           "speed": 0,
           "len": 67108864,
           "offset": 67108864
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

#. Receive notice that the second job has failed:

   .. code:: json

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive1",
           "action": "report",
           "operation": "read"
         },
         "event": "BLOCK_JOB_ERROR"
       }

    ...

    <- {
         "timestamp": {...},
         "data": {
           "speed": 0,
           "offset": 0,
           "len": 67108864,
           "error": "Input/output error",
           "device": "drive1",
           "type": "backup"
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

At the conclusion of the above example, ``drive0.inc0.qcow2`` is valid and
must be kept, but ``drive1.inc0.qcow2`` is incomplete and should be
deleted. If a VM-wide incremental backup of all drives at a point-in-time is
to be made, new backups for both drives will need to be made, taking into
account that a new incremental backup for drive0 needs to be based on top of
``drive0.inc0.qcow2``.

For this example, an incremental backup for ``drive0`` was created, but not
for ``drive1``. The last VM-wide crash-consistent backup that is available in
this case is the full backup:

.. code:: text

          [drive0.full.qcow2] <-- [drive0.inc0.qcow2]
          [drive1.full.qcow2]

To repair this, issue a new incremental backup across both drives. The result
will be backup chains that resemble the following:

.. code:: text

          [drive0.full.qcow2] <-- [drive0.inc0.qcow2] <-- [drive0.inc1.qcow2]
          [drive1.full.qcow2] <-------------------------- [drive1.inc1.qcow2]

Example: Grouped Completion Mode
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

While jobs launched by transactions normally complete or fail individually,
it's possible to instruct them to complete or fail together as a group. QMP
transactions take an optional properties structure that can affect the
behavior of the transaction.

The ``completion-mode`` transaction property can be either ``individual``
which is the default legacy behavior described above, or ``grouped``, detailed
below.

In ``grouped`` completion mode, no jobs will report success until all jobs are
ready to report success. If any job fails, all other jobs will be cancelled.

Regardless of if a participating incremental backup job failed or was
cancelled, their associated bitmaps will all be held at their existing
points-in-time, as in individual failure cases.

Here's the same multi-drive backup scenario from `Example: Partial
Transactional Failures`_, but with the ``grouped`` completion-mode property
applied:

#. Issue the multi-drive incremental backup transaction:

   .. code:: json

    -> {
         "execute": "transaction",
         "arguments": {
           "properties": {
             "completion-mode": "grouped"
           },
           "actions": [
           {
             "type": "drive-backup",
             "data": {
               "device": "drive0",
               "bitmap": "bitmap0",
               "format": "qcow2",
               "mode": "existing",
               "sync": "incremental",
               "target": "drive0.inc0.qcow2"
             }
           },
           {
             "type": "drive-backup",
             "data": {
               "device": "drive1",
               "bitmap": "bitmap0",
               "format": "qcow2",
               "mode": "existing",
               "sync": "incremental",
               "target": "drive1.inc0.qcow2"
             }
           }]
         }
       }

#. Receive notice that the Transaction was accepted, and jobs were launched:

   .. code:: json

    <- { "return": {} }

#. Receive notification that the backup job for ``drive1`` has failed:

   .. code:: json

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive1",
           "action": "report",
           "operation": "read"
         },
         "event": "BLOCK_JOB_ERROR"
       }

    <- {
         "timestamp": {...},
         "data": {
           "speed": 0,
           "offset": 0,
           "len": 67108864,
           "error": "Input/output error",
           "device": "drive1",
           "type": "backup"
         },
         "event": "BLOCK_JOB_COMPLETED"
       }

#. Receive notification that the job for ``drive0`` has been cancelled:

   .. code:: json

    <- {
         "timestamp": {...},
         "data": {
           "device": "drive0",
           "type": "backup",
           "speed": 0,
           "len": 67108864,
           "offset": 16777216
         },
         "event": "BLOCK_JOB_CANCELLED"
       }

At the conclusion of *this* example, both jobs have been aborted due to a
failure. Both destination images should be deleted and are no longer of use.

The transaction as a whole can simply be re-issued at a later time.

.. raw:: html

   <!--
   The FreeBSD Documentation License

   Redistribution and use in source (ReST) and 'compiled' forms (SGML, HTML,
   PDF, PostScript, RTF and so forth) with or without modification, are
   permitted provided that the following conditions are met:

   Redistributions of source code (ReST) must retain the above copyright notice,
   this list of conditions and the following disclaimer of this file unmodified.

   Redistributions in compiled form (transformed to other DTDs, converted to
   PDF, PostScript, RTF and other formats) must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the distribution.

   THIS DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
   IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS DOCUMENTATION, EVEN IF ADVISED OF
   THE POSSIBILITY OF SUCH DAMAGE.
   -->