You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
qemu/system/ram-block-attributes.c

445 lines
15 KiB
C

/*
* QEMU ram block attributes
*
* Copyright Intel
*
* Author:
* Chenyi Qiang <chenyi.qiang@intel.com>
*
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#include "qemu/osdep.h"
#include "qemu/error-report.h"
#include "system/ramblock.h"
#include "trace.h"
OBJECT_DEFINE_SIMPLE_TYPE_WITH_INTERFACES(RamBlockAttributes,
ram_block_attributes,
RAM_BLOCK_ATTRIBUTES,
OBJECT,
{ TYPE_RAM_DISCARD_MANAGER },
{ })
static size_t
ram_block_attributes_get_block_size(const RamBlockAttributes *attr)
{
/*
* Because page conversion could be manipulated in the size of at least 4K
* or 4K aligned, Use the host page size as the granularity to track the
* memory attribute.
*/
g_assert(attr && attr->ram_block);
g_assert(attr->ram_block->page_size == qemu_real_host_page_size());
return attr->ram_block->page_size;
}
static bool
ram_block_attributes_rdm_is_populated(const RamDiscardManager *rdm,
const MemoryRegionSection *section)
{
const RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
const size_t block_size = ram_block_attributes_get_block_size(attr);
const uint64_t first_bit = section->offset_within_region / block_size;
const uint64_t last_bit =
first_bit + int128_get64(section->size) / block_size - 1;
unsigned long first_discarded_bit;
first_discarded_bit = find_next_zero_bit(attr->bitmap, last_bit + 1,
first_bit);
return first_discarded_bit > last_bit;
}
typedef int (*ram_block_attributes_section_cb)(MemoryRegionSection *s,
void *arg);
static int
ram_block_attributes_notify_populate_cb(MemoryRegionSection *section,
void *arg)
{
RamDiscardListener *rdl = arg;
return rdl->notify_populate(rdl, section);
}
static int
ram_block_attributes_notify_discard_cb(MemoryRegionSection *section,
void *arg)
{
RamDiscardListener *rdl = arg;
rdl->notify_discard(rdl, section);
return 0;
}
static int
ram_block_attributes_for_each_populated_section(const RamBlockAttributes *attr,
MemoryRegionSection *section,
void *arg,
ram_block_attributes_section_cb cb)
{
unsigned long first_bit, last_bit;
uint64_t offset, size;
const size_t block_size = ram_block_attributes_get_block_size(attr);
int ret = 0;
first_bit = section->offset_within_region / block_size;
first_bit = find_next_bit(attr->bitmap, attr->bitmap_size,
first_bit);
while (first_bit < attr->bitmap_size) {
MemoryRegionSection tmp = *section;
offset = first_bit * block_size;
last_bit = find_next_zero_bit(attr->bitmap, attr->bitmap_size,
first_bit + 1) - 1;
size = (last_bit - first_bit + 1) * block_size;
if (!memory_region_section_intersect_range(&tmp, offset, size)) {
break;
}
ret = cb(&tmp, arg);
if (ret) {
error_report("%s: Failed to notify RAM discard listener: %s",
__func__, strerror(-ret));
break;
}
first_bit = find_next_bit(attr->bitmap, attr->bitmap_size,
last_bit + 2);
}
return ret;
}
static int
ram_block_attributes_for_each_discarded_section(const RamBlockAttributes *attr,
MemoryRegionSection *section,
void *arg,
ram_block_attributes_section_cb cb)
{
unsigned long first_bit, last_bit;
uint64_t offset, size;
const size_t block_size = ram_block_attributes_get_block_size(attr);
int ret = 0;
first_bit = section->offset_within_region / block_size;
first_bit = find_next_zero_bit(attr->bitmap, attr->bitmap_size,
first_bit);
while (first_bit < attr->bitmap_size) {
MemoryRegionSection tmp = *section;
offset = first_bit * block_size;
last_bit = find_next_bit(attr->bitmap, attr->bitmap_size,
first_bit + 1) - 1;
size = (last_bit - first_bit + 1) * block_size;
if (!memory_region_section_intersect_range(&tmp, offset, size)) {
break;
}
ret = cb(&tmp, arg);
if (ret) {
error_report("%s: Failed to notify RAM discard listener: %s",
__func__, strerror(-ret));
break;
}
first_bit = find_next_zero_bit(attr->bitmap,
attr->bitmap_size,
last_bit + 2);
}
return ret;
}
static uint64_t
ram_block_attributes_rdm_get_min_granularity(const RamDiscardManager *rdm,
const MemoryRegion *mr)
{
const RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
g_assert(mr == attr->ram_block->mr);
return ram_block_attributes_get_block_size(attr);
}
static void
ram_block_attributes_rdm_register_listener(RamDiscardManager *rdm,
RamDiscardListener *rdl,
MemoryRegionSection *section)
{
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
int ret;
g_assert(section->mr == attr->ram_block->mr);
rdl->section = memory_region_section_new_copy(section);
QLIST_INSERT_HEAD(&attr->rdl_list, rdl, next);
ret = ram_block_attributes_for_each_populated_section(attr, section, rdl,
ram_block_attributes_notify_populate_cb);
if (ret) {
error_report("%s: Failed to register RAM discard listener: %s",
__func__, strerror(-ret));
exit(1);
}
}
static void
ram_block_attributes_rdm_unregister_listener(RamDiscardManager *rdm,
RamDiscardListener *rdl)
{
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
int ret;
g_assert(rdl->section);
g_assert(rdl->section->mr == attr->ram_block->mr);
if (rdl->double_discard_supported) {
rdl->notify_discard(rdl, rdl->section);
} else {
ret = ram_block_attributes_for_each_populated_section(attr,
rdl->section, rdl, ram_block_attributes_notify_discard_cb);
if (ret) {
error_report("%s: Failed to unregister RAM discard listener: %s",
__func__, strerror(-ret));
exit(1);
}
}
memory_region_section_free_copy(rdl->section);
rdl->section = NULL;
QLIST_REMOVE(rdl, next);
}
typedef struct RamBlockAttributesReplayData {
ReplayRamDiscardState fn;
void *opaque;
} RamBlockAttributesReplayData;
static int ram_block_attributes_rdm_replay_cb(MemoryRegionSection *section,
void *arg)
{
RamBlockAttributesReplayData *data = arg;
return data->fn(section, data->opaque);
}
static int
ram_block_attributes_rdm_replay_populated(const RamDiscardManager *rdm,
MemoryRegionSection *section,
ReplayRamDiscardState replay_fn,
void *opaque)
{
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
RamBlockAttributesReplayData data = { .fn = replay_fn, .opaque = opaque };
g_assert(section->mr == attr->ram_block->mr);
return ram_block_attributes_for_each_populated_section(attr, section, &data,
ram_block_attributes_rdm_replay_cb);
}
static int
ram_block_attributes_rdm_replay_discarded(const RamDiscardManager *rdm,
MemoryRegionSection *section,
ReplayRamDiscardState replay_fn,
void *opaque)
{
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
RamBlockAttributesReplayData data = { .fn = replay_fn, .opaque = opaque };
g_assert(section->mr == attr->ram_block->mr);
return ram_block_attributes_for_each_discarded_section(attr, section, &data,
ram_block_attributes_rdm_replay_cb);
}
static bool
ram_block_attributes_is_valid_range(RamBlockAttributes *attr, uint64_t offset,
uint64_t size)
{
MemoryRegion *mr = attr->ram_block->mr;
g_assert(mr);
uint64_t region_size = memory_region_size(mr);
const size_t block_size = ram_block_attributes_get_block_size(attr);
if (!QEMU_IS_ALIGNED(offset, block_size) ||
!QEMU_IS_ALIGNED(size, block_size)) {
return false;
}
if (offset + size <= offset) {
return false;
}
if (offset + size > region_size) {
return false;
}
return true;
}
static void ram_block_attributes_notify_discard(RamBlockAttributes *attr,
uint64_t offset,
uint64_t size)
{
RamDiscardListener *rdl;
QLIST_FOREACH(rdl, &attr->rdl_list, next) {
MemoryRegionSection tmp = *rdl->section;
if (!memory_region_section_intersect_range(&tmp, offset, size)) {
continue;
}
rdl->notify_discard(rdl, &tmp);
}
}
static int
ram_block_attributes_notify_populate(RamBlockAttributes *attr,
uint64_t offset, uint64_t size)
{
RamDiscardListener *rdl;
int ret = 0;
QLIST_FOREACH(rdl, &attr->rdl_list, next) {
MemoryRegionSection tmp = *rdl->section;
if (!memory_region_section_intersect_range(&tmp, offset, size)) {
continue;
}
ret = rdl->notify_populate(rdl, &tmp);
if (ret) {
break;
}
}
return ret;
}
int ram_block_attributes_state_change(RamBlockAttributes *attr,
uint64_t offset, uint64_t size,
bool to_discard)
{
const size_t block_size = ram_block_attributes_get_block_size(attr);
const unsigned long first_bit = offset / block_size;
const unsigned long nbits = size / block_size;
const unsigned long last_bit = first_bit + nbits - 1;
const bool is_discarded = find_next_bit(attr->bitmap, attr->bitmap_size,
first_bit) > last_bit;
const bool is_populated = find_next_zero_bit(attr->bitmap,
attr->bitmap_size, first_bit) > last_bit;
unsigned long bit;
int ret = 0;
if (!ram_block_attributes_is_valid_range(attr, offset, size)) {
error_report("%s, invalid range: offset 0x%" PRIx64 ", size "
"0x%" PRIx64, __func__, offset, size);
return -EINVAL;
}
trace_ram_block_attributes_state_change(offset, size,
is_discarded ? "discarded" :
is_populated ? "populated" :
"mixture",
to_discard ? "discarded" :
"populated");
if (to_discard) {
if (is_discarded) {
/* Already private */
} else if (is_populated) {
/* Completely shared */
bitmap_clear(attr->bitmap, first_bit, nbits);
ram_block_attributes_notify_discard(attr, offset, size);
} else {
/* Unexpected mixture: process individual blocks */
for (bit = first_bit; bit < first_bit + nbits; bit++) {
if (!test_bit(bit, attr->bitmap)) {
continue;
}
clear_bit(bit, attr->bitmap);
ram_block_attributes_notify_discard(attr, bit * block_size,
block_size);
}
}
} else {
if (is_populated) {
/* Already shared */
} else if (is_discarded) {
/* Completely private */
bitmap_set(attr->bitmap, first_bit, nbits);
ret = ram_block_attributes_notify_populate(attr, offset, size);
} else {
/* Unexpected mixture: process individual blocks */
for (bit = first_bit; bit < first_bit + nbits; bit++) {
if (test_bit(bit, attr->bitmap)) {
continue;
}
set_bit(bit, attr->bitmap);
ret = ram_block_attributes_notify_populate(attr,
bit * block_size,
block_size);
if (ret) {
break;
}
}
}
}
return ret;
}
RamBlockAttributes *ram_block_attributes_create(RAMBlock *ram_block)
{
const int block_size = qemu_real_host_page_size();
RamBlockAttributes *attr;
MemoryRegion *mr = ram_block->mr;
attr = RAM_BLOCK_ATTRIBUTES(object_new(TYPE_RAM_BLOCK_ATTRIBUTES));
attr->ram_block = ram_block;
if (memory_region_set_ram_discard_manager(mr, RAM_DISCARD_MANAGER(attr))) {
object_unref(OBJECT(attr));
return NULL;
}
attr->bitmap_size =
ROUND_UP(int128_get64(mr->size), block_size) / block_size;
attr->bitmap = bitmap_new(attr->bitmap_size);
return attr;
}
void ram_block_attributes_destroy(RamBlockAttributes *attr)
{
g_assert(attr);
g_free(attr->bitmap);
memory_region_set_ram_discard_manager(attr->ram_block->mr, NULL);
object_unref(OBJECT(attr));
}
static void ram_block_attributes_init(Object *obj)
{
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(obj);
QLIST_INIT(&attr->rdl_list);
}
static void ram_block_attributes_finalize(Object *obj)
{
}
static void ram_block_attributes_class_init(ObjectClass *klass,
const void *data)
{
RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(klass);
rdmc->get_min_granularity = ram_block_attributes_rdm_get_min_granularity;
rdmc->register_listener = ram_block_attributes_rdm_register_listener;
rdmc->unregister_listener = ram_block_attributes_rdm_unregister_listener;
rdmc->is_populated = ram_block_attributes_rdm_is_populated;
rdmc->replay_populated = ram_block_attributes_rdm_replay_populated;
rdmc->replay_discarded = ram_block_attributes_rdm_replay_discarded;
}