mirror of https://gitlab.com/qemu-project/qemu
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
445 lines
15 KiB
C
445 lines
15 KiB
C
/*
|
|
* QEMU ram block attributes
|
|
*
|
|
* Copyright Intel
|
|
*
|
|
* Author:
|
|
* Chenyi Qiang <chenyi.qiang@intel.com>
|
|
*
|
|
* SPDX-License-Identifier: GPL-2.0-or-later
|
|
*/
|
|
|
|
#include "qemu/osdep.h"
|
|
#include "qemu/error-report.h"
|
|
#include "system/ramblock.h"
|
|
#include "trace.h"
|
|
|
|
OBJECT_DEFINE_SIMPLE_TYPE_WITH_INTERFACES(RamBlockAttributes,
|
|
ram_block_attributes,
|
|
RAM_BLOCK_ATTRIBUTES,
|
|
OBJECT,
|
|
{ TYPE_RAM_DISCARD_MANAGER },
|
|
{ })
|
|
|
|
static size_t
|
|
ram_block_attributes_get_block_size(const RamBlockAttributes *attr)
|
|
{
|
|
/*
|
|
* Because page conversion could be manipulated in the size of at least 4K
|
|
* or 4K aligned, Use the host page size as the granularity to track the
|
|
* memory attribute.
|
|
*/
|
|
g_assert(attr && attr->ram_block);
|
|
g_assert(attr->ram_block->page_size == qemu_real_host_page_size());
|
|
return attr->ram_block->page_size;
|
|
}
|
|
|
|
|
|
static bool
|
|
ram_block_attributes_rdm_is_populated(const RamDiscardManager *rdm,
|
|
const MemoryRegionSection *section)
|
|
{
|
|
const RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
|
|
const size_t block_size = ram_block_attributes_get_block_size(attr);
|
|
const uint64_t first_bit = section->offset_within_region / block_size;
|
|
const uint64_t last_bit =
|
|
first_bit + int128_get64(section->size) / block_size - 1;
|
|
unsigned long first_discarded_bit;
|
|
|
|
first_discarded_bit = find_next_zero_bit(attr->bitmap, last_bit + 1,
|
|
first_bit);
|
|
return first_discarded_bit > last_bit;
|
|
}
|
|
|
|
typedef int (*ram_block_attributes_section_cb)(MemoryRegionSection *s,
|
|
void *arg);
|
|
|
|
static int
|
|
ram_block_attributes_notify_populate_cb(MemoryRegionSection *section,
|
|
void *arg)
|
|
{
|
|
RamDiscardListener *rdl = arg;
|
|
|
|
return rdl->notify_populate(rdl, section);
|
|
}
|
|
|
|
static int
|
|
ram_block_attributes_notify_discard_cb(MemoryRegionSection *section,
|
|
void *arg)
|
|
{
|
|
RamDiscardListener *rdl = arg;
|
|
|
|
rdl->notify_discard(rdl, section);
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
ram_block_attributes_for_each_populated_section(const RamBlockAttributes *attr,
|
|
MemoryRegionSection *section,
|
|
void *arg,
|
|
ram_block_attributes_section_cb cb)
|
|
{
|
|
unsigned long first_bit, last_bit;
|
|
uint64_t offset, size;
|
|
const size_t block_size = ram_block_attributes_get_block_size(attr);
|
|
int ret = 0;
|
|
|
|
first_bit = section->offset_within_region / block_size;
|
|
first_bit = find_next_bit(attr->bitmap, attr->bitmap_size,
|
|
first_bit);
|
|
|
|
while (first_bit < attr->bitmap_size) {
|
|
MemoryRegionSection tmp = *section;
|
|
|
|
offset = first_bit * block_size;
|
|
last_bit = find_next_zero_bit(attr->bitmap, attr->bitmap_size,
|
|
first_bit + 1) - 1;
|
|
size = (last_bit - first_bit + 1) * block_size;
|
|
|
|
if (!memory_region_section_intersect_range(&tmp, offset, size)) {
|
|
break;
|
|
}
|
|
|
|
ret = cb(&tmp, arg);
|
|
if (ret) {
|
|
error_report("%s: Failed to notify RAM discard listener: %s",
|
|
__func__, strerror(-ret));
|
|
break;
|
|
}
|
|
|
|
first_bit = find_next_bit(attr->bitmap, attr->bitmap_size,
|
|
last_bit + 2);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int
|
|
ram_block_attributes_for_each_discarded_section(const RamBlockAttributes *attr,
|
|
MemoryRegionSection *section,
|
|
void *arg,
|
|
ram_block_attributes_section_cb cb)
|
|
{
|
|
unsigned long first_bit, last_bit;
|
|
uint64_t offset, size;
|
|
const size_t block_size = ram_block_attributes_get_block_size(attr);
|
|
int ret = 0;
|
|
|
|
first_bit = section->offset_within_region / block_size;
|
|
first_bit = find_next_zero_bit(attr->bitmap, attr->bitmap_size,
|
|
first_bit);
|
|
|
|
while (first_bit < attr->bitmap_size) {
|
|
MemoryRegionSection tmp = *section;
|
|
|
|
offset = first_bit * block_size;
|
|
last_bit = find_next_bit(attr->bitmap, attr->bitmap_size,
|
|
first_bit + 1) - 1;
|
|
size = (last_bit - first_bit + 1) * block_size;
|
|
|
|
if (!memory_region_section_intersect_range(&tmp, offset, size)) {
|
|
break;
|
|
}
|
|
|
|
ret = cb(&tmp, arg);
|
|
if (ret) {
|
|
error_report("%s: Failed to notify RAM discard listener: %s",
|
|
__func__, strerror(-ret));
|
|
break;
|
|
}
|
|
|
|
first_bit = find_next_zero_bit(attr->bitmap,
|
|
attr->bitmap_size,
|
|
last_bit + 2);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static uint64_t
|
|
ram_block_attributes_rdm_get_min_granularity(const RamDiscardManager *rdm,
|
|
const MemoryRegion *mr)
|
|
{
|
|
const RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
|
|
|
|
g_assert(mr == attr->ram_block->mr);
|
|
return ram_block_attributes_get_block_size(attr);
|
|
}
|
|
|
|
static void
|
|
ram_block_attributes_rdm_register_listener(RamDiscardManager *rdm,
|
|
RamDiscardListener *rdl,
|
|
MemoryRegionSection *section)
|
|
{
|
|
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
|
|
int ret;
|
|
|
|
g_assert(section->mr == attr->ram_block->mr);
|
|
rdl->section = memory_region_section_new_copy(section);
|
|
|
|
QLIST_INSERT_HEAD(&attr->rdl_list, rdl, next);
|
|
|
|
ret = ram_block_attributes_for_each_populated_section(attr, section, rdl,
|
|
ram_block_attributes_notify_populate_cb);
|
|
if (ret) {
|
|
error_report("%s: Failed to register RAM discard listener: %s",
|
|
__func__, strerror(-ret));
|
|
exit(1);
|
|
}
|
|
}
|
|
|
|
static void
|
|
ram_block_attributes_rdm_unregister_listener(RamDiscardManager *rdm,
|
|
RamDiscardListener *rdl)
|
|
{
|
|
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
|
|
int ret;
|
|
|
|
g_assert(rdl->section);
|
|
g_assert(rdl->section->mr == attr->ram_block->mr);
|
|
|
|
if (rdl->double_discard_supported) {
|
|
rdl->notify_discard(rdl, rdl->section);
|
|
} else {
|
|
ret = ram_block_attributes_for_each_populated_section(attr,
|
|
rdl->section, rdl, ram_block_attributes_notify_discard_cb);
|
|
if (ret) {
|
|
error_report("%s: Failed to unregister RAM discard listener: %s",
|
|
__func__, strerror(-ret));
|
|
exit(1);
|
|
}
|
|
}
|
|
|
|
memory_region_section_free_copy(rdl->section);
|
|
rdl->section = NULL;
|
|
QLIST_REMOVE(rdl, next);
|
|
}
|
|
|
|
typedef struct RamBlockAttributesReplayData {
|
|
ReplayRamDiscardState fn;
|
|
void *opaque;
|
|
} RamBlockAttributesReplayData;
|
|
|
|
static int ram_block_attributes_rdm_replay_cb(MemoryRegionSection *section,
|
|
void *arg)
|
|
{
|
|
RamBlockAttributesReplayData *data = arg;
|
|
|
|
return data->fn(section, data->opaque);
|
|
}
|
|
|
|
static int
|
|
ram_block_attributes_rdm_replay_populated(const RamDiscardManager *rdm,
|
|
MemoryRegionSection *section,
|
|
ReplayRamDiscardState replay_fn,
|
|
void *opaque)
|
|
{
|
|
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
|
|
RamBlockAttributesReplayData data = { .fn = replay_fn, .opaque = opaque };
|
|
|
|
g_assert(section->mr == attr->ram_block->mr);
|
|
return ram_block_attributes_for_each_populated_section(attr, section, &data,
|
|
ram_block_attributes_rdm_replay_cb);
|
|
}
|
|
|
|
static int
|
|
ram_block_attributes_rdm_replay_discarded(const RamDiscardManager *rdm,
|
|
MemoryRegionSection *section,
|
|
ReplayRamDiscardState replay_fn,
|
|
void *opaque)
|
|
{
|
|
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(rdm);
|
|
RamBlockAttributesReplayData data = { .fn = replay_fn, .opaque = opaque };
|
|
|
|
g_assert(section->mr == attr->ram_block->mr);
|
|
return ram_block_attributes_for_each_discarded_section(attr, section, &data,
|
|
ram_block_attributes_rdm_replay_cb);
|
|
}
|
|
|
|
static bool
|
|
ram_block_attributes_is_valid_range(RamBlockAttributes *attr, uint64_t offset,
|
|
uint64_t size)
|
|
{
|
|
MemoryRegion *mr = attr->ram_block->mr;
|
|
|
|
g_assert(mr);
|
|
|
|
uint64_t region_size = memory_region_size(mr);
|
|
const size_t block_size = ram_block_attributes_get_block_size(attr);
|
|
|
|
if (!QEMU_IS_ALIGNED(offset, block_size) ||
|
|
!QEMU_IS_ALIGNED(size, block_size)) {
|
|
return false;
|
|
}
|
|
if (offset + size <= offset) {
|
|
return false;
|
|
}
|
|
if (offset + size > region_size) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static void ram_block_attributes_notify_discard(RamBlockAttributes *attr,
|
|
uint64_t offset,
|
|
uint64_t size)
|
|
{
|
|
RamDiscardListener *rdl;
|
|
|
|
QLIST_FOREACH(rdl, &attr->rdl_list, next) {
|
|
MemoryRegionSection tmp = *rdl->section;
|
|
|
|
if (!memory_region_section_intersect_range(&tmp, offset, size)) {
|
|
continue;
|
|
}
|
|
rdl->notify_discard(rdl, &tmp);
|
|
}
|
|
}
|
|
|
|
static int
|
|
ram_block_attributes_notify_populate(RamBlockAttributes *attr,
|
|
uint64_t offset, uint64_t size)
|
|
{
|
|
RamDiscardListener *rdl;
|
|
int ret = 0;
|
|
|
|
QLIST_FOREACH(rdl, &attr->rdl_list, next) {
|
|
MemoryRegionSection tmp = *rdl->section;
|
|
|
|
if (!memory_region_section_intersect_range(&tmp, offset, size)) {
|
|
continue;
|
|
}
|
|
ret = rdl->notify_populate(rdl, &tmp);
|
|
if (ret) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
int ram_block_attributes_state_change(RamBlockAttributes *attr,
|
|
uint64_t offset, uint64_t size,
|
|
bool to_discard)
|
|
{
|
|
const size_t block_size = ram_block_attributes_get_block_size(attr);
|
|
const unsigned long first_bit = offset / block_size;
|
|
const unsigned long nbits = size / block_size;
|
|
const unsigned long last_bit = first_bit + nbits - 1;
|
|
const bool is_discarded = find_next_bit(attr->bitmap, attr->bitmap_size,
|
|
first_bit) > last_bit;
|
|
const bool is_populated = find_next_zero_bit(attr->bitmap,
|
|
attr->bitmap_size, first_bit) > last_bit;
|
|
unsigned long bit;
|
|
int ret = 0;
|
|
|
|
if (!ram_block_attributes_is_valid_range(attr, offset, size)) {
|
|
error_report("%s, invalid range: offset 0x%" PRIx64 ", size "
|
|
"0x%" PRIx64, __func__, offset, size);
|
|
return -EINVAL;
|
|
}
|
|
|
|
trace_ram_block_attributes_state_change(offset, size,
|
|
is_discarded ? "discarded" :
|
|
is_populated ? "populated" :
|
|
"mixture",
|
|
to_discard ? "discarded" :
|
|
"populated");
|
|
if (to_discard) {
|
|
if (is_discarded) {
|
|
/* Already private */
|
|
} else if (is_populated) {
|
|
/* Completely shared */
|
|
bitmap_clear(attr->bitmap, first_bit, nbits);
|
|
ram_block_attributes_notify_discard(attr, offset, size);
|
|
} else {
|
|
/* Unexpected mixture: process individual blocks */
|
|
for (bit = first_bit; bit < first_bit + nbits; bit++) {
|
|
if (!test_bit(bit, attr->bitmap)) {
|
|
continue;
|
|
}
|
|
clear_bit(bit, attr->bitmap);
|
|
ram_block_attributes_notify_discard(attr, bit * block_size,
|
|
block_size);
|
|
}
|
|
}
|
|
} else {
|
|
if (is_populated) {
|
|
/* Already shared */
|
|
} else if (is_discarded) {
|
|
/* Completely private */
|
|
bitmap_set(attr->bitmap, first_bit, nbits);
|
|
ret = ram_block_attributes_notify_populate(attr, offset, size);
|
|
} else {
|
|
/* Unexpected mixture: process individual blocks */
|
|
for (bit = first_bit; bit < first_bit + nbits; bit++) {
|
|
if (test_bit(bit, attr->bitmap)) {
|
|
continue;
|
|
}
|
|
set_bit(bit, attr->bitmap);
|
|
ret = ram_block_attributes_notify_populate(attr,
|
|
bit * block_size,
|
|
block_size);
|
|
if (ret) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
RamBlockAttributes *ram_block_attributes_create(RAMBlock *ram_block)
|
|
{
|
|
const int block_size = qemu_real_host_page_size();
|
|
RamBlockAttributes *attr;
|
|
MemoryRegion *mr = ram_block->mr;
|
|
|
|
attr = RAM_BLOCK_ATTRIBUTES(object_new(TYPE_RAM_BLOCK_ATTRIBUTES));
|
|
|
|
attr->ram_block = ram_block;
|
|
if (memory_region_set_ram_discard_manager(mr, RAM_DISCARD_MANAGER(attr))) {
|
|
object_unref(OBJECT(attr));
|
|
return NULL;
|
|
}
|
|
attr->bitmap_size =
|
|
ROUND_UP(int128_get64(mr->size), block_size) / block_size;
|
|
attr->bitmap = bitmap_new(attr->bitmap_size);
|
|
|
|
return attr;
|
|
}
|
|
|
|
void ram_block_attributes_destroy(RamBlockAttributes *attr)
|
|
{
|
|
g_assert(attr);
|
|
|
|
g_free(attr->bitmap);
|
|
memory_region_set_ram_discard_manager(attr->ram_block->mr, NULL);
|
|
object_unref(OBJECT(attr));
|
|
}
|
|
|
|
static void ram_block_attributes_init(Object *obj)
|
|
{
|
|
RamBlockAttributes *attr = RAM_BLOCK_ATTRIBUTES(obj);
|
|
|
|
QLIST_INIT(&attr->rdl_list);
|
|
}
|
|
|
|
static void ram_block_attributes_finalize(Object *obj)
|
|
{
|
|
}
|
|
|
|
static void ram_block_attributes_class_init(ObjectClass *klass,
|
|
const void *data)
|
|
{
|
|
RamDiscardManagerClass *rdmc = RAM_DISCARD_MANAGER_CLASS(klass);
|
|
|
|
rdmc->get_min_granularity = ram_block_attributes_rdm_get_min_granularity;
|
|
rdmc->register_listener = ram_block_attributes_rdm_register_listener;
|
|
rdmc->unregister_listener = ram_block_attributes_rdm_unregister_listener;
|
|
rdmc->is_populated = ram_block_attributes_rdm_is_populated;
|
|
rdmc->replay_populated = ram_block_attributes_rdm_replay_populated;
|
|
rdmc->replay_discarded = ram_block_attributes_rdm_replay_discarded;
|
|
}
|