Lines Matching refs:vm
275 static void virtio_mem_retry(struct virtio_mem *vm);
276 static int virtio_mem_create_resource(struct virtio_mem *vm);
277 static void virtio_mem_delete_resource(struct virtio_mem *vm);
283 static int register_virtio_mem_device(struct virtio_mem *vm) in register_virtio_mem_device() argument
292 list_add_rcu(&vm->next, &virtio_mem_devices); in register_virtio_mem_device()
302 static void unregister_virtio_mem_device(struct virtio_mem *vm) in unregister_virtio_mem_device() argument
306 list_del_rcu(&vm->next); in unregister_virtio_mem_device()
333 static unsigned long virtio_mem_phys_to_bb_id(struct virtio_mem *vm, in virtio_mem_phys_to_bb_id() argument
336 return addr / vm->bbm.bb_size; in virtio_mem_phys_to_bb_id()
342 static uint64_t virtio_mem_bb_id_to_phys(struct virtio_mem *vm, in virtio_mem_bb_id_to_phys() argument
345 return bb_id * vm->bbm.bb_size; in virtio_mem_bb_id_to_phys()
351 static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm, in virtio_mem_phys_to_sb_id() argument
357 return (addr - mb_addr) / vm->sbm.sb_size; in virtio_mem_phys_to_sb_id()
363 static void virtio_mem_bbm_set_bb_state(struct virtio_mem *vm, in virtio_mem_bbm_set_bb_state() argument
367 const unsigned long idx = bb_id - vm->bbm.first_bb_id; in virtio_mem_bbm_set_bb_state()
370 old_state = vm->bbm.bb_states[idx]; in virtio_mem_bbm_set_bb_state()
371 vm->bbm.bb_states[idx] = state; in virtio_mem_bbm_set_bb_state()
373 BUG_ON(vm->bbm.bb_count[old_state] == 0); in virtio_mem_bbm_set_bb_state()
374 vm->bbm.bb_count[old_state]--; in virtio_mem_bbm_set_bb_state()
375 vm->bbm.bb_count[state]++; in virtio_mem_bbm_set_bb_state()
381 static enum virtio_mem_bbm_bb_state virtio_mem_bbm_get_bb_state(struct virtio_mem *vm, in virtio_mem_bbm_get_bb_state() argument
384 return vm->bbm.bb_states[bb_id - vm->bbm.first_bb_id]; in virtio_mem_bbm_get_bb_state()
390 static int virtio_mem_bbm_bb_states_prepare_next_bb(struct virtio_mem *vm) in virtio_mem_bbm_bb_states_prepare_next_bb() argument
392 unsigned long old_bytes = vm->bbm.next_bb_id - vm->bbm.first_bb_id; in virtio_mem_bbm_bb_states_prepare_next_bb()
398 if (vm->bbm.bb_states && old_pages == new_pages) in virtio_mem_bbm_bb_states_prepare_next_bb()
405 mutex_lock(&vm->hotplug_mutex); in virtio_mem_bbm_bb_states_prepare_next_bb()
406 if (vm->bbm.bb_states) in virtio_mem_bbm_bb_states_prepare_next_bb()
407 memcpy(new_array, vm->bbm.bb_states, old_pages * PAGE_SIZE); in virtio_mem_bbm_bb_states_prepare_next_bb()
408 vfree(vm->bbm.bb_states); in virtio_mem_bbm_bb_states_prepare_next_bb()
409 vm->bbm.bb_states = new_array; in virtio_mem_bbm_bb_states_prepare_next_bb()
410 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_bbm_bb_states_prepare_next_bb()
416 for (_bb_id = vm->bbm.first_bb_id; \
417 _bb_id < vm->bbm.next_bb_id && _vm->bbm.bb_count[_state]; \
422 for (_bb_id = vm->bbm.next_bb_id - 1; \
423 _bb_id >= vm->bbm.first_bb_id && _vm->bbm.bb_count[_state]; \
430 static void virtio_mem_sbm_set_mb_state(struct virtio_mem *vm, in virtio_mem_sbm_set_mb_state() argument
433 const unsigned long idx = mb_id - vm->sbm.first_mb_id; in virtio_mem_sbm_set_mb_state()
436 old_state = vm->sbm.mb_states[idx]; in virtio_mem_sbm_set_mb_state()
437 vm->sbm.mb_states[idx] = state; in virtio_mem_sbm_set_mb_state()
439 BUG_ON(vm->sbm.mb_count[old_state] == 0); in virtio_mem_sbm_set_mb_state()
440 vm->sbm.mb_count[old_state]--; in virtio_mem_sbm_set_mb_state()
441 vm->sbm.mb_count[state]++; in virtio_mem_sbm_set_mb_state()
447 static uint8_t virtio_mem_sbm_get_mb_state(struct virtio_mem *vm, in virtio_mem_sbm_get_mb_state() argument
450 const unsigned long idx = mb_id - vm->sbm.first_mb_id; in virtio_mem_sbm_get_mb_state()
452 return vm->sbm.mb_states[idx]; in virtio_mem_sbm_get_mb_state()
458 static int virtio_mem_sbm_mb_states_prepare_next_mb(struct virtio_mem *vm) in virtio_mem_sbm_mb_states_prepare_next_mb() argument
460 int old_pages = PFN_UP(vm->sbm.next_mb_id - vm->sbm.first_mb_id); in virtio_mem_sbm_mb_states_prepare_next_mb()
461 int new_pages = PFN_UP(vm->sbm.next_mb_id - vm->sbm.first_mb_id + 1); in virtio_mem_sbm_mb_states_prepare_next_mb()
464 if (vm->sbm.mb_states && old_pages == new_pages) in virtio_mem_sbm_mb_states_prepare_next_mb()
471 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_mb_states_prepare_next_mb()
472 if (vm->sbm.mb_states) in virtio_mem_sbm_mb_states_prepare_next_mb()
473 memcpy(new_array, vm->sbm.mb_states, old_pages * PAGE_SIZE); in virtio_mem_sbm_mb_states_prepare_next_mb()
474 vfree(vm->sbm.mb_states); in virtio_mem_sbm_mb_states_prepare_next_mb()
475 vm->sbm.mb_states = new_array; in virtio_mem_sbm_mb_states_prepare_next_mb()
476 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_mb_states_prepare_next_mb()
497 static int virtio_mem_sbm_sb_state_bit_nr(struct virtio_mem *vm, in virtio_mem_sbm_sb_state_bit_nr() argument
500 return (mb_id - vm->sbm.first_mb_id) * vm->sbm.sbs_per_mb + sb_id; in virtio_mem_sbm_sb_state_bit_nr()
508 static void virtio_mem_sbm_set_sb_plugged(struct virtio_mem *vm, in virtio_mem_sbm_set_sb_plugged() argument
512 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id); in virtio_mem_sbm_set_sb_plugged()
514 __bitmap_set(vm->sbm.sb_states, bit, count); in virtio_mem_sbm_set_sb_plugged()
522 static void virtio_mem_sbm_set_sb_unplugged(struct virtio_mem *vm, in virtio_mem_sbm_set_sb_unplugged() argument
526 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id); in virtio_mem_sbm_set_sb_unplugged()
528 __bitmap_clear(vm->sbm.sb_states, bit, count); in virtio_mem_sbm_set_sb_unplugged()
534 static bool virtio_mem_sbm_test_sb_plugged(struct virtio_mem *vm, in virtio_mem_sbm_test_sb_plugged() argument
538 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id); in virtio_mem_sbm_test_sb_plugged()
541 return test_bit(bit, vm->sbm.sb_states); in virtio_mem_sbm_test_sb_plugged()
544 return find_next_zero_bit(vm->sbm.sb_states, bit + count, bit) >= in virtio_mem_sbm_test_sb_plugged()
551 static bool virtio_mem_sbm_test_sb_unplugged(struct virtio_mem *vm, in virtio_mem_sbm_test_sb_unplugged() argument
555 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, sb_id); in virtio_mem_sbm_test_sb_unplugged()
558 return find_next_bit(vm->sbm.sb_states, bit + count, bit) >= in virtio_mem_sbm_test_sb_unplugged()
566 static int virtio_mem_sbm_first_unplugged_sb(struct virtio_mem *vm, in virtio_mem_sbm_first_unplugged_sb() argument
569 const int bit = virtio_mem_sbm_sb_state_bit_nr(vm, mb_id, 0); in virtio_mem_sbm_first_unplugged_sb()
571 return find_next_zero_bit(vm->sbm.sb_states, in virtio_mem_sbm_first_unplugged_sb()
572 bit + vm->sbm.sbs_per_mb, bit) - bit; in virtio_mem_sbm_first_unplugged_sb()
578 static int virtio_mem_sbm_sb_states_prepare_next_mb(struct virtio_mem *vm) in virtio_mem_sbm_sb_states_prepare_next_mb() argument
580 const unsigned long old_nb_mb = vm->sbm.next_mb_id - vm->sbm.first_mb_id; in virtio_mem_sbm_sb_states_prepare_next_mb()
581 const unsigned long old_nb_bits = old_nb_mb * vm->sbm.sbs_per_mb; in virtio_mem_sbm_sb_states_prepare_next_mb()
582 const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->sbm.sbs_per_mb; in virtio_mem_sbm_sb_states_prepare_next_mb()
587 if (vm->sbm.sb_states && old_pages == new_pages) in virtio_mem_sbm_sb_states_prepare_next_mb()
594 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_sb_states_prepare_next_mb()
596 memcpy(new_bitmap, vm->sbm.sb_states, old_pages * PAGE_SIZE); in virtio_mem_sbm_sb_states_prepare_next_mb()
598 old_bitmap = vm->sbm.sb_states; in virtio_mem_sbm_sb_states_prepare_next_mb()
599 vm->sbm.sb_states = new_bitmap; in virtio_mem_sbm_sb_states_prepare_next_mb()
600 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_sb_states_prepare_next_mb()
610 static bool virtio_mem_could_add_memory(struct virtio_mem *vm, uint64_t size) in virtio_mem_could_add_memory() argument
612 if (WARN_ON_ONCE(size > vm->offline_threshold)) in virtio_mem_could_add_memory()
615 return atomic64_read(&vm->offline_size) + size <= vm->offline_threshold; in virtio_mem_could_add_memory()
626 static int virtio_mem_add_memory(struct virtio_mem *vm, uint64_t addr, in virtio_mem_add_memory() argument
635 if (!vm->resource_name) { in virtio_mem_add_memory()
636 vm->resource_name = kstrdup_const("System RAM (virtio_mem)", in virtio_mem_add_memory()
638 if (!vm->resource_name) in virtio_mem_add_memory()
642 dev_dbg(&vm->vdev->dev, "adding memory: 0x%llx - 0x%llx\n", addr, in virtio_mem_add_memory()
645 atomic64_add(size, &vm->offline_size); in virtio_mem_add_memory()
646 rc = add_memory_driver_managed(vm->mgid, addr, size, vm->resource_name, in virtio_mem_add_memory()
649 atomic64_sub(size, &vm->offline_size); in virtio_mem_add_memory()
650 dev_warn(&vm->vdev->dev, "adding memory failed: %d\n", rc); in virtio_mem_add_memory()
662 static int virtio_mem_sbm_add_mb(struct virtio_mem *vm, unsigned long mb_id) in virtio_mem_sbm_add_mb() argument
667 return virtio_mem_add_memory(vm, addr, size); in virtio_mem_sbm_add_mb()
673 static int virtio_mem_bbm_add_bb(struct virtio_mem *vm, unsigned long bb_id) in virtio_mem_bbm_add_bb() argument
675 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id); in virtio_mem_bbm_add_bb()
676 const uint64_t size = vm->bbm.bb_size; in virtio_mem_bbm_add_bb()
678 return virtio_mem_add_memory(vm, addr, size); in virtio_mem_bbm_add_bb()
690 static int virtio_mem_remove_memory(struct virtio_mem *vm, uint64_t addr, in virtio_mem_remove_memory() argument
695 dev_dbg(&vm->vdev->dev, "removing memory: 0x%llx - 0x%llx\n", addr, in virtio_mem_remove_memory()
699 atomic64_sub(size, &vm->offline_size); in virtio_mem_remove_memory()
704 virtio_mem_retry(vm); in virtio_mem_remove_memory()
706 dev_dbg(&vm->vdev->dev, "removing memory failed: %d\n", rc); in virtio_mem_remove_memory()
714 static int virtio_mem_sbm_remove_mb(struct virtio_mem *vm, unsigned long mb_id) in virtio_mem_sbm_remove_mb() argument
719 return virtio_mem_remove_memory(vm, addr, size); in virtio_mem_sbm_remove_mb()
730 static int virtio_mem_offline_and_remove_memory(struct virtio_mem *vm, in virtio_mem_offline_and_remove_memory() argument
736 dev_dbg(&vm->vdev->dev, in virtio_mem_offline_and_remove_memory()
742 atomic64_sub(size, &vm->offline_size); in virtio_mem_offline_and_remove_memory()
747 virtio_mem_retry(vm); in virtio_mem_offline_and_remove_memory()
749 dev_dbg(&vm->vdev->dev, in virtio_mem_offline_and_remove_memory()
759 static int virtio_mem_sbm_offline_and_remove_mb(struct virtio_mem *vm, in virtio_mem_sbm_offline_and_remove_mb() argument
765 return virtio_mem_offline_and_remove_memory(vm, addr, size); in virtio_mem_sbm_offline_and_remove_mb()
772 static int virtio_mem_bbm_offline_and_remove_bb(struct virtio_mem *vm, in virtio_mem_bbm_offline_and_remove_bb() argument
775 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id); in virtio_mem_bbm_offline_and_remove_bb()
776 const uint64_t size = vm->bbm.bb_size; in virtio_mem_bbm_offline_and_remove_bb()
778 return virtio_mem_offline_and_remove_memory(vm, addr, size); in virtio_mem_bbm_offline_and_remove_bb()
784 static void virtio_mem_retry(struct virtio_mem *vm) in virtio_mem_retry() argument
788 spin_lock_irqsave(&vm->removal_lock, flags); in virtio_mem_retry()
789 if (!vm->removing) in virtio_mem_retry()
790 queue_work(system_freezable_wq, &vm->wq); in virtio_mem_retry()
791 spin_unlock_irqrestore(&vm->removal_lock, flags); in virtio_mem_retry()
794 static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id) in virtio_mem_translate_node_id() argument
799 if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM)) in virtio_mem_translate_node_id()
809 static bool virtio_mem_overlaps_range(struct virtio_mem *vm, uint64_t start, in virtio_mem_overlaps_range() argument
812 return start < vm->addr + vm->region_size && vm->addr < start + size; in virtio_mem_overlaps_range()
819 static bool virtio_mem_contains_range(struct virtio_mem *vm, uint64_t start, in virtio_mem_contains_range() argument
822 return start >= vm->addr && start + size <= vm->addr + vm->region_size; in virtio_mem_contains_range()
825 static int virtio_mem_sbm_notify_going_online(struct virtio_mem *vm, in virtio_mem_sbm_notify_going_online() argument
828 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) { in virtio_mem_sbm_notify_going_online()
835 dev_warn_ratelimited(&vm->vdev->dev, in virtio_mem_sbm_notify_going_online()
840 static void virtio_mem_sbm_notify_offline(struct virtio_mem *vm, in virtio_mem_sbm_notify_offline() argument
843 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) { in virtio_mem_sbm_notify_offline()
846 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_notify_offline()
851 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_notify_offline()
860 static void virtio_mem_sbm_notify_online(struct virtio_mem *vm, in virtio_mem_sbm_notify_online() argument
868 switch (virtio_mem_sbm_get_mb_state(vm, mb_id)) { in virtio_mem_sbm_notify_online()
883 virtio_mem_sbm_set_mb_state(vm, mb_id, new_state); in virtio_mem_sbm_notify_online()
886 static void virtio_mem_sbm_notify_going_offline(struct virtio_mem *vm, in virtio_mem_sbm_notify_going_offline() argument
889 const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size); in virtio_mem_sbm_notify_going_offline()
893 for (sb_id = 0; sb_id < vm->sbm.sbs_per_mb; sb_id++) { in virtio_mem_sbm_notify_going_offline()
894 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1)) in virtio_mem_sbm_notify_going_offline()
897 sb_id * vm->sbm.sb_size); in virtio_mem_sbm_notify_going_offline()
902 static void virtio_mem_sbm_notify_cancel_offline(struct virtio_mem *vm, in virtio_mem_sbm_notify_cancel_offline() argument
905 const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size); in virtio_mem_sbm_notify_cancel_offline()
909 for (sb_id = 0; sb_id < vm->sbm.sbs_per_mb; sb_id++) { in virtio_mem_sbm_notify_cancel_offline()
910 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1)) in virtio_mem_sbm_notify_cancel_offline()
913 sb_id * vm->sbm.sb_size); in virtio_mem_sbm_notify_cancel_offline()
918 static void virtio_mem_bbm_notify_going_offline(struct virtio_mem *vm, in virtio_mem_bbm_notify_going_offline() argument
927 if (virtio_mem_bbm_get_bb_state(vm, bb_id) != in virtio_mem_bbm_notify_going_offline()
933 static void virtio_mem_bbm_notify_cancel_offline(struct virtio_mem *vm, in virtio_mem_bbm_notify_cancel_offline() argument
938 if (virtio_mem_bbm_get_bb_state(vm, bb_id) != in virtio_mem_bbm_notify_cancel_offline()
952 struct virtio_mem *vm = container_of(nb, struct virtio_mem, in virtio_mem_memory_notifier_cb() local
960 if (!virtio_mem_overlaps_range(vm, start, size)) in virtio_mem_memory_notifier_cb()
963 if (vm->in_sbm) { in virtio_mem_memory_notifier_cb()
974 id = virtio_mem_phys_to_bb_id(vm, start); in virtio_mem_memory_notifier_cb()
981 if (WARN_ON_ONCE(id != virtio_mem_phys_to_bb_id(vm, start + size - 1))) in virtio_mem_memory_notifier_cb()
995 mutex_lock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
996 if (vm->removing) { in virtio_mem_memory_notifier_cb()
998 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1001 vm->hotplug_active = true; in virtio_mem_memory_notifier_cb()
1002 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1003 virtio_mem_sbm_notify_going_offline(vm, id); in virtio_mem_memory_notifier_cb()
1005 virtio_mem_bbm_notify_going_offline(vm, id, in virtio_mem_memory_notifier_cb()
1010 mutex_lock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1011 if (vm->removing) { in virtio_mem_memory_notifier_cb()
1013 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1016 vm->hotplug_active = true; in virtio_mem_memory_notifier_cb()
1017 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1018 rc = virtio_mem_sbm_notify_going_online(vm, id); in virtio_mem_memory_notifier_cb()
1021 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1022 virtio_mem_sbm_notify_offline(vm, id); in virtio_mem_memory_notifier_cb()
1024 atomic64_add(size, &vm->offline_size); in virtio_mem_memory_notifier_cb()
1030 virtio_mem_retry(vm); in virtio_mem_memory_notifier_cb()
1032 vm->hotplug_active = false; in virtio_mem_memory_notifier_cb()
1033 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1036 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1037 virtio_mem_sbm_notify_online(vm, id, mhp->start_pfn); in virtio_mem_memory_notifier_cb()
1039 atomic64_sub(size, &vm->offline_size); in virtio_mem_memory_notifier_cb()
1046 if (!atomic_read(&vm->wq_active) && in virtio_mem_memory_notifier_cb()
1047 virtio_mem_could_add_memory(vm, vm->offline_threshold / 2)) in virtio_mem_memory_notifier_cb()
1048 virtio_mem_retry(vm); in virtio_mem_memory_notifier_cb()
1050 vm->hotplug_active = false; in virtio_mem_memory_notifier_cb()
1051 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1054 if (!vm->hotplug_active) in virtio_mem_memory_notifier_cb()
1056 if (vm->in_sbm) in virtio_mem_memory_notifier_cb()
1057 virtio_mem_sbm_notify_cancel_offline(vm, id); in virtio_mem_memory_notifier_cb()
1059 virtio_mem_bbm_notify_cancel_offline(vm, id, in virtio_mem_memory_notifier_cb()
1062 vm->hotplug_active = false; in virtio_mem_memory_notifier_cb()
1063 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1066 if (!vm->hotplug_active) in virtio_mem_memory_notifier_cb()
1068 vm->hotplug_active = false; in virtio_mem_memory_notifier_cb()
1069 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_memory_notifier_cb()
1235 struct virtio_mem *vm; in virtio_mem_online_page_cb() local
1239 list_for_each_entry_rcu(vm, &virtio_mem_devices, next) { in virtio_mem_online_page_cb()
1240 if (!virtio_mem_contains_range(vm, addr, PFN_PHYS(1 << order))) in virtio_mem_online_page_cb()
1243 if (vm->in_sbm) { in virtio_mem_online_page_cb()
1250 sb_id = virtio_mem_phys_to_sb_id(vm, addr); in virtio_mem_online_page_cb()
1251 do_online = virtio_mem_sbm_test_sb_plugged(vm, id, in virtio_mem_online_page_cb()
1258 id = virtio_mem_phys_to_bb_id(vm, addr); in virtio_mem_online_page_cb()
1259 do_online = virtio_mem_bbm_get_bb_state(vm, id) != in virtio_mem_online_page_cb()
1283 static uint64_t virtio_mem_send_request(struct virtio_mem *vm, in virtio_mem_send_request() argument
1291 vm->req = *req; in virtio_mem_send_request()
1294 sg_init_one(&sg_req, &vm->req, sizeof(vm->req)); in virtio_mem_send_request()
1298 sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp)); in virtio_mem_send_request()
1301 rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL); in virtio_mem_send_request()
1305 virtqueue_kick(vm->vq); in virtio_mem_send_request()
1308 wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len)); in virtio_mem_send_request()
1310 return virtio16_to_cpu(vm->vdev, vm->resp.type); in virtio_mem_send_request()
1313 static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr, in virtio_mem_send_plug_request() argument
1316 const uint64_t nb_vm_blocks = size / vm->device_block_size; in virtio_mem_send_plug_request()
1318 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG), in virtio_mem_send_plug_request()
1319 .u.plug.addr = cpu_to_virtio64(vm->vdev, addr), in virtio_mem_send_plug_request()
1320 .u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks), in virtio_mem_send_plug_request()
1324 if (atomic_read(&vm->config_changed)) in virtio_mem_send_plug_request()
1327 dev_dbg(&vm->vdev->dev, "plugging memory: 0x%llx - 0x%llx\n", addr, in virtio_mem_send_plug_request()
1330 switch (virtio_mem_send_request(vm, &req)) { in virtio_mem_send_plug_request()
1332 vm->plugged_size += size; in virtio_mem_send_plug_request()
1347 dev_dbg(&vm->vdev->dev, "plugging memory failed: %d\n", rc); in virtio_mem_send_plug_request()
1351 static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr, in virtio_mem_send_unplug_request() argument
1354 const uint64_t nb_vm_blocks = size / vm->device_block_size; in virtio_mem_send_unplug_request()
1356 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG), in virtio_mem_send_unplug_request()
1357 .u.unplug.addr = cpu_to_virtio64(vm->vdev, addr), in virtio_mem_send_unplug_request()
1358 .u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks), in virtio_mem_send_unplug_request()
1362 if (atomic_read(&vm->config_changed)) in virtio_mem_send_unplug_request()
1365 dev_dbg(&vm->vdev->dev, "unplugging memory: 0x%llx - 0x%llx\n", addr, in virtio_mem_send_unplug_request()
1368 switch (virtio_mem_send_request(vm, &req)) { in virtio_mem_send_unplug_request()
1370 vm->plugged_size -= size; in virtio_mem_send_unplug_request()
1382 dev_dbg(&vm->vdev->dev, "unplugging memory failed: %d\n", rc); in virtio_mem_send_unplug_request()
1386 static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm) in virtio_mem_send_unplug_all_request() argument
1389 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL), in virtio_mem_send_unplug_all_request()
1393 dev_dbg(&vm->vdev->dev, "unplugging all memory"); in virtio_mem_send_unplug_all_request()
1395 switch (virtio_mem_send_request(vm, &req)) { in virtio_mem_send_unplug_all_request()
1397 vm->unplug_all_required = false; in virtio_mem_send_unplug_all_request()
1398 vm->plugged_size = 0; in virtio_mem_send_unplug_all_request()
1400 atomic_set(&vm->config_changed, 1); in virtio_mem_send_unplug_all_request()
1409 dev_dbg(&vm->vdev->dev, "unplugging all memory failed: %d\n", rc); in virtio_mem_send_unplug_all_request()
1417 static int virtio_mem_sbm_plug_sb(struct virtio_mem *vm, unsigned long mb_id, in virtio_mem_sbm_plug_sb() argument
1421 sb_id * vm->sbm.sb_size; in virtio_mem_sbm_plug_sb()
1422 const uint64_t size = count * vm->sbm.sb_size; in virtio_mem_sbm_plug_sb()
1425 rc = virtio_mem_send_plug_request(vm, addr, size); in virtio_mem_sbm_plug_sb()
1427 virtio_mem_sbm_set_sb_plugged(vm, mb_id, sb_id, count); in virtio_mem_sbm_plug_sb()
1435 static int virtio_mem_sbm_unplug_sb(struct virtio_mem *vm, unsigned long mb_id, in virtio_mem_sbm_unplug_sb() argument
1439 sb_id * vm->sbm.sb_size; in virtio_mem_sbm_unplug_sb()
1440 const uint64_t size = count * vm->sbm.sb_size; in virtio_mem_sbm_unplug_sb()
1443 rc = virtio_mem_send_unplug_request(vm, addr, size); in virtio_mem_sbm_unplug_sb()
1445 virtio_mem_sbm_set_sb_unplugged(vm, mb_id, sb_id, count); in virtio_mem_sbm_unplug_sb()
1454 static int virtio_mem_bbm_unplug_bb(struct virtio_mem *vm, unsigned long bb_id) in virtio_mem_bbm_unplug_bb() argument
1456 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id); in virtio_mem_bbm_unplug_bb()
1457 const uint64_t size = vm->bbm.bb_size; in virtio_mem_bbm_unplug_bb()
1459 return virtio_mem_send_unplug_request(vm, addr, size); in virtio_mem_bbm_unplug_bb()
1467 static int virtio_mem_bbm_plug_bb(struct virtio_mem *vm, unsigned long bb_id) in virtio_mem_bbm_plug_bb() argument
1469 const uint64_t addr = virtio_mem_bb_id_to_phys(vm, bb_id); in virtio_mem_bbm_plug_bb()
1470 const uint64_t size = vm->bbm.bb_size; in virtio_mem_bbm_plug_bb()
1472 return virtio_mem_send_plug_request(vm, addr, size); in virtio_mem_bbm_plug_bb()
1484 static int virtio_mem_sbm_unplug_any_sb_raw(struct virtio_mem *vm, in virtio_mem_sbm_unplug_any_sb_raw() argument
1490 sb_id = vm->sbm.sbs_per_mb - 1; in virtio_mem_sbm_unplug_any_sb_raw()
1494 virtio_mem_sbm_test_sb_unplugged(vm, mb_id, sb_id, 1)) in virtio_mem_sbm_unplug_any_sb_raw()
1501 virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) { in virtio_mem_sbm_unplug_any_sb_raw()
1506 rc = virtio_mem_sbm_unplug_sb(vm, mb_id, sb_id, count); in virtio_mem_sbm_unplug_any_sb_raw()
1523 static int virtio_mem_sbm_unplug_mb(struct virtio_mem *vm, unsigned long mb_id) in virtio_mem_sbm_unplug_mb() argument
1525 uint64_t nb_sb = vm->sbm.sbs_per_mb; in virtio_mem_sbm_unplug_mb()
1527 return virtio_mem_sbm_unplug_any_sb_raw(vm, mb_id, &nb_sb); in virtio_mem_sbm_unplug_mb()
1533 static int virtio_mem_sbm_prepare_next_mb(struct virtio_mem *vm, in virtio_mem_sbm_prepare_next_mb() argument
1538 if (vm->sbm.next_mb_id > vm->sbm.last_usable_mb_id) in virtio_mem_sbm_prepare_next_mb()
1542 rc = virtio_mem_sbm_mb_states_prepare_next_mb(vm); in virtio_mem_sbm_prepare_next_mb()
1547 rc = virtio_mem_sbm_sb_states_prepare_next_mb(vm); in virtio_mem_sbm_prepare_next_mb()
1551 vm->sbm.mb_count[VIRTIO_MEM_SBM_MB_UNUSED]++; in virtio_mem_sbm_prepare_next_mb()
1552 *mb_id = vm->sbm.next_mb_id++; in virtio_mem_sbm_prepare_next_mb()
1562 static int virtio_mem_sbm_plug_and_add_mb(struct virtio_mem *vm, in virtio_mem_sbm_plug_and_add_mb() argument
1565 const int count = min_t(int, *nb_sb, vm->sbm.sbs_per_mb); in virtio_mem_sbm_plug_and_add_mb()
1575 rc = virtio_mem_sbm_plug_sb(vm, mb_id, 0, count); in virtio_mem_sbm_plug_and_add_mb()
1583 if (count == vm->sbm.sbs_per_mb) in virtio_mem_sbm_plug_and_add_mb()
1584 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_plug_and_add_mb()
1587 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_plug_and_add_mb()
1591 rc = virtio_mem_sbm_add_mb(vm, mb_id); in virtio_mem_sbm_plug_and_add_mb()
1595 if (virtio_mem_sbm_unplug_sb(vm, mb_id, 0, count)) in virtio_mem_sbm_plug_and_add_mb()
1597 virtio_mem_sbm_set_mb_state(vm, mb_id, new_state); in virtio_mem_sbm_plug_and_add_mb()
1613 static int virtio_mem_sbm_plug_any_sb(struct virtio_mem *vm, in virtio_mem_sbm_plug_any_sb() argument
1616 const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id); in virtio_mem_sbm_plug_any_sb()
1625 sb_id = virtio_mem_sbm_first_unplugged_sb(vm, mb_id); in virtio_mem_sbm_plug_any_sb()
1626 if (sb_id >= vm->sbm.sbs_per_mb) in virtio_mem_sbm_plug_any_sb()
1630 sb_id + count < vm->sbm.sbs_per_mb && in virtio_mem_sbm_plug_any_sb()
1631 !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id + count, 1)) in virtio_mem_sbm_plug_any_sb()
1634 rc = virtio_mem_sbm_plug_sb(vm, mb_id, sb_id, count); in virtio_mem_sbm_plug_any_sb()
1643 sb_id * vm->sbm.sb_size); in virtio_mem_sbm_plug_any_sb()
1644 nr_pages = PFN_DOWN(count * vm->sbm.sb_size); in virtio_mem_sbm_plug_any_sb()
1648 if (virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) in virtio_mem_sbm_plug_any_sb()
1649 virtio_mem_sbm_set_mb_state(vm, mb_id, old_state - 1); in virtio_mem_sbm_plug_any_sb()
1654 static int virtio_mem_sbm_plug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_sbm_plug_request() argument
1661 uint64_t nb_sb = diff / vm->sbm.sb_size; in virtio_mem_sbm_plug_request()
1669 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_plug_request()
1672 virtio_mem_sbm_for_each_mb(vm, mb_id, mb_states[i]) { in virtio_mem_sbm_plug_request()
1673 rc = virtio_mem_sbm_plug_any_sb(vm, mb_id, &nb_sb); in virtio_mem_sbm_plug_request()
1684 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_plug_request()
1687 virtio_mem_sbm_for_each_mb(vm, mb_id, VIRTIO_MEM_SBM_MB_UNUSED) { in virtio_mem_sbm_plug_request()
1688 if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes())) in virtio_mem_sbm_plug_request()
1691 rc = virtio_mem_sbm_plug_and_add_mb(vm, mb_id, &nb_sb); in virtio_mem_sbm_plug_request()
1699 if (!virtio_mem_could_add_memory(vm, memory_block_size_bytes())) in virtio_mem_sbm_plug_request()
1702 rc = virtio_mem_sbm_prepare_next_mb(vm, &mb_id); in virtio_mem_sbm_plug_request()
1705 rc = virtio_mem_sbm_plug_and_add_mb(vm, mb_id, &nb_sb); in virtio_mem_sbm_plug_request()
1713 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_plug_request()
1722 static int virtio_mem_bbm_plug_and_add_bb(struct virtio_mem *vm, in virtio_mem_bbm_plug_and_add_bb() argument
1727 if (WARN_ON_ONCE(virtio_mem_bbm_get_bb_state(vm, bb_id) != in virtio_mem_bbm_plug_and_add_bb()
1731 rc = virtio_mem_bbm_plug_bb(vm, bb_id); in virtio_mem_bbm_plug_and_add_bb()
1734 virtio_mem_bbm_set_bb_state(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED); in virtio_mem_bbm_plug_and_add_bb()
1736 rc = virtio_mem_bbm_add_bb(vm, bb_id); in virtio_mem_bbm_plug_and_add_bb()
1738 if (!virtio_mem_bbm_unplug_bb(vm, bb_id)) in virtio_mem_bbm_plug_and_add_bb()
1739 virtio_mem_bbm_set_bb_state(vm, bb_id, in virtio_mem_bbm_plug_and_add_bb()
1743 virtio_mem_bbm_set_bb_state(vm, bb_id, in virtio_mem_bbm_plug_and_add_bb()
1753 static int virtio_mem_bbm_prepare_next_bb(struct virtio_mem *vm, in virtio_mem_bbm_prepare_next_bb() argument
1758 if (vm->bbm.next_bb_id > vm->bbm.last_usable_bb_id) in virtio_mem_bbm_prepare_next_bb()
1762 rc = virtio_mem_bbm_bb_states_prepare_next_bb(vm); in virtio_mem_bbm_prepare_next_bb()
1766 vm->bbm.bb_count[VIRTIO_MEM_BBM_BB_UNUSED]++; in virtio_mem_bbm_prepare_next_bb()
1767 *bb_id = vm->bbm.next_bb_id; in virtio_mem_bbm_prepare_next_bb()
1768 vm->bbm.next_bb_id++; in virtio_mem_bbm_prepare_next_bb()
1772 static int virtio_mem_bbm_plug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_bbm_plug_request() argument
1774 uint64_t nb_bb = diff / vm->bbm.bb_size; in virtio_mem_bbm_plug_request()
1782 virtio_mem_bbm_for_each_bb(vm, bb_id, VIRTIO_MEM_BBM_BB_UNUSED) { in virtio_mem_bbm_plug_request()
1783 if (!virtio_mem_could_add_memory(vm, vm->bbm.bb_size)) in virtio_mem_bbm_plug_request()
1786 rc = virtio_mem_bbm_plug_and_add_bb(vm, bb_id); in virtio_mem_bbm_plug_request()
1796 if (!virtio_mem_could_add_memory(vm, vm->bbm.bb_size)) in virtio_mem_bbm_plug_request()
1799 rc = virtio_mem_bbm_prepare_next_bb(vm, &bb_id); in virtio_mem_bbm_plug_request()
1802 rc = virtio_mem_bbm_plug_and_add_bb(vm, bb_id); in virtio_mem_bbm_plug_request()
1816 static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_plug_request() argument
1818 if (vm->in_sbm) in virtio_mem_plug_request()
1819 return virtio_mem_sbm_plug_request(vm, diff); in virtio_mem_plug_request()
1820 return virtio_mem_bbm_plug_request(vm, diff); in virtio_mem_plug_request()
1832 static int virtio_mem_sbm_unplug_any_sb_offline(struct virtio_mem *vm, in virtio_mem_sbm_unplug_any_sb_offline() argument
1838 rc = virtio_mem_sbm_unplug_any_sb_raw(vm, mb_id, nb_sb); in virtio_mem_sbm_unplug_any_sb_offline()
1841 if (!virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) in virtio_mem_sbm_unplug_any_sb_offline()
1842 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_unplug_any_sb_offline()
1847 if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) { in virtio_mem_sbm_unplug_any_sb_offline()
1854 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_unplug_any_sb_offline()
1857 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_any_sb_offline()
1858 rc = virtio_mem_sbm_remove_mb(vm, mb_id); in virtio_mem_sbm_unplug_any_sb_offline()
1860 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_any_sb_offline()
1870 static int virtio_mem_sbm_unplug_sb_online(struct virtio_mem *vm, in virtio_mem_sbm_unplug_sb_online() argument
1874 const unsigned long nr_pages = PFN_DOWN(vm->sbm.sb_size) * count; in virtio_mem_sbm_unplug_sb_online()
1875 const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id); in virtio_mem_sbm_unplug_sb_online()
1880 sb_id * vm->sbm.sb_size); in virtio_mem_sbm_unplug_sb_online()
1887 rc = virtio_mem_sbm_unplug_sb(vm, mb_id, sb_id, count); in virtio_mem_sbm_unplug_sb_online()
1896 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_unplug_sb_online()
1900 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_unplug_sb_online()
1918 static int virtio_mem_sbm_unplug_any_sb_online(struct virtio_mem *vm, in virtio_mem_sbm_unplug_any_sb_online() argument
1925 if (*nb_sb >= vm->sbm.sbs_per_mb && in virtio_mem_sbm_unplug_any_sb_online()
1926 virtio_mem_sbm_test_sb_plugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) { in virtio_mem_sbm_unplug_any_sb_online()
1927 rc = virtio_mem_sbm_unplug_sb_online(vm, mb_id, 0, in virtio_mem_sbm_unplug_any_sb_online()
1928 vm->sbm.sbs_per_mb); in virtio_mem_sbm_unplug_any_sb_online()
1930 *nb_sb -= vm->sbm.sbs_per_mb; in virtio_mem_sbm_unplug_any_sb_online()
1937 for (sb_id = vm->sbm.sbs_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) { in virtio_mem_sbm_unplug_any_sb_online()
1940 !virtio_mem_sbm_test_sb_plugged(vm, mb_id, sb_id, 1)) in virtio_mem_sbm_unplug_any_sb_online()
1945 rc = virtio_mem_sbm_unplug_sb_online(vm, mb_id, sb_id, 1); in virtio_mem_sbm_unplug_any_sb_online()
1959 if (virtio_mem_sbm_test_sb_unplugged(vm, mb_id, 0, vm->sbm.sbs_per_mb)) { in virtio_mem_sbm_unplug_any_sb_online()
1960 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_any_sb_online()
1961 rc = virtio_mem_sbm_offline_and_remove_mb(vm, mb_id); in virtio_mem_sbm_unplug_any_sb_online()
1962 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_any_sb_online()
1964 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_sbm_unplug_any_sb_online()
1983 static int virtio_mem_sbm_unplug_any_sb(struct virtio_mem *vm, in virtio_mem_sbm_unplug_any_sb() argument
1987 const int old_state = virtio_mem_sbm_get_mb_state(vm, mb_id); in virtio_mem_sbm_unplug_any_sb()
1994 return virtio_mem_sbm_unplug_any_sb_online(vm, mb_id, nb_sb); in virtio_mem_sbm_unplug_any_sb()
1997 return virtio_mem_sbm_unplug_any_sb_offline(vm, mb_id, nb_sb); in virtio_mem_sbm_unplug_any_sb()
2002 static int virtio_mem_sbm_unplug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_sbm_unplug_request() argument
2012 uint64_t nb_sb = diff / vm->sbm.sb_size; in virtio_mem_sbm_unplug_request()
2024 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2034 virtio_mem_sbm_for_each_mb_rev(vm, mb_id, mb_states[i]) { in virtio_mem_sbm_unplug_request()
2035 rc = virtio_mem_sbm_unplug_any_sb(vm, mb_id, &nb_sb); in virtio_mem_sbm_unplug_request()
2038 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2040 mutex_lock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2043 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2048 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2051 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_sbm_unplug_request()
2062 static int virtio_mem_bbm_offline_remove_and_unplug_bb(struct virtio_mem *vm, in virtio_mem_bbm_offline_remove_and_unplug_bb() argument
2065 const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id)); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2066 const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2072 if (WARN_ON_ONCE(virtio_mem_bbm_get_bb_state(vm, bb_id) != in virtio_mem_bbm_offline_remove_and_unplug_bb()
2083 mutex_lock(&vm->hotplug_mutex); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2084 virtio_mem_bbm_set_bb_state(vm, bb_id, in virtio_mem_bbm_offline_remove_and_unplug_bb()
2098 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2101 rc = virtio_mem_bbm_offline_and_remove_bb(vm, bb_id); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2104 mutex_lock(&vm->hotplug_mutex); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2110 rc = virtio_mem_bbm_unplug_bb(vm, bb_id); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2112 virtio_mem_bbm_set_bb_state(vm, bb_id, in virtio_mem_bbm_offline_remove_and_unplug_bb()
2115 virtio_mem_bbm_set_bb_state(vm, bb_id, in virtio_mem_bbm_offline_remove_and_unplug_bb()
2126 virtio_mem_bbm_set_bb_state(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2127 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_bbm_offline_remove_and_unplug_bb()
2134 static bool virtio_mem_bbm_bb_is_offline(struct virtio_mem *vm, in virtio_mem_bbm_bb_is_offline() argument
2137 const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id)); in virtio_mem_bbm_bb_is_offline()
2138 const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size); in virtio_mem_bbm_bb_is_offline()
2153 static bool virtio_mem_bbm_bb_is_movable(struct virtio_mem *vm, in virtio_mem_bbm_bb_is_movable() argument
2156 const unsigned long start_pfn = PFN_DOWN(virtio_mem_bb_id_to_phys(vm, bb_id)); in virtio_mem_bbm_bb_is_movable()
2157 const unsigned long nr_pages = PFN_DOWN(vm->bbm.bb_size); in virtio_mem_bbm_bb_is_movable()
2173 static int virtio_mem_bbm_unplug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_bbm_unplug_request() argument
2175 uint64_t nb_bb = diff / vm->bbm.bb_size; in virtio_mem_bbm_unplug_request()
2187 virtio_mem_bbm_for_each_bb_rev(vm, bb_id, VIRTIO_MEM_BBM_BB_ADDED) { in virtio_mem_bbm_unplug_request()
2194 if (i == 0 && !virtio_mem_bbm_bb_is_offline(vm, bb_id)) in virtio_mem_bbm_unplug_request()
2196 if (i == 1 && !virtio_mem_bbm_bb_is_movable(vm, bb_id)) in virtio_mem_bbm_unplug_request()
2198 rc = virtio_mem_bbm_offline_remove_and_unplug_bb(vm, bb_id); in virtio_mem_bbm_unplug_request()
2216 static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff) in virtio_mem_unplug_request() argument
2218 if (vm->in_sbm) in virtio_mem_unplug_request()
2219 return virtio_mem_sbm_unplug_request(vm, diff); in virtio_mem_unplug_request()
2220 return virtio_mem_bbm_unplug_request(vm, diff); in virtio_mem_unplug_request()
2227 static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm) in virtio_mem_unplug_pending_mb() argument
2232 if (!vm->in_sbm) { in virtio_mem_unplug_pending_mb()
2233 virtio_mem_bbm_for_each_bb(vm, id, in virtio_mem_unplug_pending_mb()
2235 rc = virtio_mem_bbm_unplug_bb(vm, id); in virtio_mem_unplug_pending_mb()
2238 virtio_mem_bbm_set_bb_state(vm, id, in virtio_mem_unplug_pending_mb()
2244 virtio_mem_sbm_for_each_mb(vm, id, VIRTIO_MEM_SBM_MB_PLUGGED) { in virtio_mem_unplug_pending_mb()
2245 rc = virtio_mem_sbm_unplug_mb(vm, id); in virtio_mem_unplug_pending_mb()
2248 virtio_mem_sbm_set_mb_state(vm, id, in virtio_mem_unplug_pending_mb()
2258 static void virtio_mem_refresh_config(struct virtio_mem *vm) in virtio_mem_refresh_config() argument
2264 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size, in virtio_mem_refresh_config()
2266 if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size)) in virtio_mem_refresh_config()
2267 vm->plugged_size = new_plugged_size; in virtio_mem_refresh_config()
2270 virtio_cread_le(vm->vdev, struct virtio_mem_config, in virtio_mem_refresh_config()
2272 end_addr = min(vm->addr + usable_region_size - 1, in virtio_mem_refresh_config()
2275 if (vm->in_sbm) { in virtio_mem_refresh_config()
2276 vm->sbm.last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr); in virtio_mem_refresh_config()
2278 vm->sbm.last_usable_mb_id--; in virtio_mem_refresh_config()
2280 vm->bbm.last_usable_bb_id = virtio_mem_phys_to_bb_id(vm, in virtio_mem_refresh_config()
2282 if (!IS_ALIGNED(end_addr + 1, vm->bbm.bb_size)) in virtio_mem_refresh_config()
2283 vm->bbm.last_usable_bb_id--; in virtio_mem_refresh_config()
2293 virtio_cread_le(vm->vdev, struct virtio_mem_config, requested_size, in virtio_mem_refresh_config()
2294 &vm->requested_size); in virtio_mem_refresh_config()
2296 dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size); in virtio_mem_refresh_config()
2297 dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size); in virtio_mem_refresh_config()
2305 struct virtio_mem *vm = container_of(work, struct virtio_mem, wq); in virtio_mem_run_wq() local
2309 if (unlikely(vm->in_kdump)) { in virtio_mem_run_wq()
2310 dev_warn_once(&vm->vdev->dev, in virtio_mem_run_wq()
2315 hrtimer_cancel(&vm->retry_timer); in virtio_mem_run_wq()
2317 if (vm->broken) in virtio_mem_run_wq()
2320 atomic_set(&vm->wq_active, 1); in virtio_mem_run_wq()
2325 if (unlikely(vm->unplug_all_required)) in virtio_mem_run_wq()
2326 rc = virtio_mem_send_unplug_all_request(vm); in virtio_mem_run_wq()
2328 if (atomic_read(&vm->config_changed)) { in virtio_mem_run_wq()
2329 atomic_set(&vm->config_changed, 0); in virtio_mem_run_wq()
2330 virtio_mem_refresh_config(vm); in virtio_mem_run_wq()
2335 rc = virtio_mem_unplug_pending_mb(vm); in virtio_mem_run_wq()
2337 if (!rc && vm->requested_size != vm->plugged_size) { in virtio_mem_run_wq()
2338 if (vm->requested_size > vm->plugged_size) { in virtio_mem_run_wq()
2339 diff = vm->requested_size - vm->plugged_size; in virtio_mem_run_wq()
2340 rc = virtio_mem_plug_request(vm, diff); in virtio_mem_run_wq()
2342 diff = vm->plugged_size - vm->requested_size; in virtio_mem_run_wq()
2343 rc = virtio_mem_unplug_request(vm, diff); in virtio_mem_run_wq()
2349 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS; in virtio_mem_run_wq()
2369 hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms), in virtio_mem_run_wq()
2377 dev_err(&vm->vdev->dev, in virtio_mem_run_wq()
2379 vm->broken = true; in virtio_mem_run_wq()
2382 atomic_set(&vm->wq_active, 0); in virtio_mem_run_wq()
2387 struct virtio_mem *vm = container_of(timer, struct virtio_mem, in virtio_mem_timer_expired() local
2390 virtio_mem_retry(vm); in virtio_mem_timer_expired()
2391 vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2, in virtio_mem_timer_expired()
2398 struct virtio_mem *vm = vq->vdev->priv; in virtio_mem_handle_response() local
2400 wake_up(&vm->host_resp); in virtio_mem_handle_response()
2403 static int virtio_mem_init_vq(struct virtio_mem *vm) in virtio_mem_init_vq() argument
2407 vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response, in virtio_mem_init_vq()
2411 vm->vq = vq; in virtio_mem_init_vq()
2416 static int virtio_mem_init_hotplug(struct virtio_mem *vm) in virtio_mem_init_hotplug() argument
2423 if (!IS_ALIGNED(vm->addr, memory_block_size_bytes())) in virtio_mem_init_hotplug()
2424 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2426 if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes())) in virtio_mem_init_hotplug()
2427 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2429 if (vm->addr < pluggable_range.start || in virtio_mem_init_hotplug()
2430 vm->addr + vm->region_size - 1 > pluggable_range.end) in virtio_mem_init_hotplug()
2431 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2435 vm->offline_threshold = max_t(uint64_t, 2 * memory_block_size_bytes(), in virtio_mem_init_hotplug()
2448 sb_size = max_t(uint64_t, vm->device_block_size, sb_size); in virtio_mem_init_hotplug()
2452 vm->in_sbm = true; in virtio_mem_init_hotplug()
2453 vm->sbm.sb_size = sb_size; in virtio_mem_init_hotplug()
2454 vm->sbm.sbs_per_mb = memory_block_size_bytes() / in virtio_mem_init_hotplug()
2455 vm->sbm.sb_size; in virtio_mem_init_hotplug()
2458 addr = max_t(uint64_t, vm->addr, pluggable_range.start) + in virtio_mem_init_hotplug()
2460 vm->sbm.first_mb_id = virtio_mem_phys_to_mb_id(addr); in virtio_mem_init_hotplug()
2461 vm->sbm.next_mb_id = vm->sbm.first_mb_id; in virtio_mem_init_hotplug()
2464 vm->bbm.bb_size = max_t(uint64_t, vm->device_block_size, in virtio_mem_init_hotplug()
2469 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2471 } else if (bbm_block_size < vm->bbm.bb_size) { in virtio_mem_init_hotplug()
2472 dev_warn(&vm->vdev->dev, in virtio_mem_init_hotplug()
2475 vm->bbm.bb_size = bbm_block_size; in virtio_mem_init_hotplug()
2480 addr = max_t(uint64_t, vm->addr, pluggable_range.start) + in virtio_mem_init_hotplug()
2481 vm->bbm.bb_size - 1; in virtio_mem_init_hotplug()
2482 vm->bbm.first_bb_id = virtio_mem_phys_to_bb_id(vm, addr); in virtio_mem_init_hotplug()
2483 vm->bbm.next_bb_id = vm->bbm.first_bb_id; in virtio_mem_init_hotplug()
2486 vm->offline_threshold = max_t(uint64_t, 2 * vm->bbm.bb_size, in virtio_mem_init_hotplug()
2487 vm->offline_threshold); in virtio_mem_init_hotplug()
2490 dev_info(&vm->vdev->dev, "memory block size: 0x%lx", in virtio_mem_init_hotplug()
2492 if (vm->in_sbm) in virtio_mem_init_hotplug()
2493 dev_info(&vm->vdev->dev, "subblock size: 0x%llx", in virtio_mem_init_hotplug()
2494 (unsigned long long)vm->sbm.sb_size); in virtio_mem_init_hotplug()
2496 dev_info(&vm->vdev->dev, "big block size: 0x%llx", in virtio_mem_init_hotplug()
2497 (unsigned long long)vm->bbm.bb_size); in virtio_mem_init_hotplug()
2500 rc = virtio_mem_create_resource(vm); in virtio_mem_init_hotplug()
2505 if (vm->in_sbm) in virtio_mem_init_hotplug()
2508 unit_pages = PHYS_PFN(vm->bbm.bb_size); in virtio_mem_init_hotplug()
2509 rc = memory_group_register_dynamic(vm->nid, unit_pages); in virtio_mem_init_hotplug()
2512 vm->mgid = rc; in virtio_mem_init_hotplug()
2519 if (vm->plugged_size) { in virtio_mem_init_hotplug()
2520 vm->unplug_all_required = true; in virtio_mem_init_hotplug()
2521 dev_info(&vm->vdev->dev, "unplugging all memory is required\n"); in virtio_mem_init_hotplug()
2525 vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb; in virtio_mem_init_hotplug()
2526 rc = register_memory_notifier(&vm->memory_notifier); in virtio_mem_init_hotplug()
2529 rc = register_virtio_mem_device(vm); in virtio_mem_init_hotplug()
2535 unregister_memory_notifier(&vm->memory_notifier); in virtio_mem_init_hotplug()
2537 memory_group_unregister(vm->mgid); in virtio_mem_init_hotplug()
2539 virtio_mem_delete_resource(vm); in virtio_mem_init_hotplug()
2544 static int virtio_mem_send_state_request(struct virtio_mem *vm, uint64_t addr, in virtio_mem_send_state_request() argument
2547 const uint64_t nb_vm_blocks = size / vm->device_block_size; in virtio_mem_send_state_request()
2549 .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_STATE), in virtio_mem_send_state_request()
2550 .u.state.addr = cpu_to_virtio64(vm->vdev, addr), in virtio_mem_send_state_request()
2551 .u.state.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks), in virtio_mem_send_state_request()
2555 dev_dbg(&vm->vdev->dev, "requesting state: 0x%llx - 0x%llx\n", addr, in virtio_mem_send_state_request()
2558 switch (virtio_mem_send_request(vm, &req)) { in virtio_mem_send_state_request()
2560 return virtio16_to_cpu(vm->vdev, vm->resp.u.state.state); in virtio_mem_send_state_request()
2568 dev_dbg(&vm->vdev->dev, "requesting state failed: %d\n", rc); in virtio_mem_send_state_request()
2575 struct virtio_mem *vm = container_of(cb, struct virtio_mem, in virtio_mem_vmcore_pfn_is_ram() local
2581 if (!virtio_mem_contains_range(vm, addr, PAGE_SIZE)) in virtio_mem_vmcore_pfn_is_ram()
2583 if (!vm->plugged_size) in virtio_mem_vmcore_pfn_is_ram()
2590 mutex_lock(&vm->hotplug_mutex); in virtio_mem_vmcore_pfn_is_ram()
2592 addr = ALIGN_DOWN(addr, vm->device_block_size); in virtio_mem_vmcore_pfn_is_ram()
2593 if (addr != vm->last_block_addr) { in virtio_mem_vmcore_pfn_is_ram()
2594 rc = virtio_mem_send_state_request(vm, addr, in virtio_mem_vmcore_pfn_is_ram()
2595 vm->device_block_size); in virtio_mem_vmcore_pfn_is_ram()
2598 vm->last_block_plugged = true; in virtio_mem_vmcore_pfn_is_ram()
2600 vm->last_block_plugged = false; in virtio_mem_vmcore_pfn_is_ram()
2601 vm->last_block_addr = addr; in virtio_mem_vmcore_pfn_is_ram()
2604 is_ram = vm->last_block_plugged; in virtio_mem_vmcore_pfn_is_ram()
2605 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_vmcore_pfn_is_ram()
2610 static int virtio_mem_init_kdump(struct virtio_mem *vm) in virtio_mem_init_kdump() argument
2613 dev_info(&vm->vdev->dev, "memory hot(un)plug disabled in kdump kernel\n"); in virtio_mem_init_kdump()
2614 vm->vmcore_cb.pfn_is_ram = virtio_mem_vmcore_pfn_is_ram; in virtio_mem_init_kdump()
2615 register_vmcore_cb(&vm->vmcore_cb); in virtio_mem_init_kdump()
2618 dev_warn(&vm->vdev->dev, "disabled in kdump kernel without vmcore\n"); in virtio_mem_init_kdump()
2623 static int virtio_mem_init(struct virtio_mem *vm) in virtio_mem_init() argument
2627 if (!vm->vdev->config->get) { in virtio_mem_init()
2628 dev_err(&vm->vdev->dev, "config access disabled\n"); in virtio_mem_init()
2633 virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size, in virtio_mem_init()
2634 &vm->plugged_size); in virtio_mem_init()
2635 virtio_cread_le(vm->vdev, struct virtio_mem_config, block_size, in virtio_mem_init()
2636 &vm->device_block_size); in virtio_mem_init()
2637 virtio_cread_le(vm->vdev, struct virtio_mem_config, node_id, in virtio_mem_init()
2639 vm->nid = virtio_mem_translate_node_id(vm, node_id); in virtio_mem_init()
2640 virtio_cread_le(vm->vdev, struct virtio_mem_config, addr, &vm->addr); in virtio_mem_init()
2641 virtio_cread_le(vm->vdev, struct virtio_mem_config, region_size, in virtio_mem_init()
2642 &vm->region_size); in virtio_mem_init()
2645 if (vm->nid == NUMA_NO_NODE) in virtio_mem_init()
2646 vm->nid = memory_add_physaddr_to_nid(vm->addr); in virtio_mem_init()
2648 dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr); in virtio_mem_init()
2649 dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size); in virtio_mem_init()
2650 dev_info(&vm->vdev->dev, "device block size: 0x%llx", in virtio_mem_init()
2651 (unsigned long long)vm->device_block_size); in virtio_mem_init()
2652 if (vm->nid != NUMA_NO_NODE && IS_ENABLED(CONFIG_NUMA)) in virtio_mem_init()
2653 dev_info(&vm->vdev->dev, "nid: %d", vm->nid); in virtio_mem_init()
2659 if (vm->in_kdump) in virtio_mem_init()
2660 return virtio_mem_init_kdump(vm); in virtio_mem_init()
2661 return virtio_mem_init_hotplug(vm); in virtio_mem_init()
2664 static int virtio_mem_create_resource(struct virtio_mem *vm) in virtio_mem_create_resource() argument
2670 const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL); in virtio_mem_create_resource()
2676 vm->parent_resource = __request_mem_region(vm->addr, vm->region_size, in virtio_mem_create_resource()
2679 if (!vm->parent_resource) { in virtio_mem_create_resource()
2681 dev_warn(&vm->vdev->dev, "could not reserve device region\n"); in virtio_mem_create_resource()
2682 dev_info(&vm->vdev->dev, in virtio_mem_create_resource()
2688 vm->parent_resource->flags &= ~IORESOURCE_BUSY; in virtio_mem_create_resource()
2692 static void virtio_mem_delete_resource(struct virtio_mem *vm) in virtio_mem_delete_resource() argument
2696 if (!vm->parent_resource) in virtio_mem_delete_resource()
2699 name = vm->parent_resource->name; in virtio_mem_delete_resource()
2700 release_resource(vm->parent_resource); in virtio_mem_delete_resource()
2701 kfree(vm->parent_resource); in virtio_mem_delete_resource()
2703 vm->parent_resource = NULL; in virtio_mem_delete_resource()
2711 static bool virtio_mem_has_memory_added(struct virtio_mem *vm) in virtio_mem_has_memory_added() argument
2715 return walk_iomem_res_desc(IORES_DESC_NONE, flags, vm->addr, in virtio_mem_has_memory_added()
2716 vm->addr + vm->region_size, NULL, in virtio_mem_has_memory_added()
2722 struct virtio_mem *vm; in virtio_mem_probe() local
2728 vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL); in virtio_mem_probe()
2729 if (!vm) in virtio_mem_probe()
2732 init_waitqueue_head(&vm->host_resp); in virtio_mem_probe()
2733 vm->vdev = vdev; in virtio_mem_probe()
2734 INIT_WORK(&vm->wq, virtio_mem_run_wq); in virtio_mem_probe()
2735 mutex_init(&vm->hotplug_mutex); in virtio_mem_probe()
2736 INIT_LIST_HEAD(&vm->next); in virtio_mem_probe()
2737 spin_lock_init(&vm->removal_lock); in virtio_mem_probe()
2738 hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in virtio_mem_probe()
2739 vm->retry_timer.function = virtio_mem_timer_expired; in virtio_mem_probe()
2740 vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS; in virtio_mem_probe()
2741 vm->in_kdump = is_kdump_kernel(); in virtio_mem_probe()
2744 rc = virtio_mem_init_vq(vm); in virtio_mem_probe()
2749 rc = virtio_mem_init(vm); in virtio_mem_probe()
2756 if (!vm->in_kdump) { in virtio_mem_probe()
2757 atomic_set(&vm->config_changed, 1); in virtio_mem_probe()
2758 queue_work(system_freezable_wq, &vm->wq); in virtio_mem_probe()
2765 kfree(vm); in virtio_mem_probe()
2771 static void virtio_mem_deinit_hotplug(struct virtio_mem *vm) in virtio_mem_deinit_hotplug() argument
2780 mutex_lock(&vm->hotplug_mutex); in virtio_mem_deinit_hotplug()
2781 spin_lock_irq(&vm->removal_lock); in virtio_mem_deinit_hotplug()
2782 vm->removing = true; in virtio_mem_deinit_hotplug()
2783 spin_unlock_irq(&vm->removal_lock); in virtio_mem_deinit_hotplug()
2784 mutex_unlock(&vm->hotplug_mutex); in virtio_mem_deinit_hotplug()
2787 cancel_work_sync(&vm->wq); in virtio_mem_deinit_hotplug()
2788 hrtimer_cancel(&vm->retry_timer); in virtio_mem_deinit_hotplug()
2790 if (vm->in_sbm) { in virtio_mem_deinit_hotplug()
2795 virtio_mem_sbm_for_each_mb(vm, mb_id, in virtio_mem_deinit_hotplug()
2797 rc = virtio_mem_sbm_remove_mb(vm, mb_id); in virtio_mem_deinit_hotplug()
2799 virtio_mem_sbm_set_mb_state(vm, mb_id, in virtio_mem_deinit_hotplug()
2810 unregister_virtio_mem_device(vm); in virtio_mem_deinit_hotplug()
2811 unregister_memory_notifier(&vm->memory_notifier); in virtio_mem_deinit_hotplug()
2818 if (virtio_mem_has_memory_added(vm)) { in virtio_mem_deinit_hotplug()
2819 dev_warn(&vm->vdev->dev, in virtio_mem_deinit_hotplug()
2822 virtio_mem_delete_resource(vm); in virtio_mem_deinit_hotplug()
2823 kfree_const(vm->resource_name); in virtio_mem_deinit_hotplug()
2824 memory_group_unregister(vm->mgid); in virtio_mem_deinit_hotplug()
2828 if (vm->in_sbm) { in virtio_mem_deinit_hotplug()
2829 vfree(vm->sbm.mb_states); in virtio_mem_deinit_hotplug()
2830 vfree(vm->sbm.sb_states); in virtio_mem_deinit_hotplug()
2832 vfree(vm->bbm.bb_states); in virtio_mem_deinit_hotplug()
2836 static void virtio_mem_deinit_kdump(struct virtio_mem *vm) in virtio_mem_deinit_kdump() argument
2839 unregister_vmcore_cb(&vm->vmcore_cb); in virtio_mem_deinit_kdump()
2845 struct virtio_mem *vm = vdev->priv; in virtio_mem_remove() local
2847 if (vm->in_kdump) in virtio_mem_remove()
2848 virtio_mem_deinit_kdump(vm); in virtio_mem_remove()
2850 virtio_mem_deinit_hotplug(vm); in virtio_mem_remove()
2856 kfree(vm); in virtio_mem_remove()
2862 struct virtio_mem *vm = vdev->priv; in virtio_mem_config_changed() local
2864 if (unlikely(vm->in_kdump)) in virtio_mem_config_changed()
2867 atomic_set(&vm->config_changed, 1); in virtio_mem_config_changed()
2868 virtio_mem_retry(vm); in virtio_mem_config_changed()