Lines Matching refs:dmirror
59 struct dmirror *dmirror; member
66 struct dmirror { struct
122 struct dmirror *dmirror; in dmirror_fops_open() local
126 dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL); in dmirror_fops_open()
127 if (dmirror == NULL) in dmirror_fops_open()
130 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice); in dmirror_fops_open()
131 mutex_init(&dmirror->mutex); in dmirror_fops_open()
132 xa_init(&dmirror->pt); in dmirror_fops_open()
134 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm, in dmirror_fops_open()
137 kfree(dmirror); in dmirror_fops_open()
141 filp->private_data = dmirror; in dmirror_fops_open()
147 struct dmirror *dmirror = filp->private_data; in dmirror_fops_release() local
149 mmu_interval_notifier_remove(&dmirror->notifier); in dmirror_fops_release()
150 xa_destroy(&dmirror->pt); in dmirror_fops_release()
151 kfree(dmirror); in dmirror_fops_release()
162 static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) in dmirror_do_fault() argument
188 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); in dmirror_do_fault()
196 static void dmirror_do_update(struct dmirror *dmirror, unsigned long start, in dmirror_do_update() argument
207 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT, in dmirror_do_update()
209 xa_erase(&dmirror->pt, pfn); in dmirror_do_update()
216 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier); in dmirror_interval_invalidate() local
223 range->owner == dmirror->mdevice) in dmirror_interval_invalidate()
227 mutex_lock(&dmirror->mutex); in dmirror_interval_invalidate()
228 else if (!mutex_trylock(&dmirror->mutex)) in dmirror_interval_invalidate()
232 dmirror_do_update(dmirror, range->start, range->end); in dmirror_interval_invalidate()
234 mutex_unlock(&dmirror->mutex); in dmirror_interval_invalidate()
242 static int dmirror_range_fault(struct dmirror *dmirror, in dmirror_range_fault() argument
245 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_range_fault()
266 mutex_lock(&dmirror->mutex); in dmirror_range_fault()
269 mutex_unlock(&dmirror->mutex); in dmirror_range_fault()
275 ret = dmirror_do_fault(dmirror, range); in dmirror_range_fault()
277 mutex_unlock(&dmirror->mutex); in dmirror_range_fault()
282 static int dmirror_fault(struct dmirror *dmirror, unsigned long start, in dmirror_fault() argument
285 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_fault()
289 .notifier = &dmirror->notifier, in dmirror_fault()
294 .dev_private_owner = dmirror->mdevice, in dmirror_fault()
306 ret = dmirror_range_fault(dmirror, &range); in dmirror_fault()
315 static int dmirror_do_read(struct dmirror *dmirror, unsigned long start, in dmirror_do_read() argument
328 entry = xa_load(&dmirror->pt, pfn); in dmirror_do_read()
344 static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) in dmirror_read() argument
361 mutex_lock(&dmirror->mutex); in dmirror_read()
362 ret = dmirror_do_read(dmirror, start, end, &bounce); in dmirror_read()
363 mutex_unlock(&dmirror->mutex); in dmirror_read()
368 ret = dmirror_fault(dmirror, start, end, false); in dmirror_read()
384 static int dmirror_do_write(struct dmirror *dmirror, unsigned long start, in dmirror_do_write() argument
397 entry = xa_load(&dmirror->pt, pfn); in dmirror_do_write()
413 static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) in dmirror_write() argument
435 mutex_lock(&dmirror->mutex); in dmirror_write()
436 ret = dmirror_do_write(dmirror, start, end, &bounce); in dmirror_write()
437 mutex_unlock(&dmirror->mutex); in dmirror_write()
442 ret = dmirror_fault(dmirror, start, end, true); in dmirror_write()
576 struct dmirror *dmirror) in dmirror_migrate_alloc_and_copy() argument
578 struct dmirror_device *mdevice = dmirror->mdevice; in dmirror_migrate_alloc_and_copy()
614 rpage->zone_device_data = dmirror; in dmirror_migrate_alloc_and_copy()
623 static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, in dmirror_check_atomic() argument
631 entry = xa_load(&dmirror->pt, pfn); in dmirror_check_atomic()
640 struct page **pages, struct dmirror *dmirror) in dmirror_atomic_map() argument
646 mutex_lock(&dmirror->mutex); in dmirror_atomic_map()
656 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); in dmirror_atomic_map()
658 mutex_unlock(&dmirror->mutex); in dmirror_atomic_map()
665 mutex_unlock(&dmirror->mutex); in dmirror_atomic_map()
670 struct dmirror *dmirror) in dmirror_migrate_finalize_and_map() argument
679 mutex_lock(&dmirror->mutex); in dmirror_migrate_finalize_and_map()
700 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); in dmirror_migrate_finalize_and_map()
702 mutex_unlock(&dmirror->mutex); in dmirror_migrate_finalize_and_map()
707 mutex_unlock(&dmirror->mutex); in dmirror_migrate_finalize_and_map()
711 static int dmirror_exclusive(struct dmirror *dmirror, in dmirror_exclusive() argument
716 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_exclusive()
742 mapped = dmirror_atomic_map(addr, next, pages, dmirror); in dmirror_exclusive()
763 mutex_lock(&dmirror->mutex); in dmirror_exclusive()
764 ret = dmirror_do_read(dmirror, start, end, &bounce); in dmirror_exclusive()
765 mutex_unlock(&dmirror->mutex); in dmirror_exclusive()
777 static int dmirror_migrate(struct dmirror *dmirror, in dmirror_migrate() argument
782 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_migrate()
816 args.pgmap_owner = dmirror->mdevice; in dmirror_migrate()
822 dmirror_migrate_alloc_and_copy(&args, dmirror); in dmirror_migrate()
824 dmirror_migrate_finalize_and_map(&args, dmirror); in dmirror_migrate()
834 mutex_lock(&dmirror->mutex); in dmirror_migrate()
835 ret = dmirror_do_read(dmirror, start, end, &bounce); in dmirror_migrate()
836 mutex_unlock(&dmirror->mutex); in dmirror_migrate()
852 static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range, in dmirror_mkentry() argument
869 if (dmirror->mdevice == dmirror_page_to_device(page)) in dmirror_mkentry()
893 struct dmirror *dmirror = dmi->dmirror; in dmirror_snapshot_invalidate() local
896 mutex_lock(&dmirror->mutex); in dmirror_snapshot_invalidate()
897 else if (!mutex_trylock(&dmirror->mutex)) in dmirror_snapshot_invalidate()
906 mutex_unlock(&dmirror->mutex); in dmirror_snapshot_invalidate()
914 static int dmirror_range_snapshot(struct dmirror *dmirror, in dmirror_range_snapshot() argument
918 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_range_snapshot()
926 notifier.dmirror = dmirror; in dmirror_range_snapshot()
952 mutex_lock(&dmirror->mutex); in dmirror_range_snapshot()
955 mutex_unlock(&dmirror->mutex); in dmirror_range_snapshot()
963 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]); in dmirror_range_snapshot()
965 mutex_unlock(&dmirror->mutex); in dmirror_range_snapshot()
971 static int dmirror_snapshot(struct dmirror *dmirror, in dmirror_snapshot() argument
974 struct mm_struct *mm = dmirror->notifier.mm; in dmirror_snapshot()
984 .dev_private_owner = dmirror->mdevice, in dmirror_snapshot()
1009 ret = dmirror_range_snapshot(dmirror, &range, perm); in dmirror_snapshot()
1033 struct dmirror *dmirror; in dmirror_fops_unlocked_ioctl() local
1036 dmirror = filp->private_data; in dmirror_fops_unlocked_ioctl()
1037 if (!dmirror) in dmirror_fops_unlocked_ioctl()
1053 ret = dmirror_read(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1057 ret = dmirror_write(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1061 ret = dmirror_migrate(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1065 ret = dmirror_exclusive(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1069 ret = dmirror_check_atomic(dmirror, cmd.addr, in dmirror_fops_unlocked_ioctl()
1074 ret = dmirror_snapshot(dmirror, &cmd); in dmirror_fops_unlocked_ioctl()
1115 struct dmirror *dmirror) in dmirror_devmem_fault_alloc_and_copy() argument
1137 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT); in dmirror_devmem_fault_alloc_and_copy()
1152 struct dmirror *dmirror; in dmirror_devmem_fault() local
1161 dmirror = rpage->zone_device_data; in dmirror_devmem_fault()
1169 args.pgmap_owner = dmirror->mdevice; in dmirror_devmem_fault()
1175 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); in dmirror_devmem_fault()