Lines Matching refs:sg
240 struct gmap *sg, *next; in gmap_remove() local
246 list_for_each_entry_safe(sg, next, &gmap->children, list) { in gmap_remove()
247 list_del(&sg->list); in gmap_remove()
248 gmap_put(sg); in gmap_remove()
1183 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr, in gmap_insert_rmap() argument
1188 BUG_ON(!gmap_is_shadow(sg)); in gmap_insert_rmap()
1189 slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT); in gmap_insert_rmap()
1192 &sg->guest_table_lock); in gmap_insert_rmap()
1193 radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap); in gmap_insert_rmap()
1196 radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT, in gmap_insert_rmap()
1211 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr, in gmap_protect_rmap() argument
1221 BUG_ON(!gmap_is_shadow(sg)); in gmap_protect_rmap()
1222 parent = sg->parent; in gmap_protect_rmap()
1239 spin_lock(&sg->guest_table_lock); in gmap_protect_rmap()
1243 gmap_insert_rmap(sg, vmaddr, rmap); in gmap_protect_rmap()
1244 spin_unlock(&sg->guest_table_lock); in gmap_protect_rmap()
1292 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr) in gmap_unshadow_page() argument
1296 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_page()
1297 table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */ in gmap_unshadow_page()
1300 gmap_call_notifier(sg, raddr, raddr + _PAGE_SIZE - 1); in gmap_unshadow_page()
1301 ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table); in gmap_unshadow_page()
1312 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_pgt() argument
1317 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_pgt()
1329 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_pgt() argument
1334 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_pgt()
1335 ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */ in gmap_unshadow_pgt()
1338 gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1); in gmap_unshadow_pgt()
1343 __gmap_unshadow_pgt(sg, raddr, pgt); in gmap_unshadow_pgt()
1358 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_sgt() argument
1365 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_sgt()
1371 __gmap_unshadow_pgt(sg, raddr, pgt); in __gmap_unshadow_sgt()
1386 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_sgt() argument
1391 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_sgt()
1392 r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */ in gmap_unshadow_sgt()
1395 gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1); in gmap_unshadow_sgt()
1400 __gmap_unshadow_sgt(sg, raddr, sgt); in gmap_unshadow_sgt()
1415 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r3t() argument
1422 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_r3t()
1428 __gmap_unshadow_sgt(sg, raddr, sgt); in __gmap_unshadow_r3t()
1443 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r3t() argument
1448 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_r3t()
1449 r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */ in gmap_unshadow_r3t()
1452 gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1); in gmap_unshadow_r3t()
1457 __gmap_unshadow_r3t(sg, raddr, r3t); in gmap_unshadow_r3t()
1472 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r2t() argument
1479 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_r2t()
1485 __gmap_unshadow_r3t(sg, raddr, r3t); in __gmap_unshadow_r2t()
1500 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r2t() argument
1505 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow_r2t()
1506 r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */ in gmap_unshadow_r2t()
1509 gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1); in gmap_unshadow_r2t()
1514 __gmap_unshadow_r2t(sg, raddr, r2t); in gmap_unshadow_r2t()
1529 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r1t() argument
1536 BUG_ON(!gmap_is_shadow(sg)); in __gmap_unshadow_r1t()
1542 __gmap_unshadow_r2t(sg, raddr, r2t); in __gmap_unshadow_r1t()
1559 static void gmap_unshadow(struct gmap *sg) in gmap_unshadow() argument
1563 BUG_ON(!gmap_is_shadow(sg)); in gmap_unshadow()
1564 if (sg->removed) in gmap_unshadow()
1566 sg->removed = 1; in gmap_unshadow()
1567 gmap_call_notifier(sg, 0, -1UL); in gmap_unshadow()
1568 gmap_flush_tlb(sg); in gmap_unshadow()
1569 table = (unsigned long *)(sg->asce & _ASCE_ORIGIN); in gmap_unshadow()
1570 switch (sg->asce & _ASCE_TYPE_MASK) { in gmap_unshadow()
1572 __gmap_unshadow_r1t(sg, 0, table); in gmap_unshadow()
1575 __gmap_unshadow_r2t(sg, 0, table); in gmap_unshadow()
1578 __gmap_unshadow_r3t(sg, 0, table); in gmap_unshadow()
1581 __gmap_unshadow_sgt(sg, 0, table); in gmap_unshadow()
1599 struct gmap *sg; in gmap_find_shadow() local
1601 list_for_each_entry(sg, &parent->children, list) { in gmap_find_shadow()
1602 if (sg->orig_asce != asce || sg->edat_level != edat_level || in gmap_find_shadow()
1603 sg->removed) in gmap_find_shadow()
1605 if (!sg->initialized) in gmap_find_shadow()
1607 refcount_inc(&sg->ref_count); in gmap_find_shadow()
1608 return sg; in gmap_find_shadow()
1625 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level) in gmap_shadow_valid() argument
1627 if (sg->removed) in gmap_shadow_valid()
1629 return sg->orig_asce == asce && sg->edat_level == edat_level; in gmap_shadow_valid()
1651 struct gmap *sg, *new; in gmap_shadow() local
1658 sg = gmap_find_shadow(parent, asce, edat_level); in gmap_shadow()
1660 if (sg) in gmap_shadow()
1661 return sg; in gmap_shadow()
1676 sg = gmap_find_shadow(parent, asce, edat_level); in gmap_shadow()
1677 if (sg) { in gmap_shadow()
1680 return sg; in gmap_shadow()
1684 list_for_each_entry(sg, &parent->children, list) { in gmap_shadow()
1685 if (sg->orig_asce & _ASCE_REAL_SPACE) { in gmap_shadow()
1686 spin_lock(&sg->guest_table_lock); in gmap_shadow()
1687 gmap_unshadow(sg); in gmap_shadow()
1688 spin_unlock(&sg->guest_table_lock); in gmap_shadow()
1689 list_del(&sg->list); in gmap_shadow()
1690 gmap_put(sg); in gmap_shadow()
1740 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, in gmap_shadow_r2t() argument
1748 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_r2t()
1758 spin_lock(&sg->guest_table_lock); in gmap_shadow_r2t()
1759 table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */ in gmap_shadow_r2t()
1775 if (sg->edat_level >= 1) in gmap_shadow_r2t()
1777 list_add(&page->lru, &sg->crst_list); in gmap_shadow_r2t()
1781 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1784 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1790 rc = gmap_protect_rmap(sg, raddr, origin + offset, len); in gmap_shadow_r2t()
1791 spin_lock(&sg->guest_table_lock); in gmap_shadow_r2t()
1793 table = gmap_table_walk(sg, saddr, 4); in gmap_shadow_r2t()
1800 gmap_unshadow_r2t(sg, raddr); in gmap_shadow_r2t()
1802 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1805 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r2t()
1824 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, in gmap_shadow_r3t() argument
1832 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_r3t()
1842 spin_lock(&sg->guest_table_lock); in gmap_shadow_r3t()
1843 table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */ in gmap_shadow_r3t()
1859 if (sg->edat_level >= 1) in gmap_shadow_r3t()
1861 list_add(&page->lru, &sg->crst_list); in gmap_shadow_r3t()
1865 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1868 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1874 rc = gmap_protect_rmap(sg, raddr, origin + offset, len); in gmap_shadow_r3t()
1875 spin_lock(&sg->guest_table_lock); in gmap_shadow_r3t()
1877 table = gmap_table_walk(sg, saddr, 3); in gmap_shadow_r3t()
1884 gmap_unshadow_r3t(sg, raddr); in gmap_shadow_r3t()
1886 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1889 spin_unlock(&sg->guest_table_lock); in gmap_shadow_r3t()
1908 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, in gmap_shadow_sgt() argument
1916 BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE)); in gmap_shadow_sgt()
1926 spin_lock(&sg->guest_table_lock); in gmap_shadow_sgt()
1927 table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */ in gmap_shadow_sgt()
1943 if (sg->edat_level >= 1) in gmap_shadow_sgt()
1945 list_add(&page->lru, &sg->crst_list); in gmap_shadow_sgt()
1949 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1952 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1958 rc = gmap_protect_rmap(sg, raddr, origin + offset, len); in gmap_shadow_sgt()
1959 spin_lock(&sg->guest_table_lock); in gmap_shadow_sgt()
1961 table = gmap_table_walk(sg, saddr, 2); in gmap_shadow_sgt()
1968 gmap_unshadow_sgt(sg, raddr); in gmap_shadow_sgt()
1970 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1973 spin_unlock(&sg->guest_table_lock); in gmap_shadow_sgt()
1992 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr, in gmap_shadow_pgt_lookup() argument
2000 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_pgt_lookup()
2001 spin_lock(&sg->guest_table_lock); in gmap_shadow_pgt_lookup()
2002 table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */ in gmap_shadow_pgt_lookup()
2013 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt_lookup()
2032 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, in gmap_shadow_pgt() argument
2040 BUG_ON(!gmap_is_shadow(sg) || (pgt & _SEGMENT_ENTRY_LARGE)); in gmap_shadow_pgt()
2042 page = page_table_alloc_pgste(sg->mm); in gmap_shadow_pgt()
2050 spin_lock(&sg->guest_table_lock); in gmap_shadow_pgt()
2051 table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */ in gmap_shadow_pgt()
2066 list_add(&page->lru, &sg->pt_list); in gmap_shadow_pgt()
2070 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
2073 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
2077 rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE); in gmap_shadow_pgt()
2078 spin_lock(&sg->guest_table_lock); in gmap_shadow_pgt()
2080 table = gmap_table_walk(sg, saddr, 1); in gmap_shadow_pgt()
2087 gmap_unshadow_pgt(sg, raddr); in gmap_shadow_pgt()
2089 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
2092 spin_unlock(&sg->guest_table_lock); in gmap_shadow_pgt()
2111 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte) in gmap_shadow_page() argument
2121 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_page()
2122 parent = sg->parent; in gmap_shadow_page()
2143 spin_lock(&sg->guest_table_lock); in gmap_shadow_page()
2145 tptep = (pte_t *) gmap_table_walk(sg, saddr, 0); in gmap_shadow_page()
2147 spin_unlock(&sg->guest_table_lock); in gmap_shadow_page()
2152 rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte); in gmap_shadow_page()
2155 gmap_insert_rmap(sg, vmaddr, rmap); in gmap_shadow_page()
2160 spin_unlock(&sg->guest_table_lock); in gmap_shadow_page()
2179 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, in gmap_shadow_notify() argument
2185 BUG_ON(!gmap_is_shadow(sg)); in gmap_shadow_notify()
2187 spin_lock(&sg->guest_table_lock); in gmap_shadow_notify()
2188 if (sg->removed) { in gmap_shadow_notify()
2189 spin_unlock(&sg->guest_table_lock); in gmap_shadow_notify()
2193 start = sg->orig_asce & _ASCE_ORIGIN; in gmap_shadow_notify()
2194 end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE; in gmap_shadow_notify()
2195 if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start && in gmap_shadow_notify()
2198 gmap_unshadow(sg); in gmap_shadow_notify()
2199 spin_unlock(&sg->guest_table_lock); in gmap_shadow_notify()
2200 list_del(&sg->list); in gmap_shadow_notify()
2201 gmap_put(sg); in gmap_shadow_notify()
2205 head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT); in gmap_shadow_notify()
2211 gmap_unshadow_r2t(sg, raddr); in gmap_shadow_notify()
2214 gmap_unshadow_r3t(sg, raddr); in gmap_shadow_notify()
2217 gmap_unshadow_sgt(sg, raddr); in gmap_shadow_notify()
2220 gmap_unshadow_pgt(sg, raddr); in gmap_shadow_notify()
2223 gmap_unshadow_page(sg, raddr); in gmap_shadow_notify()
2228 spin_unlock(&sg->guest_table_lock); in gmap_shadow_notify()
2246 struct gmap *gmap, *sg, *next; in ptep_notify() local
2263 list_for_each_entry_safe(sg, next, in ptep_notify()
2265 gmap_shadow_notify(sg, vmaddr, gaddr); in ptep_notify()