Diff markup
001 001
002 002
003 003
004 004
005 005
006 006
007 007
008 008
009 009
010 010
011 011
012 012
013 013
014 014
015 015
016 016
017 017
018 018
019 #include <sos/assert.h> 019 #include <sos/assert.h>
020 #include <sos/list.h> 020 #include <sos/list.h>
021 #include <sos/physmem.h> 021 #include <sos/physmem.h>
022 #include <sos/kmem_slab.h> 022 #include <sos/kmem_slab.h>
023 #include <drivers/bochs.h> 023 #include <drivers/bochs.h>
024 #include <hwcore/mm_context.h> 024 #include <hwcore/mm_context.h>
025 #include <hwcore/paging.h> 025 #include <hwcore/paging.h>
026 #include <drivers/zero.h> 026 #include <drivers/zero.h>
027 027
028 #include "umem_vmm.h" 028 #include "umem_vmm.h"
029 029
030 030
031 struct sos_umem_vmm_as 031 struct sos_umem_vmm_as
032 { 032 {
033 033
034 struct sos_process * process; 034 struct sos_process * process;
035 035
036 036
037 struct sos_mm_context * mm_context; 037 struct sos_mm_context * mm_context;
038 038
039 039
040 struct sos_umem_vmm_vr * list_vr; 040 struct sos_umem_vmm_vr * list_vr;
041 041
042 042
043 sos_uaddr_t heap_start; 043 sos_uaddr_t heap_start;
044 sos_size_t heap_size; 044 sos_size_t heap_size;
045 045
046 046
047 sos_size_t phys_total; 047 sos_size_t phys_total;
048 struct vm_usage 048 struct vm_usage
049 { 049 {
050 sos_size_t overall; 050 sos_size_t overall;
051 sos_size_t ro, rw, code 051 sos_size_t ro, rw, code ;
052 } vm_total, vm_shrd; 052 } vm_total, vm_shrd;
053 053
054 054
055 sos_size_t pgflt_cow; 055 sos_size_t pgflt_cow;
056 sos_size_t pgflt_page_in; 056 sos_size_t pgflt_page_in;
057 sos_size_t pgflt_invalid; 057 sos_size_t pgflt_invalid;
058 }; 058 };
059 059
060 060
061 struct sos_umem_vmm_vr 061 struct sos_umem_vmm_vr
062 { 062 {
063 063
064 struct sos_umem_vmm_as *address_space; 064 struct sos_umem_vmm_as *address_space;
065 065
066 066
067 sos_uaddr_t start; 067 sos_uaddr_t start;
068 sos_size_t size; 068 sos_size_t size;
069 069
070 070
071 071
072 sos_ui32_t access_rights; 072 sos_ui32_t access_rights;
073 073
074 074
075 075
076 076
077 sos_ui32_t flags; 077 sos_ui32_t flags;
078 078
079 079
080 080
081 081
082 082
083 struct sos_umem_vmm_vr_ops *ops; 083 struct sos_umem_vmm_vr_ops *ops;
084 084
085 085
086 struct sos_umem_vmm_mapped_resource *mapped_ 086 struct sos_umem_vmm_mapped_resource *mapped_resource;
087 sos_luoffset_t offset_in_resource; 087 sos_luoffset_t offset_in_resource;
088 088
089 089
090 090
091 struct sos_umem_vmm_vr *prev_in_as, *next_in 091 struct sos_umem_vmm_vr *prev_in_as, *next_in_as;
092 092
093 093
094 094
095 struct sos_umem_vmm_vr *prev_in_mapped_resou 095 struct sos_umem_vmm_vr *prev_in_mapped_resource, *next_in_mapped_resource;
096 }; 096 };
097 097
098 098
099 099
100 100
101 101
102 static struct sos_kslab_cache * cache_of_as; 102 static struct sos_kslab_cache * cache_of_as;
103 static struct sos_kslab_cache * cache_of_vr; 103 static struct sos_kslab_cache * cache_of_vr;
104 104
105 105
106 106
107 void sos_dump_as(const struct sos_umem_vmm_as 107 void sos_dump_as(const struct sos_umem_vmm_as * as, const char *str)
108 { 108 {
109 struct sos_umem_vmm_vr *vr; 109 struct sos_umem_vmm_vr *vr;
110 int nb_vr; 110 int nb_vr;
111 111
112 sos_bochs_printf("AS %p - %s:\n", as, str); 112 sos_bochs_printf("AS %p - %s:\n", as, str);
113 sos_bochs_printf(" physical mem: %x\n", 113 sos_bochs_printf(" physical mem: %x\n",
114 as->phys_total); 114 as->phys_total);
115 sos_bochs_printf(" VM (all/ro+rw/exec) tot 115 sos_bochs_printf(" VM (all/ro+rw/exec) tot:%x/%x+%x/%x shrd:%x/%x+%x/%x\n",
116 as->vm_total.overall, 116 as->vm_total.overall,
117 as->vm_total.ro, as->vm_tot 117 as->vm_total.ro, as->vm_total.rw, as->vm_total.code,
118 as->vm_shrd.overall, 118 as->vm_shrd.overall,
119 as->vm_shrd.ro, as->vm_shrd 119 as->vm_shrd.ro, as->vm_shrd.rw, as->vm_shrd.code);
120 sos_bochs_printf(" pgflt cow=%d pgin=%d in 120 sos_bochs_printf(" pgflt cow=%d pgin=%d inv=%d\n",
121 as->pgflt_cow, as->pgflt_pa 121 as->pgflt_cow, as->pgflt_page_in, as->pgflt_invalid);
122 list_foreach_named(as->list_vr, vr, nb_vr, p 122 list_foreach_named(as->list_vr, vr, nb_vr, prev_in_as, next_in_as)
123 { 123 {
124 sos_bochs_printf(" VR[%d]=%x: [%x,%x[ ( 124 sos_bochs_printf(" VR[%d]=%x: [%x,%x[ (sz=%x) mr=(%x)+%llx %c%c%c fl=%x\n",
125 nb_vr, (unsigned)vr, 125 nb_vr, (unsigned)vr,
126 vr->start, vr->start + 126 vr->start, vr->start + vr->size, vr->size,
127 (unsigned)vr->mapped_re 127 (unsigned)vr->mapped_resource,
128 vr->offset_in_resource, 128 vr->offset_in_resource,
129 (vr->access_rights & SO 129 (vr->access_rights & SOS_VM_MAP_PROT_READ)?'r':'-',
130 (vr->access_rights & SO 130 (vr->access_rights & SOS_VM_MAP_PROT_WRITE)?'w':'-',
131 (vr->access_rights & SO 131 (vr->access_rights & SOS_VM_MAP_PROT_EXEC)?'x':'-',
132 (unsigned)vr->flags); 132 (unsigned)vr->flags);
133 } 133 }
134 sos_bochs_printf("FIN (%s)\n", str); 134 sos_bochs_printf("FIN (%s)\n", str);
135 } 135 }
136 136
137 137
138 138
139 139
140 140
141 141
142 sos_paddr_t sos_zero_page = 0 !! 142 sos_paddr_t sos_zero_physpage = 0 ;
>> 143 sos_vaddr_t sos_zero_kernelpage = 0 ;
143 144
144 145
145 146
146 147
147 148
148 149
149 150
150 151
151 152
152 static struct sos_umem_vmm_vr * 153 static struct sos_umem_vmm_vr *
153 find_enclosing_or_next_vr(struct sos_umem_vmm_ 154 find_enclosing_or_next_vr(struct sos_umem_vmm_as * as,
154 sos_uaddr_t uaddr); 155 sos_uaddr_t uaddr);
155 156
156 157
157 158
158 159
159 160
160 161
161 static struct sos_umem_vmm_vr * 162 static struct sos_umem_vmm_vr *
162 find_first_intersecting_vr(struct sos_umem_vmm 163 find_first_intersecting_vr(struct sos_umem_vmm_as * as,
163 sos_uaddr_t start_u 164 sos_uaddr_t start_uaddr, sos_size_t size);
164 165
165 166
166 167
167 168
168 169
169 170
170 171
171 172
172 173
173 static sos_uaddr_t 174 static sos_uaddr_t
174 find_first_free_interval(struct sos_umem_vmm_a 175 find_first_free_interval(struct sos_umem_vmm_as * as,
175 sos_uaddr_t hint_uadd 176 sos_uaddr_t hint_uaddr, sos_size_t size);
176 177
177 178
178 179
179 180
180 static void 181 static void
181 as_account_change_of_vr_protection(struct sos_ 182 as_account_change_of_vr_protection(struct sos_umem_vmm_as * as,
182 sos_bool_t 183 sos_bool_t is_shared,
183 sos_size_t 184 sos_size_t size,
184 sos_ui32_t 185 sos_ui32_t prev_access_rights,
185 sos_ui32_t 186 sos_ui32_t new_access_rights);
186 187
187 188
188 sos_ret_t sos_umem_vmm_subsystem_setup() 189 sos_ret_t sos_umem_vmm_subsystem_setup()
189 { 190 {
190 sos_vaddr_t vaddr_zero_page; <<
191 <<
192 191
193 192
194 vaddr_zero_page = sos_kmem_vmm_alloc(1, SOS_ !! 193 sos_zero_kernelpage = sos_kmem_vmm_alloc(1, SOS_KMEM_VMM_MAP);
195 if (vaddr_zero_page == (sos_vaddr_t)NULL) !! 194 if (sos_zero_kernelpage == (sos_vaddr_t)NULL)
196 return -SOS_ENOMEM; 195 return -SOS_ENOMEM;
197 memset((void*)vaddr_zero_page, 0x0, SOS_PAGE !! 196 memset((void*)sos_zero_kernelpage, 0x0, SOS_PAGE_SIZE);
198 197
199 198
200 sos_zero_page = sos_paging_get_paddr(vaddr_z !! 199 sos_zero_physpage = sos_paging_get_paddr(sos_zero_kernelpage);
201 SOS_ASSERT_FATAL(NULL != (void*)sos_zero_pag !! 200 SOS_ASSERT_FATAL(NULL != (void*)sos_zero_physpage);
202 sos_physmem_ref_physpage_at(sos_zero_page); !! 201 sos_physmem_ref_physpage_at(sos_zero_physpage);
203 <<
204 <<
205 <<
206 sos_paging_unmap(vaddr_zero_page); <<
207 202
208 203
209 cache_of_as 204 cache_of_as
210 = sos_kmem_cache_create("Address space str 205 = sos_kmem_cache_create("Address space structures",
211 sizeof(struct sos_ 206 sizeof(struct sos_umem_vmm_as),
212 1, 0, 207 1, 0,
213 SOS_KSLAB_CREATE_M 208 SOS_KSLAB_CREATE_MAP
214 | SOS_KSLAB_CREATE 209 | SOS_KSLAB_CREATE_ZERO);
215 if (! cache_of_as) 210 if (! cache_of_as)
216 { 211 {
217 sos_physmem_unref_physpage(sos_zero_page !! 212 sos_physmem_unref_physpage(sos_zero_physpage);
218 return -SOS_ENOMEM; 213 return -SOS_ENOMEM;
219 } 214 }
220 215
221 cache_of_vr 216 cache_of_vr
222 = sos_kmem_cache_create("Virtual Region st 217 = sos_kmem_cache_create("Virtual Region structures",
223 sizeof(struct sos_ 218 sizeof(struct sos_umem_vmm_vr),
224 1, 0, 219 1, 0,
225 SOS_KSLAB_CREATE_M 220 SOS_KSLAB_CREATE_MAP
226 | SOS_KSLAB_CREATE 221 | SOS_KSLAB_CREATE_ZERO);
227 if (! cache_of_vr) 222 if (! cache_of_vr)
228 { 223 {
229 sos_physmem_unref_physpage(sos_zero_page !! 224 sos_physmem_unref_physpage(sos_zero_physpage);
230 sos_kmem_cache_destroy(cache_of_as); 225 sos_kmem_cache_destroy(cache_of_as);
231 return -SOS_ENOMEM; 226 return -SOS_ENOMEM;
232 } 227 }
233 228
234 return SOS_OK; 229 return SOS_OK;
235 } 230 }
236 231
237 232
238 struct sos_umem_vmm_as * 233 struct sos_umem_vmm_as *
239 sos_umem_vmm_create_empty_as(struct sos_proces 234 sos_umem_vmm_create_empty_as(struct sos_process *owner)
240 { 235 {
241 struct sos_umem_vmm_as * as 236 struct sos_umem_vmm_as * as
242 = (struct sos_umem_vmm_as *) sos_kmem_cach 237 = (struct sos_umem_vmm_as *) sos_kmem_cache_alloc(cache_of_as, 0);
243 if (! as) 238 if (! as)
244 return NULL; 239 return NULL;
245 240
246 as->mm_context = sos_mm_context_create(); 241 as->mm_context = sos_mm_context_create();
247 if (NULL == as->mm_context) 242 if (NULL == as->mm_context)
248 { 243 {
249 244
250 sos_kmem_cache_free((sos_vaddr_t)as); 245 sos_kmem_cache_free((sos_vaddr_t)as);
251 return NULL; 246 return NULL;
252 } 247 }
253 248
254 as->process = owner; 249 as->process = owner;
255 return as; 250 return as;
256 } 251 }
257 252
258 253
259 struct sos_umem_vmm_as * 254 struct sos_umem_vmm_as *
260 sos_umem_vmm_duplicate_current_thread_as(struc 255 sos_umem_vmm_duplicate_current_thread_as(struct sos_process *owner)
261 { 256 {
262 __label__ undo_creation; 257 __label__ undo_creation;
263 struct sos_umem_vmm_as * my_as; 258 struct sos_umem_vmm_as * my_as;
264 struct sos_umem_vmm_vr * model_vr; 259 struct sos_umem_vmm_vr * model_vr;
265 int nb_vr; 260 int nb_vr;
266 261
267 struct sos_umem_vmm_as * new_as 262 struct sos_umem_vmm_as * new_as
268 = (struct sos_umem_vmm_as *) sos_kmem_cach 263 = (struct sos_umem_vmm_as *) sos_kmem_cache_alloc(cache_of_as, 0);
269 if (! new_as) 264 if (! new_as)
270 return NULL; 265 return NULL;
271 266
272 my_as = sos_process_get_address_space(sos_th 267 my_as = sos_process_get_address_space(sos_thread_get_current()->process);
273 new_as->process = owner; 268 new_as->process = owner;
274 list_init_named(new_as->list_vr, prev_in_as, 269 list_init_named(new_as->list_vr, prev_in_as, next_in_as);
275 270
276 271
277 272
278 273
279 274
280 275
281 SOS_ASSERT_FATAL(SOS_OK 276 SOS_ASSERT_FATAL(SOS_OK
282 == sos_thread_prepare_user_ 277 == sos_thread_prepare_user_space_access(my_as,
283 278 (sos_vaddr_t)
284 279 NULL));
285 280
286 281
287 list_foreach_named(my_as->list_vr, model_vr, 282 list_foreach_named(my_as->list_vr, model_vr, nb_vr, prev_in_as, next_in_as)
288 { 283 {
289 struct sos_umem_vmm_vr * vr; 284 struct sos_umem_vmm_vr * vr;
290 285
291 286
292 if ( !(model_vr->flags & SOS_VR_MAP_SHAR 287 if ( !(model_vr->flags & SOS_VR_MAP_SHARED)
293 && (model_vr->access_rights & SOS_V 288 && (model_vr->access_rights & SOS_VM_MAP_PROT_WRITE) )
294 { 289 {
295 290
296 291
297 SOS_ASSERT_FATAL(SOS_OK 292 SOS_ASSERT_FATAL(SOS_OK
298 == sos_paging_prepa 293 == sos_paging_prepare_COW(model_vr->start,
299 294 model_vr->size));
300 } 295 }
301 296
302 297
303 vr = (struct sos_umem_vmm_vr *) sos_kmem 298 vr = (struct sos_umem_vmm_vr *) sos_kmem_cache_alloc(cache_of_vr, 0);
304 if (! vr) 299 if (! vr)
305 goto undo_creation; 300 goto undo_creation;
306 memcpy(vr, model_vr, sizeof(*vr)); 301 memcpy(vr, model_vr, sizeof(*vr));
307 vr->address_space = new_as; 302 vr->address_space = new_as;
308 303
309 304
310 if (vr->ops && vr->ops->ref) 305 if (vr->ops && vr->ops->ref)
311 vr->ops->ref(vr); 306 vr->ops->ref(vr);
312 307
313 308
314 list_add_tail_named(new_as->list_vr, vr, 309 list_add_tail_named(new_as->list_vr, vr, prev_in_as, next_in_as);
315 310
316 311
317 list_add_tail_named(model_vr->mapped_res 312 list_add_tail_named(model_vr->mapped_resource->list_vr, vr,
318 prev_in_mapped_resou 313 prev_in_mapped_resource,
319 next_in_mapped_resou 314 next_in_mapped_resource);
320 } 315 }
321 316
322 317
323 new_as->mm_context = sos_mm_context_duplicat 318 new_as->mm_context = sos_mm_context_duplicate(my_as->mm_context);
324 if (NULL == new_as->mm_context) 319 if (NULL == new_as->mm_context)
325 goto undo_creation; 320 goto undo_creation;
326 321
327 322
328 new_as->heap_start = my_as->heap_start; 323 new_as->heap_start = my_as->heap_start;
329 new_as->heap_size = my_as->heap_size; 324 new_as->heap_size = my_as->heap_size;
330 new_as->phys_total = my_as->phys_total; 325 new_as->phys_total = my_as->phys_total;
331 memcpy(& new_as->vm_total, & my_as->vm_total 326 memcpy(& new_as->vm_total, & my_as->vm_total, sizeof(struct vm_usage));
332 memcpy(& new_as->vm_shrd, & my_as->vm_shrd, 327 memcpy(& new_as->vm_shrd, & my_as->vm_shrd, sizeof(struct vm_usage));
333 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_us 328 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_user_space_access());
334 return new_as; 329 return new_as;
335 330
336 331
337 undo_creation: 332 undo_creation:
338 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_us 333 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_user_space_access());
339 sos_umem_vmm_delete_as(new_as); 334 sos_umem_vmm_delete_as(new_as);
340 return NULL; 335 return NULL;
341 } 336 }
342 337
343 338
344 sos_ret_t 339 sos_ret_t
345 sos_umem_vmm_delete_as(struct sos_umem_vmm_as 340 sos_umem_vmm_delete_as(struct sos_umem_vmm_as * as)
346 { 341 {
347 while(! list_is_empty_named(as->list_vr, pre 342 while(! list_is_empty_named(as->list_vr, prev_in_as, next_in_as))
348 { 343 {
349 struct sos_umem_vmm_vr * vr; 344 struct sos_umem_vmm_vr * vr;
350 vr = list_get_head_named(as->list_vr, pr 345 vr = list_get_head_named(as->list_vr, prev_in_as, next_in_as);
351 346
352 347
353 list_pop_head_named(as->list_vr, prev_in 348 list_pop_head_named(as->list_vr, prev_in_as, next_in_as);
354 list_delete_named(vr->mapped_resource->l 349 list_delete_named(vr->mapped_resource->list_vr, vr,
355 prev_in_mapped_resourc 350 prev_in_mapped_resource,
356 next_in_mapped_resourc 351 next_in_mapped_resource);
357 352
358 353
359 354
360 if (vr->ops) 355 if (vr->ops)
361 { 356 {
362 if (vr->ops->unmap) 357 if (vr->ops->unmap)
363 vr->ops->unmap(vr, vr->start, vr-> 358 vr->ops->unmap(vr, vr->start, vr->size);
364 if (vr->ops->unref) 359 if (vr->ops->unref)
365 vr->ops->unref(vr); 360 vr->ops->unref(vr);
366 } 361 }
367 362
368 sos_kmem_cache_free((sos_vaddr_t)vr); 363 sos_kmem_cache_free((sos_vaddr_t)vr);
369 } 364 }
370 365
371 366
372 if (as->mm_context) 367 if (as->mm_context)
373 sos_mm_context_unref(as->mm_context); 368 sos_mm_context_unref(as->mm_context);
374 369
375 370
376 sos_kmem_cache_free((sos_vaddr_t)as); 371 sos_kmem_cache_free((sos_vaddr_t)as);
377 372
378 return SOS_OK; 373 return SOS_OK;
379 } 374 }
380 375
381 376
382 struct sos_process * 377 struct sos_process *
383 sos_umem_vmm_get_process(struct sos_umem_vmm_a 378 sos_umem_vmm_get_process(struct sos_umem_vmm_as * as)
384 { 379 {
385 return as->process; 380 return as->process;
386 } 381 }
387 382
388 383
389 struct sos_mm_context * 384 struct sos_mm_context *
390 sos_umem_vmm_get_mm_context(struct sos_umem_vm 385 sos_umem_vmm_get_mm_context(struct sos_umem_vmm_as * as)
391 { 386 {
392 return as->mm_context; 387 return as->mm_context;
393 } 388 }
394 389
395 390
396 struct sos_umem_vmm_vr * 391 struct sos_umem_vmm_vr *
397 sos_umem_vmm_get_vr_at_address(struct sos_umem 392 sos_umem_vmm_get_vr_at_address(struct sos_umem_vmm_as * as,
398 sos_uaddr_t uad 393 sos_uaddr_t uaddr)
399 { 394 {
400 struct sos_umem_vmm_vr * vr; 395 struct sos_umem_vmm_vr * vr;
401 vr = find_enclosing_or_next_vr(as, uaddr); 396 vr = find_enclosing_or_next_vr(as, uaddr);
402 if (! vr) 397 if (! vr)
403 return NULL; 398 return NULL;
404 399
405 400
406 if (uaddr < vr->start) 401 if (uaddr < vr->start)
407 return NULL; 402 return NULL;
408 403
409 return vr; 404 return vr;
410 } 405 }
411 406
412 407
413 struct sos_umem_vmm_as * 408 struct sos_umem_vmm_as *
414 sos_umem_vmm_get_as_of_vr(struct sos_umem_vmm_ 409 sos_umem_vmm_get_as_of_vr(struct sos_umem_vmm_vr * vr)
415 { 410 {
416 return vr->address_space; 411 return vr->address_space;
417 } 412 }
418 413
419 414
420 struct sos_umem_vmm_vr_ops * 415 struct sos_umem_vmm_vr_ops *
421 sos_umem_vmm_get_ops_of_vr(struct sos_umem_vmm 416 sos_umem_vmm_get_ops_of_vr(struct sos_umem_vmm_vr * vr)
422 { 417 {
423 return vr->ops; 418 return vr->ops;
424 } 419 }
425 420
426 421
427 sos_ui32_t sos_umem_vmm_get_prot_of_vr(struct 422 sos_ui32_t sos_umem_vmm_get_prot_of_vr(struct sos_umem_vmm_vr * vr)
428 { 423 {
429 return vr->access_rights; 424 return vr->access_rights;
430 } 425 }
431 426
432 427
433 sos_ui32_t sos_umem_vmm_get_flags_of_vr(struct 428 sos_ui32_t sos_umem_vmm_get_flags_of_vr(struct sos_umem_vmm_vr * vr)
434 { 429 {
435 return vr->flags; 430 return vr->flags;
436 } 431 }
437 432
438 433
439 struct sos_umem_vmm_mapped_resource * 434 struct sos_umem_vmm_mapped_resource *
440 sos_umem_vmm_get_mapped_resource_of_vr(struct 435 sos_umem_vmm_get_mapped_resource_of_vr(struct sos_umem_vmm_vr * vr)
441 { 436 {
442 return vr->mapped_resource; 437 return vr->mapped_resource;
443 } 438 }
444 439
445 440
446 sos_uaddr_t sos_umem_vmm_get_start_of_vr(struc 441 sos_uaddr_t sos_umem_vmm_get_start_of_vr(struct sos_umem_vmm_vr * vr)
447 { 442 {
448 return vr->start; 443 return vr->start;
449 } 444 }
450 445
451 446
452 sos_size_t sos_umem_vmm_get_size_of_vr(struct 447 sos_size_t sos_umem_vmm_get_size_of_vr(struct sos_umem_vmm_vr * vr)
453 { 448 {
454 return vr->size; 449 return vr->size;
455 } 450 }
456 451
457 452
458 sos_luoffset_t sos_umem_vmm_get_offset_in_reso 453 sos_luoffset_t sos_umem_vmm_get_offset_in_resource(struct sos_umem_vmm_vr * vr)
459 { 454 {
460 return vr->offset_in_resource; 455 return vr->offset_in_resource;
461 } 456 }
462 457
463 458
464 sos_ret_t 459 sos_ret_t
465 sos_umem_vmm_set_ops_of_vr(struct sos_umem_vmm 460 sos_umem_vmm_set_ops_of_vr(struct sos_umem_vmm_vr * vr,
466 struct sos_umem_vmm 461 struct sos_umem_vmm_vr_ops * ops)
467 { 462 {
468 463
469 SOS_ASSERT_FATAL(NULL == vr->ops); 464 SOS_ASSERT_FATAL(NULL == vr->ops);
470 465
471 vr->ops = ops; 466 vr->ops = ops;
472 return SOS_OK; 467 return SOS_OK;
473 } 468 }
474 469
475 470
476 471
477 472
478 473
479 474
480 #define INTERNAL_MAP_CALLED_FROM_MREMAP (1 << 475 #define INTERNAL_MAP_CALLED_FROM_MREMAP (1 << 8)
481 476
482 sos_ret_t 477 sos_ret_t
483 sos_umem_vmm_map(struct sos_umem_vmm_as * as, 478 sos_umem_vmm_map(struct sos_umem_vmm_as * as,
484 sos_uaddr_t * uaddr 479 sos_uaddr_t * uaddr, sos_size_t size,
485 sos_ui32_t access_rights, 480 sos_ui32_t access_rights,
486 sos_ui32_t flags, 481 sos_ui32_t flags,
487 struct sos_umem_vmm_mapped_re 482 struct sos_umem_vmm_mapped_resource * resource,
488 sos_luoffset_t offset_in_reso 483 sos_luoffset_t offset_in_resource)
489 { 484 {
490 __label__ return_mmap; 485 __label__ return_mmap;
491 sos_uaddr_t hint_uaddr; 486 sos_uaddr_t hint_uaddr;
492 struct sos_umem_vmm_vr *prev_vr, *next_vr, * 487 struct sos_umem_vmm_vr *prev_vr, *next_vr, *vr, *preallocated_vr;
493 sos_bool_t merge_with_preceding, merge_with_ 488 sos_bool_t merge_with_preceding, merge_with_next, used_preallocated_vr;
494 sos_bool_t internal_map_called_from_mremap 489 sos_bool_t internal_map_called_from_mremap
495 = (flags & INTERNAL_MAP_CALLED_FROM_MREMAP 490 = (flags & INTERNAL_MAP_CALLED_FROM_MREMAP);
496 491
497 sos_ret_t retval = SOS_OK; 492 sos_ret_t retval = SOS_OK;
498 used_preallocated_vr = FALSE; 493 used_preallocated_vr = FALSE;
499 hint_uaddr = *uaddr; 494 hint_uaddr = *uaddr;
500 495
501 496
502 *uaddr = (sos_vaddr_t)NULL; 497 *uaddr = (sos_vaddr_t)NULL;
503 498
504 if (! resource) 499 if (! resource)
505 return -SOS_EINVAL; 500 return -SOS_EINVAL;
506 if (! resource->mmap) 501 if (! resource->mmap)
507 return -SOS_EPERM; 502 return -SOS_EPERM;
508 503
509 if (! SOS_IS_PAGE_ALIGNED(hint_uaddr)) 504 if (! SOS_IS_PAGE_ALIGNED(hint_uaddr))
510 return -SOS_EINVAL; 505 return -SOS_EINVAL;
511 506
512 if (size <= 0) 507 if (size <= 0)
513 return -SOS_EINVAL; 508 return -SOS_EINVAL;
514 size = SOS_PAGE_ALIGN_SUP(size); 509 size = SOS_PAGE_ALIGN_SUP(size);
515 510
516 if (flags & SOS_VR_MAP_SHARED) 511 if (flags & SOS_VR_MAP_SHARED)
517 { 512 {
518 513
519 if ( ( (access_rights & SOS_VM_MAP_PROT_ 514 if ( ( (access_rights & SOS_VM_MAP_PROT_READ)
520 && !(resource->allowed_access_rig 515 && !(resource->allowed_access_rights & SOS_VM_MAP_PROT_READ) )
521 || ( (access_rights & SOS_VM_MAP_PR 516 || ( (access_rights & SOS_VM_MAP_PROT_WRITE)
522 && !(resource->allowed_access_ 517 && !(resource->allowed_access_rights & SOS_VM_MAP_PROT_WRITE) )
523 || ( (access_rights & SOS_VM_MAP_PR 518 || ( (access_rights & SOS_VM_MAP_PROT_EXEC)
524 && !(resource->allowed_access_ 519 && !(resource->allowed_access_rights & SOS_VM_MAP_PROT_EXEC)) )
525 return -SOS_EPERM; 520 return -SOS_EPERM;
526 } 521 }
527 522
528 523
529 if ( !internal_map_called_from_mremap 524 if ( !internal_map_called_from_mremap
530 && ( resource->flags & SOS_MAPPED_RESOU 525 && ( resource->flags & SOS_MAPPED_RESOURCE_ANONYMOUS ) )
531 526
532 { 527 {
533 528
534 } 529 }
535 530
536 531
537 else if (offset_in_resource + size <= offset 532 else if (offset_in_resource + size <= offset_in_resource)
538 return -SOS_EINVAL; 533 return -SOS_EINVAL;
539 534
540 535
541 access_rights &= (SOS_VM_MAP_PROT_READ 536 access_rights &= (SOS_VM_MAP_PROT_READ
542 | SOS_VM_MAP_PROT_WRITE 537 | SOS_VM_MAP_PROT_WRITE
543 | SOS_VM_MAP_PROT_EXEC); 538 | SOS_VM_MAP_PROT_EXEC);
544 flags &= (SOS_VR_MAP_SHARED 539 flags &= (SOS_VR_MAP_SHARED
545 | SOS_VR_MAP_FIXED); 540 | SOS_VR_MAP_FIXED);
546 541
547 542
548 543
549 544
550 preallocated_vr 545 preallocated_vr
551 = (struct sos_umem_vmm_vr *)sos_kmem_cache 546 = (struct sos_umem_vmm_vr *)sos_kmem_cache_alloc(cache_of_vr, 0);
552 if (! preallocated_vr) 547 if (! preallocated_vr)
553 return -SOS_ENOMEM; 548 return -SOS_ENOMEM;
554 549
555 550
556 if (flags & SOS_VR_MAP_FIXED) 551 if (flags & SOS_VR_MAP_FIXED)
557 { 552 {
558 553
559 554
560 555
561 556
562 557
563 if (hint_uaddr < SOS_PAGING_BASE_USER_AD 558 if (hint_uaddr < SOS_PAGING_BASE_USER_ADDRESS)
564 { retval = -SOS_EINVAL; goto return_mm 559 { retval = -SOS_EINVAL; goto return_mmap; }
565 if (hint_uaddr > SOS_PAGING_TOP_USER_ADD 560 if (hint_uaddr > SOS_PAGING_TOP_USER_ADDRESS - size)
566 { retval = -SOS_EINVAL; goto return_mm 561 { retval = -SOS_EINVAL; goto return_mmap; }
567 562
568 563
569 retval = sos_umem_vmm_unmap(as, hint_uad 564 retval = sos_umem_vmm_unmap(as, hint_uaddr, size);
570 if (SOS_OK != retval) 565 if (SOS_OK != retval)
571 { goto return_mmap; } 566 { goto return_mmap; }
572 } 567 }
573 else 568 else
574 { 569 {
575 570
576 571
577 572
578 573
579 574
580 hint_uaddr = find_first_free_interval(as 575 hint_uaddr = find_first_free_interval(as, hint_uaddr, size);
581 if (! hint_uaddr) 576 if (! hint_uaddr)
582 { retval = -SOS_ENOMEM; goto return_mm 577 { retval = -SOS_ENOMEM; goto return_mmap; }
583 } 578 }
584 579
585 580
586 581
587 582
588 if ( !internal_map_called_from_mremap 583 if ( !internal_map_called_from_mremap
589 && (resource->flags & SOS_MAPPED_RESOUR 584 && (resource->flags & SOS_MAPPED_RESOURCE_ANONYMOUS ) )
590 offset_in_resource = hint_uaddr; 585 offset_in_resource = hint_uaddr;
591 586
592 587
593 588
594 next_vr = find_enclosing_or_next_vr(as, hint 589 next_vr = find_enclosing_or_next_vr(as, hint_uaddr);
595 if (next_vr) 590 if (next_vr)
596 { 591 {
597 592
598 prev_vr = next_vr->prev_in_as; 593 prev_vr = next_vr->prev_in_as;
599 594
600 595
601 if (prev_vr->start > hint_uaddr) 596 if (prev_vr->start > hint_uaddr)
602 prev_vr = NULL; 597 prev_vr = NULL;
603 } 598 }
604 else 599 else
605 { 600 {
606 601
607 prev_vr = list_get_tail_named(as->list_v 602 prev_vr = list_get_tail_named(as->list_vr, prev_in_as, next_in_as);
608 } 603 }
609 604
610 605
611 merge_with_preceding 606 merge_with_preceding
612 = ( (NULL != prev_vr) 607 = ( (NULL != prev_vr)
613 && (prev_vr->mapped_resource == resour 608 && (prev_vr->mapped_resource == resource)
614 && (prev_vr->offset_in_resource + prev 609 && (prev_vr->offset_in_resource + prev_vr->size == offset_in_resource)
615 && (prev_vr->start + prev_vr->size == 610 && (prev_vr->start + prev_vr->size == hint_uaddr)
616 && (prev_vr->flags == flags) 611 && (prev_vr->flags == flags)
617 && (prev_vr->access_rights == access_r 612 && (prev_vr->access_rights == access_rights) );
618 613
619 614
620 merge_with_next 615 merge_with_next
621 = ( (NULL != next_vr) 616 = ( (NULL != next_vr)
622 && (next_vr->mapped_resource == resour 617 && (next_vr->mapped_resource == resource)
623 && (offset_in_resource + size == next_ 618 && (offset_in_resource + size == next_vr->offset_in_resource)
624 && (hint_uaddr + size == next_vr->star 619 && (hint_uaddr + size == next_vr->start)
625 && (next_vr->flags == flags) 620 && (next_vr->flags == flags)
626 && (next_vr->access_rights == access_r 621 && (next_vr->access_rights == access_rights) );
627 622
628 if (merge_with_preceding && merge_with_next) 623 if (merge_with_preceding && merge_with_next)
629 { 624 {
630 625
631 vr = prev_vr; 626 vr = prev_vr;
632 vr->size += size + next_vr->size; 627 vr->size += size + next_vr->size;
633 628
634 629
635 list_delete_named(as->list_vr, next_vr, 630 list_delete_named(as->list_vr, next_vr, prev_in_as, next_in_as);
636 list_delete_named(next_vr->mapped_resour 631 list_delete_named(next_vr->mapped_resource->list_vr, next_vr,
637 prev_in_mapped_resourc 632 prev_in_mapped_resource, next_in_mapped_resource);
638 633
639 if (next_vr->ops && next_vr->ops->unref) 634 if (next_vr->ops && next_vr->ops->unref)
640 next_vr->ops->unref(next_vr); 635 next_vr->ops->unref(next_vr);
641 636
642 sos_kmem_vmm_free((sos_vaddr_t) next_vr) 637 sos_kmem_vmm_free((sos_vaddr_t) next_vr);
643 } 638 }
644 else if (merge_with_preceding) 639 else if (merge_with_preceding)
645 { 640 {
646 641
647 vr = prev_vr; 642 vr = prev_vr;
648 vr->size += size; 643 vr->size += size;
649 } 644 }
650 else if (merge_with_next) 645 else if (merge_with_next)
651 { 646 {
652 647
653 vr = next_vr; 648 vr = next_vr;
654 vr->start -= size; 649 vr->start -= size;
655 vr->size += size; 650 vr->size += size;
656 } 651 }
657 else 652 else
658 { 653 {
659 654
660 655
661 vr = preallocated_vr; 656 vr = preallocated_vr;
662 used_preallocated_vr = TRUE; 657 used_preallocated_vr = TRUE;
663 658
664 vr->start = hint_uaddr; 659 vr->start = hint_uaddr;
665 vr->size = size; 660 vr->size = size;
666 vr->access_rights = access_rights; 661 vr->access_rights = access_rights;
667 vr->flags = flags; 662 vr->flags = flags;
668 vr->mapped_resource = resource; 663 vr->mapped_resource = resource;
669 vr->offset_in_resource = offset_in_resou 664 vr->offset_in_resource = offset_in_resource;
670 665
671 666
672 vr->address_space = as; 667 vr->address_space = as;
673 if (prev_vr) 668 if (prev_vr)
674 list_insert_after_named(as->list_vr, p 669 list_insert_after_named(as->list_vr, prev_vr, vr,
675 prev_in_as, ne 670 prev_in_as, next_in_as);
676 else 671 else
677 list_add_head_named(as->list_vr, vr, p 672 list_add_head_named(as->list_vr, vr, prev_in_as, next_in_as);
678 list_add_tail_named(vr->mapped_resource- 673 list_add_tail_named(vr->mapped_resource->list_vr, vr,
679 prev_in_mapped_resou 674 prev_in_mapped_resource,
680 next_in_mapped_resou 675 next_in_mapped_resource);
681 676
682 677
683 if (resource && resource->mmap) 678 if (resource && resource->mmap)
684 { 679 {
685 retval = resource->mmap(vr); 680 retval = resource->mmap(vr);
686 if (SOS_OK != retval) 681 if (SOS_OK != retval)
687 { 682 {
688 retval = sos_umem_vmm_unmap(as, 683 retval = sos_umem_vmm_unmap(as, vr->start, vr->size);
689 goto return_mmap; 684 goto return_mmap;
690 } 685 }
691 686
692 687
693 SOS_ASSERT_FATAL(vr->ops && vr->ops- 688 SOS_ASSERT_FATAL(vr->ops && vr->ops->page_in);
694 } 689 }
695 690
696 if (vr->ops && vr->ops->ref) 691 if (vr->ops && vr->ops->ref)
697 vr->ops->ref(vr); 692 vr->ops->ref(vr);
698 } 693 }
699 694
700 695
701 *uaddr = hint_uaddr; 696 *uaddr = hint_uaddr;
702 as_account_change_of_vr_protection(as, vr->f 697 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
703 size, 0, 698 size, 0, vr->access_rights);
704 retval = SOS_OK; 699 retval = SOS_OK;
705 700
706 return_mmap: 701 return_mmap:
707 if (! used_preallocated_vr) 702 if (! used_preallocated_vr)
708 sos_kmem_vmm_free((sos_vaddr_t)preallocate 703 sos_kmem_vmm_free((sos_vaddr_t)preallocated_vr);
709 704
710 return retval; 705 return retval;
711 } 706 }
712 707
713 708
714 sos_ret_t 709 sos_ret_t
715 sos_umem_vmm_unmap(struct sos_umem_vmm_as * as 710 sos_umem_vmm_unmap(struct sos_umem_vmm_as * as,
716 sos_uaddr_t uaddr, sos_size 711 sos_uaddr_t uaddr, sos_size_t size)
717 { 712 {
718 struct sos_umem_vmm_vr *vr, *preallocated_vr 713 struct sos_umem_vmm_vr *vr, *preallocated_vr;
719 sos_bool_t need_to_setup_mmu; 714 sos_bool_t need_to_setup_mmu;
720 sos_bool_t used_preallocated_vr; 715 sos_bool_t used_preallocated_vr;
721 716
722 if (! SOS_IS_PAGE_ALIGNED(uaddr)) 717 if (! SOS_IS_PAGE_ALIGNED(uaddr))
723 return -SOS_EINVAL; 718 return -SOS_EINVAL;
724 if (size <= 0) 719 if (size <= 0)
725 return -SOS_EINVAL; 720 return -SOS_EINVAL;
726 size = SOS_PAGE_ALIGN_SUP(size); 721 size = SOS_PAGE_ALIGN_SUP(size);
727 722
728 723
729 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS) 724 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS)
730 return -SOS_EINVAL; 725 return -SOS_EINVAL;
731 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS - si 726 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS - size)
732 return -SOS_EINVAL; 727 return -SOS_EINVAL;
733 728
734 729
735 730
736 731
737 732
738 733
739 used_preallocated_vr = FALSE; 734 used_preallocated_vr = FALSE;
740 preallocated_vr 735 preallocated_vr
741 = (struct sos_umem_vmm_vr *)sos_kmem_cache 736 = (struct sos_umem_vmm_vr *)sos_kmem_cache_alloc(cache_of_vr, 0);
742 if (! preallocated_vr) 737 if (! preallocated_vr)
743 return -SOS_ENOMEM; 738 return -SOS_ENOMEM;
744 739
745 740
746 vr = find_first_intersecting_vr(as, uaddr, s 741 vr = find_first_intersecting_vr(as, uaddr, size);
747 742
748 743
749 while (NULL != vr) 744 while (NULL != vr)
750 { 745 {
751 746
752 747
753 if (vr->start + vr->size <= uaddr) 748 if (vr->start + vr->size <= uaddr)
754 749
755 break; 750 break;
756 751
757 752
758 if (uaddr + size <= vr->start) 753 if (uaddr + size <= vr->start)
759 754
760 break; 755 break;
761 756
762 757
763 if ((vr->start >= uaddr) 758 if ((vr->start >= uaddr)
764 && (vr->start + vr->size <= uaddr + 759 && (vr->start + vr->size <= uaddr + size))
765 { 760 {
766 struct sos_umem_vmm_vr *next_vr; 761 struct sos_umem_vmm_vr *next_vr;
767 762
768 763
769 if (vr->ops && vr->ops->unmap) 764 if (vr->ops && vr->ops->unmap)
770 vr->ops->unmap(vr, vr->start, vr-> 765 vr->ops->unmap(vr, vr->start, vr->size);
771 766
772 767
773 next_vr = vr->next_in_as; 768 next_vr = vr->next_in_as;
774 if (next_vr == vr) 769 if (next_vr == vr)
775 next_vr = NULL; 770 next_vr = NULL;
776 list_delete_named(as->list_vr, vr, p 771 list_delete_named(as->list_vr, vr, prev_in_as, next_in_as);
777 772
778 773
779 list_delete_named(vr->mapped_resourc 774 list_delete_named(vr->mapped_resource->list_vr, vr,
780 prev_in_mapped_res 775 prev_in_mapped_resource,
781 next_in_mapped_res 776 next_in_mapped_resource);
782 777
783 if (vr->ops && vr->ops->unref) 778 if (vr->ops && vr->ops->unref)
784 vr->ops->unref(vr); 779 vr->ops->unref(vr);
785 780
786 as_account_change_of_vr_protection(a 781 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
787 v 782 vr->size, vr->access_rights, 0);
788 sos_kmem_vmm_free((sos_vaddr_t)vr); 783 sos_kmem_vmm_free((sos_vaddr_t)vr);
789 784
790 785
791 vr = next_vr; 786 vr = next_vr;
792 continue; 787 continue;
793 } 788 }
794 789
795 790
796 else if ( (vr->start < uaddr) 791 else if ( (vr->start < uaddr)
797 && (vr->start + vr->size > uad 792 && (vr->start + vr->size > uaddr + size) )
798 { 793 {
799 794
800 795
801 796
802 used_preallocated_vr = TRUE; 797 used_preallocated_vr = TRUE;
803 memcpy(preallocated_vr, vr, sizeof(* 798 memcpy(preallocated_vr, vr, sizeof(*vr));
804 799
805 800
806 preallocated_vr->start = uaddr + siz 801 preallocated_vr->start = uaddr + size;
807 preallocated_vr->size = vr->start + 802 preallocated_vr->size = vr->start + vr->size - (uaddr + size);
808 preallocated_vr->offset_in_resource 803 preallocated_vr->offset_in_resource += uaddr + size - vr->start;
809 vr->size 804 vr->size = uaddr - vr->start;
810 805
811 806
812 list_insert_after_named(as->list_vr, 807 list_insert_after_named(as->list_vr, vr, preallocated_vr,
813 prev_in_as, 808 prev_in_as, next_in_as);
814 list_add_tail_named(vr->mapped_resou 809 list_add_tail_named(vr->mapped_resource->list_vr, preallocated_vr,
815 prev_in_mapped_r 810 prev_in_mapped_resource,
816 next_in_mapped_r 811 next_in_mapped_resource);
817 812
818 813
819 if (vr->ops && vr->ops->unmap) 814 if (vr->ops && vr->ops->unmap)
820 vr->ops->unmap(vr, uaddr, size); 815 vr->ops->unmap(vr, uaddr, size);
821 if (preallocated_vr->ops && prealloc 816 if (preallocated_vr->ops && preallocated_vr->ops->ref)
822 preallocated_vr->ops->ref(prealloc 817 preallocated_vr->ops->ref(preallocated_vr);
823 818
824 819
825 as_account_change_of_vr_protection(a 820 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
826 s 821 size, vr->access_rights, 0);
827 822
828 823
829 break; 824 break;
830 } 825 }
831 826
832 827
833 else if (uaddr <= vr->start) 828 else if (uaddr <= vr->start)
834 { 829 {
835 sos_size_t translation = uaddr + siz 830 sos_size_t translation = uaddr + size - vr->start;
836 831
837 832
838 vr->size -= translatio 833 vr->size -= translation;
839 vr->offset_in_resource += translatio 834 vr->offset_in_resource += translation;
840 vr->start += translatio 835 vr->start += translation;
841 836
842 837
843 if (vr->ops && vr->ops->unmap) 838 if (vr->ops && vr->ops->unmap)
844 vr->ops->unmap(vr, uaddr + size, 839 vr->ops->unmap(vr, uaddr + size,
845 translation); 840 translation);
846 841
847 842
848 as_account_change_of_vr_protection(a 843 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
849 t 844 translation,
850 v 845 vr->access_rights, 0);
851 846
852 847
853 848
854 break; 849 break;
855 } 850 }
856 851
857 852
858 else if (uaddr + size >= vr->start + vr- 853 else if (uaddr + size >= vr->start + vr->size)
859 { 854 {
860 sos_size_t unmapped_size = vr->start 855 sos_size_t unmapped_size = vr->start + vr->size - uaddr;
861 856
862 857
863 vr->size = uaddr - vr->start; 858 vr->size = uaddr - vr->start;
864 859
865 860
866 if (vr->ops && vr->ops->unmap) 861 if (vr->ops && vr->ops->unmap)
867 vr->ops->unmap(vr, uaddr, unmapped 862 vr->ops->unmap(vr, uaddr, unmapped_size);
868 863
869 864
870 as_account_change_of_vr_protection(a 865 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
871 u 866 unmapped_size,
872 v 867 vr->access_rights, 0);
873 868
874 vr = vr->next_in_as; 869 vr = vr->next_in_as;
875 continue; 870 continue;
876 } 871 }
877 872
878 sos_display_fatal_error("BUG uaddr=%x sz 873 sos_display_fatal_error("BUG uaddr=%x sz=%x vr_start=%x, vr_sz=%x",
879 uaddr, size, vr- 874 uaddr, size, vr->start, vr->size);
880 } 875 }
881 876
882 need_to_setup_mmu = (sos_thread_get_current( 877 need_to_setup_mmu = (sos_thread_get_current()->squatted_mm_context
883 != as->mm_context); 878 != as->mm_context);
884 if (need_to_setup_mmu) 879 if (need_to_setup_mmu)
885 SOS_ASSERT_FATAL(SOS_OK 880 SOS_ASSERT_FATAL(SOS_OK
886 == sos_thread_prepare_use 881 == sos_thread_prepare_user_space_access(as,
887 882 (sos_vaddr_t)
888 883 NULL));
889 { 884 {
890 sos_size_t sz_unmapped = sos_paging_unmap_ 885 sos_size_t sz_unmapped = sos_paging_unmap_interval(uaddr, size);
891 SOS_ASSERT_FATAL(sz_unmapped >= 0); 886 SOS_ASSERT_FATAL(sz_unmapped >= 0);
892 as->phys_total -= sz_unmapped; 887 as->phys_total -= sz_unmapped;
893 } 888 }
894 if (need_to_setup_mmu) 889 if (need_to_setup_mmu)
895 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_ 890 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_user_space_access());
896 891
897 if (! used_preallocated_vr) 892 if (! used_preallocated_vr)
898 sos_kmem_vmm_free((sos_vaddr_t)preallocate 893 sos_kmem_vmm_free((sos_vaddr_t)preallocated_vr);
899 894
900 return SOS_OK; 895 return SOS_OK;
901 } 896 }
902 897
903 898
904 sos_ret_t 899 sos_ret_t
905 sos_umem_vmm_chprot(struct sos_umem_vmm_as * a 900 sos_umem_vmm_chprot(struct sos_umem_vmm_as * as,
906 sos_uaddr_t uaddr, sos_siz 901 sos_uaddr_t uaddr, sos_size_t size,
907 sos_ui32_t new_access_righ 902 sos_ui32_t new_access_rights)
908 { 903 {
909 struct sos_umem_vmm_vr *start_vr, *vr, 904 struct sos_umem_vmm_vr *start_vr, *vr,
910 *preallocated_middle_vr, *preallocated_rig 905 *preallocated_middle_vr, *preallocated_right_vr;
911 sos_bool_t used_preallocated_middle_vr, used 906 sos_bool_t used_preallocated_middle_vr, used_preallocated_right_vr;
912 907
913 if (! SOS_IS_PAGE_ALIGNED(uaddr)) 908 if (! SOS_IS_PAGE_ALIGNED(uaddr))
914 return -SOS_EINVAL; 909 return -SOS_EINVAL;
915 if (size <= 0) 910 if (size <= 0)
916 return -SOS_EINVAL; 911 return -SOS_EINVAL;
917 size = SOS_PAGE_ALIGN_SUP(size); 912 size = SOS_PAGE_ALIGN_SUP(size);
918 913
919 914
920 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS) 915 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS)
921 return -SOS_EINVAL; 916 return -SOS_EINVAL;
922 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS - si 917 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS - size)
923 return -SOS_EINVAL; 918 return -SOS_EINVAL;
924 919
925 920
926 921
927 used_preallocated_middle_vr = FALSE; 922 used_preallocated_middle_vr = FALSE;
928 used_preallocated_right_vr = FALSE; 923 used_preallocated_right_vr = FALSE;
929 preallocated_middle_vr 924 preallocated_middle_vr
930 = (struct sos_umem_vmm_vr *)sos_kmem_cache 925 = (struct sos_umem_vmm_vr *)sos_kmem_cache_alloc(cache_of_vr, 0);
931 if (! preallocated_middle_vr) 926 if (! preallocated_middle_vr)
932 return -SOS_ENOMEM; 927 return -SOS_ENOMEM;
933 preallocated_right_vr 928 preallocated_right_vr
934 = (struct sos_umem_vmm_vr *)sos_kmem_cache 929 = (struct sos_umem_vmm_vr *)sos_kmem_cache_alloc(cache_of_vr, 0);
935 if (! preallocated_right_vr) 930 if (! preallocated_right_vr)
936 { 931 {
937 sos_kmem_vmm_free((sos_vaddr_t)prealloca 932 sos_kmem_vmm_free((sos_vaddr_t)preallocated_middle_vr);
938 return -SOS_ENOMEM; 933 return -SOS_ENOMEM;
939 } 934 }
940 935
941 936
942 start_vr = find_first_intersecting_vr(as, ua 937 start_vr = find_first_intersecting_vr(as, uaddr, size);
943 if (NULL == start_vr) 938 if (NULL == start_vr)
944 return SOS_OK; 939 return SOS_OK;
945 940
946 941
947 942
948 vr = start_vr; 943 vr = start_vr;
949 while (TRUE) 944 while (TRUE)
950 { 945 {
951 946
952 947
953 if (vr->start + vr->size <= uaddr) 948 if (vr->start + vr->size <= uaddr)
954 949
955 break; 950 break;
956 951
957 952
958 if (uaddr + size < vr->start) 953 if (uaddr + size < vr->start)
959 954
960 break; 955 break;
961 956
962 if (vr->flags & SOS_VR_MAP_SHARED) 957 if (vr->flags & SOS_VR_MAP_SHARED)
963 { 958 {
964 959
965 960
966 if ( ( (new_access_rights & SOS_VM_M 961 if ( ( (new_access_rights & SOS_VM_MAP_PROT_READ)
967 && !(vr->mapped_resource->all 962 && !(vr->mapped_resource->allowed_access_rights
968 & SOS_VM_MAP_PROT_READ) 963 & SOS_VM_MAP_PROT_READ) )
969 || ( (new_access_rights & SOS_V 964 || ( (new_access_rights & SOS_VM_MAP_PROT_WRITE)
970 && !(vr->mapped_resource-> 965 && !(vr->mapped_resource->allowed_access_rights
971 & SOS_VM_MAP_PROT_WRI 966 & SOS_VM_MAP_PROT_WRITE) )
972 || ( (new_access_rights & SOS_V 967 || ( (new_access_rights & SOS_VM_MAP_PROT_EXEC)
973 && !(vr->mapped_resource-> 968 && !(vr->mapped_resource->allowed_access_rights
974 & SOS_VM_MAP_PROT_EXE 969 & SOS_VM_MAP_PROT_EXEC) ) )
975 return -SOS_EPERM; 970 return -SOS_EPERM;
976 } 971 }
977 972
978 vr = vr->next_in_as; 973 vr = vr->next_in_as;
979 } 974 }
980 975
981 976
982 977
983 vr = start_vr; 978 vr = start_vr;
984 while (TRUE) 979 while (TRUE)
985 { 980 {
986 981
987 982
988 983
989 if (vr->start + vr->size <= uaddr) 984 if (vr->start + vr->size <= uaddr)
990 985
991 break; 986 break;
992 987
993 988
994 if (uaddr + size <= vr->start) 989 if (uaddr + size <= vr->start)
995 990
996 break; 991 break;
997 992
998 993
999 if (vr->access_rights == new_access_righ 994 if (vr->access_rights == new_access_rights)
1000 995
1001 { 996 {
1002 vr = vr->next_in_as; 997 vr = vr->next_in_as;
1003 continue; 998 continue;
1004 } 999 }
1005 1000
1006 1001
1007 if ((vr->start >= uaddr) 1002 if ((vr->start >= uaddr)
1008 && (vr->start + vr->size <= uaddr + 1003 && (vr->start + vr->size <= uaddr + size))
1009 { 1004 {
1010 1005
1011 as_account_change_of_vr_protection( 1006 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1012 1007 vr->size, vr->access_rights,
1013 1008 new_access_rights);
1014 vr->access_rights = new_access_righ 1009 vr->access_rights = new_access_rights;
1015 1010
1016 if (vr->flags & SOS_VR_MAP_SHARED) 1011 if (vr->flags & SOS_VR_MAP_SHARED)
1017 1012
1018 1013
1019 sos_paging_set_prot_of_interval(v 1014 sos_paging_set_prot_of_interval(vr->start, vr->size,
1020 n 1015 new_access_rights);
1021 else 1016 else
1022 1017
1023 { 1018 {
1024 1019
1025 1020
1026 1021
1027 1022
1028 1023
1029 if (! (new_access_rights & SOS_ 1024 if (! (new_access_rights & SOS_VM_MAP_PROT_WRITE))
1030 sos_paging_set_prot_of_interv 1025 sos_paging_set_prot_of_interval(vr->start, vr->size,
1031 1026 new_access_rights);
1032 } 1027 }
1033 1028
1034 vr = vr->next_in_as; 1029 vr = vr->next_in_as;
1035 continue; 1030 continue;
1036 } 1031 }
1037 1032
1038 1033
1039 else if ( (vr->start < uaddr) 1034 else if ( (vr->start < uaddr)
1040 && (vr->start + vr->size > ua 1035 && (vr->start + vr->size > uaddr + size) )
1041 { 1036 {
1042 1037
1043 1038
1044 1039
1045 SOS_ASSERT_FATAL(! used_preallocate 1040 SOS_ASSERT_FATAL(! used_preallocated_middle_vr);
1046 SOS_ASSERT_FATAL(! used_preallocate 1041 SOS_ASSERT_FATAL(! used_preallocated_right_vr);
1047 used_preallocated_middle_vr = TRUE; 1042 used_preallocated_middle_vr = TRUE;
1048 memcpy(preallocated_middle_vr, vr, 1043 memcpy(preallocated_middle_vr, vr, sizeof(*vr));
1049 used_preallocated_right_vr = TRUE; 1044 used_preallocated_right_vr = TRUE;
1050 memcpy(preallocated_right_vr, vr, s 1045 memcpy(preallocated_right_vr, vr, sizeof(*vr));
1051 1046
1052 1047
1053 preallocated_middle_vr->start = uad 1048 preallocated_middle_vr->start = uaddr;
1054 preallocated_middle_vr->size = siz 1049 preallocated_middle_vr->size = size;
1055 preallocated_right_vr->start = uad 1050 preallocated_right_vr->start = uaddr + size;
1056 preallocated_right_vr->size = vr- 1051 preallocated_right_vr->size = vr->start + vr->size
1057 - 1052 - (uaddr + size);
1058 preallocated_middle_vr->offset_in_r 1053 preallocated_middle_vr->offset_in_resource
1059 += uaddr - vr->start; 1054 += uaddr - vr->start;
1060 preallocated_right_vr->offset_in_re 1055 preallocated_right_vr->offset_in_resource
1061 += uaddr + size - vr->start; 1056 += uaddr + size - vr->start;
1062 vr->size = uaddr - vr->start; 1057 vr->size = uaddr - vr->start;
1063 1058
1064 1059
1065 preallocated_middle_vr->access_righ 1060 preallocated_middle_vr->access_rights = new_access_rights;
1066 as_account_change_of_vr_protection( 1061 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1067 1062 size, vr->access_rights,
1068 1063 new_access_rights);
1069 1064
1070 1065
1071 list_insert_after_named(as->list_vr 1066 list_insert_after_named(as->list_vr, vr, preallocated_middle_vr,
1072 prev_in_as, 1067 prev_in_as, next_in_as);
1073 list_insert_after_named(as->list_vr 1068 list_insert_after_named(as->list_vr, preallocated_middle_vr,
1074 preallocate 1069 preallocated_right_vr,
1075 prev_in_as, 1070 prev_in_as, next_in_as);
1076 1071
1077 list_add_tail_named(vr->mapped_reso 1072 list_add_tail_named(vr->mapped_resource->list_vr,
1078 preallocated_mi 1073 preallocated_middle_vr,
1079 prev_in_mapped_ 1074 prev_in_mapped_resource,
1080 next_in_mapped_ 1075 next_in_mapped_resource);
1081 list_add_tail_named(vr->mapped_reso 1076 list_add_tail_named(vr->mapped_resource->list_vr,
1082 preallocated_ri 1077 preallocated_right_vr,
1083 prev_in_mapped_ 1078 prev_in_mapped_resource,
1084 next_in_mapped_ 1079 next_in_mapped_resource);
1085 1080
1086 1081
1087 if (!(preallocated_middle_vr->flags 1082 if (!(preallocated_middle_vr->flags & SOS_VR_MAP_SHARED)
1088 && (new_access_rights & SOS_VM_ 1083 && (new_access_rights & SOS_VM_MAP_PROT_WRITE))
1089 1084
1090 sos_paging_prepare_COW(preallocat 1085 sos_paging_prepare_COW(preallocated_middle_vr->start,
1091 preallocat 1086 preallocated_middle_vr->size);
1092 else 1087 else
1093 sos_paging_set_prot_of_interval(p 1088 sos_paging_set_prot_of_interval(preallocated_middle_vr->start,
1094 p 1089 preallocated_middle_vr->size,
1095 n 1090 new_access_rights);
1096 1091
1097 if (preallocated_right_vr->ops && p 1092 if (preallocated_right_vr->ops && preallocated_right_vr->ops->ref)
1098 preallocated_right_vr->ops->ref(p 1093 preallocated_right_vr->ops->ref(preallocated_right_vr);
1099 if (preallocated_middle_vr->ops && 1094 if (preallocated_middle_vr->ops && preallocated_middle_vr->ops->ref)
1100 preallocated_middle_vr->ops->ref( 1095 preallocated_middle_vr->ops->ref(preallocated_middle_vr);
1101 1096
1102 1097
1103 break; 1098 break;
1104 } 1099 }
1105 1100
1106 1101
1107 else if (uaddr <= vr->start) 1102 else if (uaddr <= vr->start)
1108 { 1103 {
1109 1104
1110 sos_uoffset_t offset_in_region = ua 1105 sos_uoffset_t offset_in_region = uaddr + size - vr->start;
1111 1106
1112 1107
1113 SOS_ASSERT_FATAL(! used_preallocate 1108 SOS_ASSERT_FATAL(! used_preallocated_middle_vr);
1114 used_preallocated_middle_vr = TRUE; 1109 used_preallocated_middle_vr = TRUE;
1115 memcpy(preallocated_middle_vr, vr, 1110 memcpy(preallocated_middle_vr, vr, sizeof(*vr));
1116 1111
1117 1112
1118 preallocated_middle_vr->start += of 1113 preallocated_middle_vr->start += offset_in_region;
1119 preallocated_middle_vr->size -= of 1114 preallocated_middle_vr->size -= offset_in_region;
1120 vr->size = of 1115 vr->size = offset_in_region;
1121 preallocated_middle_vr->offset_in_r 1116 preallocated_middle_vr->offset_in_resource += offset_in_region;
1122 1117
1123 1118
1124 as_account_change_of_vr_protection( 1119 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1125 1120 vr->size,
1126 1121 vr->access_rights,
1127 1122 new_access_rights);
1128 vr->access_rights = new_access_righ 1123 vr->access_rights = new_access_rights;
1129 1124
1130 1125
1131 list_insert_after_named(as->list_vr 1126 list_insert_after_named(as->list_vr, vr,
1132 preallocate 1127 preallocated_middle_vr,
1133 prev_in_as, 1128 prev_in_as, next_in_as);
1134 list_add_tail_named(vr->mapped_reso 1129 list_add_tail_named(vr->mapped_resource->list_vr,
1135 preallocated_mi 1130 preallocated_middle_vr,
1136 prev_in_mapped_ 1131 prev_in_mapped_resource,
1137 next_in_mapped_ 1132 next_in_mapped_resource);
1138 1133
1139 1134
1140 if (!(vr->flags & SOS_VR_MAP_SHARED 1135 if (!(vr->flags & SOS_VR_MAP_SHARED)
1141 && (new_access_rights & SOS_VM_ 1136 && (new_access_rights & SOS_VM_MAP_PROT_WRITE))
1142 1137
1143 sos_paging_prepare_COW(vr->start, 1138 sos_paging_prepare_COW(vr->start, vr->size);
1144 else 1139 else
1145 sos_paging_set_prot_of_interval(v 1140 sos_paging_set_prot_of_interval(vr->start, vr->size,
1146 n 1141 new_access_rights);
1147 1142
1148 if (preallocated_middle_vr->ops && 1143 if (preallocated_middle_vr->ops && preallocated_middle_vr->ops->ref)
1149 preallocated_middle_vr->ops->ref( 1144 preallocated_middle_vr->ops->ref(preallocated_middle_vr);
1150 1145
1151 1146
1152 1147
1153 break; 1148 break;
1154 } 1149 }
1155 1150
1156 1151
1157 else if (uaddr + size >= vr->start + vr 1152 else if (uaddr + size >= vr->start + vr->size)
1158 { 1153 {
1159 1154
1160 sos_uoffset_t offset_in_region = ua 1155 sos_uoffset_t offset_in_region = uaddr - vr->start;
1161 1156
1162 1157
1163 SOS_ASSERT_FATAL(! used_preallocate 1158 SOS_ASSERT_FATAL(! used_preallocated_right_vr);
1164 used_preallocated_right_vr = TRUE; 1159 used_preallocated_right_vr = TRUE;
1165 memcpy(preallocated_right_vr, vr, s 1160 memcpy(preallocated_right_vr, vr, sizeof(*vr));
1166 1161
1167 1162
1168 preallocated_right_vr->start 1163 preallocated_right_vr->start += offset_in_region;
1169 preallocated_right_vr->size 1164 preallocated_right_vr->size -= offset_in_region;
1170 vr->size 1165 vr->size = offset_in_region;
1171 preallocated_right_vr->offset_in_re 1166 preallocated_right_vr->offset_in_resource += offset_in_region;
1172 1167
1173 1168
1174 as_account_change_of_vr_protection( 1169 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1175 1170 preallocated_right_vr->size,
1176 1171 vr->access_rights,
1177 1172 new_access_rights);
1178 preallocated_right_vr->access_right 1173 preallocated_right_vr->access_rights = new_access_rights;
1179 1174
1180 1175
1181 list_insert_after_named(as->list_vr 1176 list_insert_after_named(as->list_vr, vr,
1182 preallocate 1177 preallocated_right_vr,
1183 prev_in_as, 1178 prev_in_as, next_in_as);
1184 list_add_tail_named(vr->mapped_reso 1179 list_add_tail_named(vr->mapped_resource->list_vr,
1185 preallocated_ri 1180 preallocated_right_vr,
1186 prev_in_mapped_ 1181 prev_in_mapped_resource,
1187 next_in_mapped_ 1182 next_in_mapped_resource);
1188 1183
1189 1184
1190 if (!(preallocated_right_vr->flags 1185 if (!(preallocated_right_vr->flags & SOS_VR_MAP_SHARED)
1191 && (new_access_rights & SOS_VM_ 1186 && (new_access_rights & SOS_VM_MAP_PROT_WRITE))
1192 1187
1193 sos_paging_prepare_COW(preallocat 1188 sos_paging_prepare_COW(preallocated_right_vr->start,
1194 preallocat 1189 preallocated_right_vr->size);
1195 else 1190 else
1196 sos_paging_set_prot_of_interval(p 1191 sos_paging_set_prot_of_interval(preallocated_right_vr->start,
1197 p 1192 preallocated_right_vr->size,
1198 n 1193 new_access_rights);
1199 1194
1200 if (preallocated_right_vr->ops && p 1195 if (preallocated_right_vr->ops && preallocated_right_vr->ops->ref)
1201 preallocated_right_vr->ops->ref(p 1196 preallocated_right_vr->ops->ref(preallocated_right_vr);
1202 1197
1203 vr = vr->next_in_as; 1198 vr = vr->next_in_as;
1204 continue; 1199 continue;
1205 } 1200 }
1206 1201
1207 sos_display_fatal_error("BUG"); 1202 sos_display_fatal_error("BUG");
1208 } 1203 }
1209 1204
1210 if (! used_preallocated_middle_vr) 1205 if (! used_preallocated_middle_vr)
1211 sos_kmem_vmm_free((sos_vaddr_t)preallocat 1206 sos_kmem_vmm_free((sos_vaddr_t)preallocated_middle_vr);
1212 if (! used_preallocated_right_vr) 1207 if (! used_preallocated_right_vr)
1213 sos_kmem_vmm_free((sos_vaddr_t)preallocat 1208 sos_kmem_vmm_free((sos_vaddr_t)preallocated_right_vr);
1214 1209
1215 return SOS_OK; 1210 return SOS_OK;
1216 } 1211 }
1217 1212
1218 1213
1219 sos_ret_t 1214 sos_ret_t
1220 sos_umem_vmm_resize(struct sos_umem_vmm_as * 1215 sos_umem_vmm_resize(struct sos_umem_vmm_as * as,
1221 sos_uaddr_t old_uaddr, so 1216 sos_uaddr_t old_uaddr, sos_size_t old_size,
1222 sos_uaddr_t *new_uaddr, s 1217 sos_uaddr_t *new_uaddr, sos_size_t new_size,
1223 sos_ui32_t flags) 1218 sos_ui32_t flags)
1224 { 1219 {
1225 sos_luoffset_t new_offset_in_resource; 1220 sos_luoffset_t new_offset_in_resource;
1226 sos_bool_t must_move_vr = FALSE; 1221 sos_bool_t must_move_vr = FALSE;
1227 struct sos_umem_vmm_vr *vr, *prev_vr, *next 1222 struct sos_umem_vmm_vr *vr, *prev_vr, *next_vr;
1228 1223
1229 1224
1230 if (*new_uaddr < SOS_PAGING_BASE_USER_ADDRE 1225 if (*new_uaddr < SOS_PAGING_BASE_USER_ADDRESS)
1231 return -SOS_EINVAL; 1226 return -SOS_EINVAL;
1232 if (*new_uaddr > SOS_PAGING_TOP_USER_ADDRES 1227 if (*new_uaddr > SOS_PAGING_TOP_USER_ADDRESS - new_size)
1233 return -SOS_EINVAL; 1228 return -SOS_EINVAL;
1234 1229
1235 old_uaddr = SOS_PAGE_ALIGN_INF(old_uaddr); 1230 old_uaddr = SOS_PAGE_ALIGN_INF(old_uaddr);
1236 old_size = SOS_PAGE_ALIGN_SUP(old_size); 1231 old_size = SOS_PAGE_ALIGN_SUP(old_size);
1237 if (! SOS_IS_PAGE_ALIGNED(*new_uaddr)) 1232 if (! SOS_IS_PAGE_ALIGNED(*new_uaddr))
1238 return -SOS_EINVAL; 1233 return -SOS_EINVAL;
1239 if (new_size <= 0) 1234 if (new_size <= 0)
1240 return -SOS_EINVAL; 1235 return -SOS_EINVAL;
1241 new_size = SOS_PAGE_ALIGN_SUP(new_size); 1236 new_size = SOS_PAGE_ALIGN_SUP(new_size);
1242 1237
1243 1238
1244 vr = find_first_intersecting_vr(as, old_uad 1239 vr = find_first_intersecting_vr(as, old_uaddr, old_size);
1245 if (! vr) 1240 if (! vr)
1246 return -SOS_EINVAL; 1241 return -SOS_EINVAL;
1247 1242
1248 1243
1249 if ( (vr->start > old_uaddr) 1244 if ( (vr->start > old_uaddr)
1250 || (vr->start + vr->size < old_uaddr + 1245 || (vr->start + vr->size < old_uaddr + old_size) )
1251 return -SOS_EINVAL; 1246 return -SOS_EINVAL;
1252 1247
1253 1248
1254 1249
1255 prev_vr = vr->prev_in_as; 1250 prev_vr = vr->prev_in_as;
1256 if (prev_vr->start >= vr->start) 1251 if (prev_vr->start >= vr->start)
1257 prev_vr = NULL; 1252 prev_vr = NULL;
1258 next_vr = vr->prev_in_as; 1253 next_vr = vr->prev_in_as;
1259 if (next_vr->start <= vr->start) 1254 if (next_vr->start <= vr->start)
1260 next_vr = NULL; 1255 next_vr = NULL;
1261 1256
1262 1257
1263 1258
1264 1259
1265 1260
1266 1261
1267 1262
1268 if ( (*new_uaddr < vr->start) 1263 if ( (*new_uaddr < vr->start)
1269 && (vr->start - *new_uaddr > vr->offse 1264 && (vr->start - *new_uaddr > vr->offset_in_resource) )
1270 return -SOS_EINVAL; 1265 return -SOS_EINVAL;
1271 1266
1272 1267
1273 if (vr->start > *new_uaddr) 1268 if (vr->start > *new_uaddr)
1274 new_offset_in_resource 1269 new_offset_in_resource
1275 = vr->offset_in_resource 1270 = vr->offset_in_resource
1276 - (vr->start - *new_uaddr); 1271 - (vr->start - *new_uaddr);
1277 else 1272 else
1278 new_offset_in_resource 1273 new_offset_in_resource
1279 = vr->offset_in_resource 1274 = vr->offset_in_resource
1280 + (*new_uaddr - vr->start); 1275 + (*new_uaddr - vr->start);
1281 1276
1282 1277
1283 1278
1284 if (prev_vr && (prev_vr->start + prev_vr->s 1279 if (prev_vr && (prev_vr->start + prev_vr->size > *new_uaddr))
1285 must_move_vr |= TRUE; 1280 must_move_vr |= TRUE;
1286 if (next_vr && (next_vr->start < *new_uaddr 1281 if (next_vr && (next_vr->start < *new_uaddr + new_size))
1287 must_move_vr |= TRUE; 1282 must_move_vr |= TRUE;
1288 1283
1289 1284
1290 if (*new_uaddr < SOS_PAGING_BASE_USER_ADDRE 1285 if (*new_uaddr < SOS_PAGING_BASE_USER_ADDRESS)
1291 must_move_vr |= TRUE; 1286 must_move_vr |= TRUE;
1292 if (*new_uaddr > SOS_PAGING_TOP_USER_ADDRES 1287 if (*new_uaddr > SOS_PAGING_TOP_USER_ADDRESS - new_size)
1293 must_move_vr |= TRUE; 1288 must_move_vr |= TRUE;
1294 1289
1295 1290
1296 if ( must_move_vr && !(flags & SOS_VR_REMAP 1291 if ( must_move_vr && !(flags & SOS_VR_REMAP_MAYMOVE) )
1297 return -SOS_EINVAL; 1292 return -SOS_EINVAL;
1298 1293
1299 1294
1300 1295
1301 if (must_move_vr) 1296 if (must_move_vr)
1302 { 1297 {
1303 sos_uaddr_t uaddr, result_uaddr; 1298 sos_uaddr_t uaddr, result_uaddr;
1304 sos_ret_t retval; 1299 sos_ret_t retval;
1305 1300
1306 result_uaddr = *new_uaddr; 1301 result_uaddr = *new_uaddr;
1307 retval = sos_umem_vmm_map(as, & result_ 1302 retval = sos_umem_vmm_map(as, & result_uaddr, new_size,
1308 vr->access_ri 1303 vr->access_rights,
1309 vr->flags | I 1304 vr->flags | INTERNAL_MAP_CALLED_FROM_MREMAP,
1310 vr->mapped_re 1305 vr->mapped_resource,
1311 new_offset_in 1306 new_offset_in_resource);
1312 if (SOS_OK != retval) 1307 if (SOS_OK != retval)
1313 return retval; 1308 return retval;
1314 1309
1315 1310
1316 for (uaddr = vr->start ; 1311 for (uaddr = vr->start ;
1317 uaddr < vr->start + vr->size ; 1312 uaddr < vr->start + vr->size ;
1318 uaddr += SOS_PAGE_SIZE) 1313 uaddr += SOS_PAGE_SIZE)
1319 { 1314 {
1320 sos_paddr_t paddr; 1315 sos_paddr_t paddr;
1321 sos_ui32_t prot; 1316 sos_ui32_t prot;
1322 sos_uaddr_t vaddr; 1317 sos_uaddr_t vaddr;
1323 1318
1324 if (uaddr < *new_uaddr) 1319 if (uaddr < *new_uaddr)
1325 continue; 1320 continue;
1326 if (uaddr > *new_uaddr + new_size) 1321 if (uaddr > *new_uaddr + new_size)
1327 continue; 1322 continue;
1328 1323
1329 1324
1330 1325
1331 if (vr->start >= *new_uaddr) 1326 if (vr->start >= *new_uaddr)
1332 vaddr = result_uaddr 1327 vaddr = result_uaddr
1333 + (uaddr - vr->start) 1328 + (uaddr - vr->start)
1334 + (vr->start - *new_uaddr); 1329 + (vr->start - *new_uaddr);
1335 else 1330 else
1336 vaddr = result_uaddr 1331 vaddr = result_uaddr
1337 + (uaddr - vr->start) 1332 + (uaddr - vr->start)
1338 - (*new_uaddr - vr->start); 1333 - (*new_uaddr - vr->start);
1339 1334
1340 paddr = sos_paging_get_paddr(uaddr) 1335 paddr = sos_paging_get_paddr(uaddr);
1341 if (! paddr) 1336 if (! paddr)
1342 1337
1343 continue; 1338 continue;
1344 1339
1345 prot = sos_paging_get_prot(uaddr); 1340 prot = sos_paging_get_prot(uaddr);
1346 SOS_ASSERT_FATAL(prot); 1341 SOS_ASSERT_FATAL(prot);
1347 1342
1348 1343
1349 retval = sos_paging_map(paddr, vadd 1344 retval = sos_paging_map(paddr, vaddr, TRUE, prot);
1350 if (SOS_OK != retval) 1345 if (SOS_OK != retval)
1351 { 1346 {
1352 sos_umem_vmm_unmap(as, result_u 1347 sos_umem_vmm_unmap(as, result_uaddr, new_size);
1353 return retval; 1348 return retval;
1354 } 1349 }
1355 } 1350 }
1356 1351
1357 retval = sos_umem_vmm_unmap(as, vr->sta 1352 retval = sos_umem_vmm_unmap(as, vr->start, vr->size);
1358 if (SOS_OK != retval) 1353 if (SOS_OK != retval)
1359 { 1354 {
1360 sos_umem_vmm_unmap(as, result_uaddr 1355 sos_umem_vmm_unmap(as, result_uaddr, new_size);
1361 return retval; 1356 return retval;
1362 } 1357 }
1363 1358
1364 *new_uaddr = result_uaddr; 1359 *new_uaddr = result_uaddr;
1365 return retval; 1360 return retval;
1366 } 1361 }
1367 1362
1368 1363
1369 1364
1370 1365
1371 if (*new_uaddr + new_size < vr->start + vr- 1366 if (*new_uaddr + new_size < vr->start + vr->size)
1372 sos_umem_vmm_unmap(as, *new_uaddr + new_s 1367 sos_umem_vmm_unmap(as, *new_uaddr + new_size,
1373 vr->start + vr->size - 1368 vr->start + vr->size - (*new_uaddr + new_size));
1374 else 1369 else
1375 { 1370 {
1376 as_account_change_of_vr_protection(as, 1371 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1377 *new 1372 *new_uaddr + new_size
1378 - 1373 - (vr->start + vr->size),
1379 0, v 1374 0, vr->access_rights);
1380 vr->size += *new_uaddr + new_size - (vr 1375 vr->size += *new_uaddr + new_size - (vr->start + vr->size);
1381 } 1376 }
1382 1377
1383 if (*new_uaddr > vr->start) 1378 if (*new_uaddr > vr->start)
1384 sos_umem_vmm_unmap(as, vr->start, *new_ua 1379 sos_umem_vmm_unmap(as, vr->start, *new_uaddr - vr->start);
1385 else 1380 else
1386 { 1381 {
1387 as_account_change_of_vr_protection(as, 1382 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1388 vr-> 1383 vr->start - *new_uaddr,
1389 0, v 1384 0, vr->access_rights);
1390 vr->size += vr->start - *new_uaddr; 1385 vr->size += vr->start - *new_uaddr;
1391 vr->start = *new_uaddr; 1386 vr->start = *new_uaddr;
1392 vr->offset_in_resource = new_offset_in_ 1387 vr->offset_in_resource = new_offset_in_resource;
1393 } 1388 }
1394 1389
1395 SOS_ASSERT_FATAL(vr->start == *new_uaddr); 1390 SOS_ASSERT_FATAL(vr->start == *new_uaddr);
1396 SOS_ASSERT_FATAL(vr->size == new_size); 1391 SOS_ASSERT_FATAL(vr->size == new_size);
1397 SOS_ASSERT_FATAL(vr->offset_in_resource == 1392 SOS_ASSERT_FATAL(vr->offset_in_resource == new_offset_in_resource);
1398 1393
1399 return SOS_OK; 1394 return SOS_OK;
1400 } 1395 }
1401 1396
1402 1397
1403 sos_ret_t sos_umem_vmm_try_resolve_page_fault 1398 sos_ret_t sos_umem_vmm_try_resolve_page_fault(sos_uaddr_t uaddr,
1404 1399 sos_bool_t write_access,
1405 1400 sos_bool_t user_access)
1406 { 1401 {
1407 struct sos_process *process = sos_threa 1402 struct sos_process *process = sos_thread_get_current()->process;
1408 struct sos_umem_vmm_as *as; 1403 struct sos_umem_vmm_as *as;
1409 struct sos_umem_vmm_vr *vr; 1404 struct sos_umem_vmm_vr *vr;
1410 1405
1411 if (! process) 1406 if (! process)
1412 return -SOS_EFAULT; 1407 return -SOS_EFAULT;
1413 1408
1414 as = sos_process_get_address_space(process) 1409 as = sos_process_get_address_space(process);
1415 if (! as) 1410 if (! as)
1416 return -SOS_EFAULT; 1411 return -SOS_EFAULT;
1417 1412
1418 vr = find_first_intersecting_vr(as, uaddr, 1413 vr = find_first_intersecting_vr(as, uaddr, 1);
1419 if (! vr) 1414 if (! vr)
1420 return -SOS_EFAULT; 1415 return -SOS_EFAULT;
1421 1416
1422 1417
1423 if (write_access && !(vr->access_rights & S 1418 if (write_access && !(vr->access_rights & SOS_VM_MAP_PROT_WRITE))
1424 return -SOS_EFAULT; 1419 return -SOS_EFAULT;
1425 1420
1426 1421
1427 if (write_access && !(vr->flags & SOS_VR_MA 1422 if (write_access && !(vr->flags & SOS_VR_MAP_SHARED))
1428 { 1423 {
1429 if (SOS_OK == sos_paging_try_resolve_CO 1424 if (SOS_OK == sos_paging_try_resolve_COW(uaddr))
1430 { 1425 {
1431 as->pgflt_cow ++; 1426 as->pgflt_cow ++;
1432 return SOS_OK; 1427 return SOS_OK;
1433 } 1428 }
1434 } 1429 }
1435 1430
1436 1431
1437 if (SOS_OK != vr->ops->page_in(vr, uaddr, w 1432 if (SOS_OK != vr->ops->page_in(vr, uaddr, write_access))
1438 { 1433 {
1439 as->pgflt_invalid ++; 1434 as->pgflt_invalid ++;
1440 return -SOS_EFAULT; 1435 return -SOS_EFAULT;
1441 } 1436 }
1442 1437
1443 as->phys_total += SOS_PAGE_SIZE; 1438 as->phys_total += SOS_PAGE_SIZE;
1444 as->pgflt_page_in ++; 1439 as->pgflt_page_in ++;
1445 1440
1446 1441
1447 if (!(vr->flags & SOS_VR_MAP_SHARED)) 1442 if (!(vr->flags & SOS_VR_MAP_SHARED))
1448 { 1443 {
1449 sos_paging_prepare_COW(SOS_PAGE_ALIGN_I 1444 sos_paging_prepare_COW(SOS_PAGE_ALIGN_INF(uaddr),
1450 SOS_PAGE_SIZE); 1445 SOS_PAGE_SIZE);
1451 } 1446 }
1452 1447
1453 return SOS_OK; 1448 return SOS_OK;
1454 } 1449 }
1455 1450
1456 1451
1457 sos_ret_t 1452 sos_ret_t
1458 sos_umem_vmm_init_heap(struct sos_umem_vmm_as 1453 sos_umem_vmm_init_heap(struct sos_umem_vmm_as * as,
1459 sos_uaddr_t heap_start 1454 sos_uaddr_t heap_start)
1460 { 1455 {
1461 SOS_ASSERT_FATAL(! as->heap_start); 1456 SOS_ASSERT_FATAL(! as->heap_start);
1462 1457
1463 as->heap_start = heap_start; 1458 as->heap_start = heap_start;
1464 as->heap_size = 0; 1459 as->heap_size = 0;
1465 return SOS_OK; 1460 return SOS_OK;
1466 } 1461 }
1467 1462
1468 1463
1469 sos_uaddr_t 1464 sos_uaddr_t
1470 sos_umem_vmm_brk(struct sos_umem_vmm_as * as, 1465 sos_umem_vmm_brk(struct sos_umem_vmm_as * as,
1471 sos_uaddr_t new_top_uaddr) 1466 sos_uaddr_t new_top_uaddr)
1472 { 1467 {
1473 sos_uaddr_t new_start; 1468 sos_uaddr_t new_start;
1474 sos_size_t new_size; 1469 sos_size_t new_size;
1475 SOS_ASSERT_FATAL(as->heap_start); 1470 SOS_ASSERT_FATAL(as->heap_start);
1476 1471
1477 if (! new_top_uaddr) 1472 if (! new_top_uaddr)
1478 return as->heap_start + as->heap_size; 1473 return as->heap_start + as->heap_size;
1479 1474
1480 if (new_top_uaddr == as->heap_start + as->h 1475 if (new_top_uaddr == as->heap_start + as->heap_size)
1481 return as->heap_start + as->heap_size; 1476 return as->heap_start + as->heap_size;
1482 1477
1483 if (new_top_uaddr < as->heap_start) 1478 if (new_top_uaddr < as->heap_start)
1484 return (sos_uaddr_t)NULL; 1479 return (sos_uaddr_t)NULL;
1485 1480
1486 new_top_uaddr = SOS_PAGE_ALIGN_SUP(new_top_ 1481 new_top_uaddr = SOS_PAGE_ALIGN_SUP(new_top_uaddr);
1487 new_start = as->heap_start; 1482 new_start = as->heap_start;
1488 new_size = new_top_uaddr - as->heap_start; 1483 new_size = new_top_uaddr - as->heap_start;
1489 1484
1490 1485
1491 if (! as->heap_size) 1486 if (! as->heap_size)
1492 { 1487 {
1493 if (SOS_OK != sos_dev_zero_map(as, & as 1488 if (SOS_OK != sos_dev_zero_map(as, & as->heap_start,
1494 new_size 1489 new_size,
1495 SOS_VM_M 1490 SOS_VM_MAP_PROT_READ
1496 | SOS_VM 1491 | SOS_VM_MAP_PROT_WRITE,
1497 0 1492 0 ))
1498 return (sos_uaddr_t)NULL; 1493 return (sos_uaddr_t)NULL;
1499 1494
1500 as->heap_size = new_size; 1495 as->heap_size = new_size;
1501 return as->heap_start + as->heap_size; 1496 return as->heap_start + as->heap_size;
1502 } 1497 }
1503 1498
1504 1499
1505 if (new_size <= 0) 1500 if (new_size <= 0)
1506 { 1501 {
1507 if (SOS_OK != sos_umem_vmm_unmap(as, 1502 if (SOS_OK != sos_umem_vmm_unmap(as,
1508 as->he 1503 as->heap_start, as->heap_size))
1509 return (sos_uaddr_t)NULL; 1504 return (sos_uaddr_t)NULL;
1510 } 1505 }
1511 else 1506 else
1512 { 1507 {
1513 if (SOS_OK != sos_umem_vmm_resize(as, 1508 if (SOS_OK != sos_umem_vmm_resize(as,
1514 as->h 1509 as->heap_start, as->heap_size,
1515 & new 1510 & new_start, new_size,
1516 0)) 1511 0))
1517 return (sos_uaddr_t)NULL; 1512 return (sos_uaddr_t)NULL;
1518 } 1513 }
1519 1514
1520 SOS_ASSERT_FATAL(new_start == as->heap_star 1515 SOS_ASSERT_FATAL(new_start == as->heap_start);
1521 as->heap_size = new_size; 1516 as->heap_size = new_size;
1522 return new_top_uaddr; 1517 return new_top_uaddr;
1523 } 1518 }
1524 1519
1525 1520
1526 static struct sos_umem_vmm_vr * 1521 static struct sos_umem_vmm_vr *
1527 find_enclosing_or_next_vr(struct sos_umem_vmm 1522 find_enclosing_or_next_vr(struct sos_umem_vmm_as * as,
1528 sos_uaddr_t uaddr) 1523 sos_uaddr_t uaddr)
1529 { 1524 {
1530 struct sos_umem_vmm_vr *vr; 1525 struct sos_umem_vmm_vr *vr;
1531 int nb_vr; 1526 int nb_vr;
1532 1527
1533 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS) 1528 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS)
1534 return NULL; 1529 return NULL;
1535 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS) 1530 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS)
1536 return NULL; 1531 return NULL;
1537 1532
1538 list_foreach_named(as->list_vr, vr, nb_vr, 1533 list_foreach_named(as->list_vr, vr, nb_vr, prev_in_as, next_in_as)
1539 { 1534 {
1540 1535
1541 1536
1542 if (uaddr <= vr->start + (vr->size - 1) 1537 if (uaddr <= vr->start + (vr->size - 1))
1543 return vr; 1538 return vr;
1544 } 1539 }
1545 1540
1546 return NULL; 1541 return NULL;
1547 } 1542 }
1548 1543
1549 1544
1550 static struct sos_umem_vmm_vr * 1545 static struct sos_umem_vmm_vr *
1551 find_first_intersecting_vr(struct sos_umem_vm 1546 find_first_intersecting_vr(struct sos_umem_vmm_as * as,
1552 sos_uaddr_t start_ 1547 sos_uaddr_t start_uaddr, sos_size_t size)
1553 { 1548 {
1554 struct sos_umem_vmm_vr * vr; 1549 struct sos_umem_vmm_vr * vr;
1555 vr = find_enclosing_or_next_vr(as, start_ua 1550 vr = find_enclosing_or_next_vr(as, start_uaddr);
1556 if (! vr) 1551 if (! vr)
1557 return NULL; 1552 return NULL;
1558 1553
1559 if (start_uaddr + size <= vr->start) 1554 if (start_uaddr + size <= vr->start)
1560 return NULL; 1555 return NULL;
1561 1556
1562 return vr; 1557 return vr;
1563 } 1558 }
1564 1559
1565 1560
1566 static sos_uaddr_t 1561 static sos_uaddr_t
1567 find_first_free_interval(struct sos_umem_vmm_ 1562 find_first_free_interval(struct sos_umem_vmm_as * as,
1568 sos_uaddr_t hint_uad 1563 sos_uaddr_t hint_uaddr, sos_size_t size)
1569 { 1564 {
1570 struct sos_umem_vmm_vr * initial_vr, * vr; 1565 struct sos_umem_vmm_vr * initial_vr, * vr;
1571 1566
1572 if (hint_uaddr < SOS_PAGING_BASE_USER_ADDRE 1567 if (hint_uaddr < SOS_PAGING_BASE_USER_ADDRESS)
1573 hint_uaddr = SOS_PAGING_BASE_USER_ADDRESS 1568 hint_uaddr = SOS_PAGING_BASE_USER_ADDRESS;
1574 1569
1575 if (hint_uaddr > SOS_PAGING_TOP_USER_ADDRES 1570 if (hint_uaddr > SOS_PAGING_TOP_USER_ADDRESS - size + 1)
1576 return (sos_uaddr_t)NULL; 1571 return (sos_uaddr_t)NULL;
1577 1572
1578 initial_vr = vr = find_enclosing_or_next_vr 1573 initial_vr = vr = find_enclosing_or_next_vr(as, hint_uaddr);
1579 if (! vr) 1574 if (! vr)
1580 1575
1581 return hint_uaddr; 1576 return hint_uaddr;
1582 1577
1583 1578
1584 do 1579 do
1585 { 1580 {
1586 1581
1587 if (hint_uaddr + size <= vr->start) 1582 if (hint_uaddr + size <= vr->start)
1588 1583
1589 return hint_uaddr; 1584 return hint_uaddr;
1590 1585
1591 1586
1592 1587
1593 if (vr->next_in_as->start >= hint_uaddr 1588 if (vr->next_in_as->start >= hint_uaddr)
1594 1589
1595 hint_uaddr = vr->start + vr->size; 1590 hint_uaddr = vr->start + vr->size;
1596 else 1591 else
1597 { 1592 {
1598 1593
1599 1594
1600 1595
1601 if (hint_uaddr <= SOS_PAGING_TOP_US 1596 if (hint_uaddr <= SOS_PAGING_TOP_USER_ADDRESS - size)
1602 return hint_uaddr; 1597 return hint_uaddr;
1603 1598
1604 hint_uaddr = SOS_PAGING_BASE_USER_A 1599 hint_uaddr = SOS_PAGING_BASE_USER_ADDRESS;
1605 } 1600 }
1606 1601
1607 1602
1608 vr = vr->next_in_as; 1603 vr = vr->next_in_as;
1609 } 1604 }
1610 while (vr != initial_vr); 1605 while (vr != initial_vr);
1611 1606
1612 1607
1613 1608
1614 1609
1615 return (sos_uaddr_t)NULL; 1610 return (sos_uaddr_t)NULL;
1616 } 1611 }
1617 1612
1618 1613
1619 static void 1614 static void
1620 as_account_change_of_vr_protection(struct sos 1615 as_account_change_of_vr_protection(struct sos_umem_vmm_as * as,
1621 sos_bool_t 1616 sos_bool_t is_shared,
1622 sos_size_t 1617 sos_size_t size,
1623 sos_ui32_t 1618 sos_ui32_t prev_access_rights,
1624 sos_ui32_t 1619 sos_ui32_t new_access_rights)
1625 { 1620 {
1626 if (prev_access_rights == new_access_rights 1621 if (prev_access_rights == new_access_rights)
1627 return; 1622 return;
1628 1623
1629 #define _UPDATE_VMSTAT(field,is_increment) \ 1624 #define _UPDATE_VMSTAT(field,is_increment) \
1630 ({ if (is_increment > 0) \ 1625 ({ if (is_increment > 0) \
1631 as->field += size; \ 1626 as->field += size; \
1632 else \ 1627 else \
1633 { SOS_ASSERT_FATAL(as->field >= size); 1628 { SOS_ASSERT_FATAL(as->field >= size); as->field -= size; } })
1634 #define UPDATE_VMSTAT(field,is_increment) \ 1629 #define UPDATE_VMSTAT(field,is_increment) \
1635 ({ if (is_shared) _UPDATE_VMSTAT(vm_shrd.fi 1630 ({ if (is_shared) _UPDATE_VMSTAT(vm_shrd.field, is_increment); \
1636 _UPDATE_VMSTAT(vm_total.field, is_increm 1631 _UPDATE_VMSTAT(vm_total.field, is_increment); \
1637 SOS_ASSERT_FATAL(as->vm_total.field >= a 1632 SOS_ASSERT_FATAL(as->vm_total.field >= as->vm_shrd.field); })
1638 1633
1639 if ( (new_access_rights & SOS_VM_MAP_PROT_W 1634 if ( (new_access_rights & SOS_VM_MAP_PROT_WRITE)
1640 && !(prev_access_rights & SOS_VM_MAP_P 1635 && !(prev_access_rights & SOS_VM_MAP_PROT_WRITE))
1641 { 1636 {
1642 UPDATE_VMSTAT(rw, +1); 1637 UPDATE_VMSTAT(rw, +1);
1643 if (prev_access_rights & SOS_VM_MAP_PRO 1638 if (prev_access_rights & SOS_VM_MAP_PROT_READ)
1644 UPDATE_VMSTAT(ro, -1); 1639 UPDATE_VMSTAT(ro, -1);
1645 } 1640 }
1646 else if ( !(new_access_rights & SOS_VM_MAP_ 1641 else if ( !(new_access_rights & SOS_VM_MAP_PROT_WRITE)
1647 && (prev_access_rights & SOS_VM_M 1642 && (prev_access_rights & SOS_VM_MAP_PROT_WRITE))
1648 { 1643 {
1649 if (new_access_rights & SOS_VM_MAP_PROT 1644 if (new_access_rights & SOS_VM_MAP_PROT_READ)
1650 UPDATE_VMSTAT(ro, +1); 1645 UPDATE_VMSTAT(ro, +1);
1651 UPDATE_VMSTAT(rw, -1); 1646 UPDATE_VMSTAT(rw, -1);
1652 } 1647 }
1653 else if (new_access_rights & SOS_VM_MAP_PRO 1648 else if (new_access_rights & SOS_VM_MAP_PROT_READ)
1654 UPDATE_VMSTAT(ro, +1); 1649 UPDATE_VMSTAT(ro, +1);
1655 else if (!(new_access_rights & SOS_VM_MAP_P 1650 else if (!(new_access_rights & SOS_VM_MAP_PROT_READ))
1656 UPDATE_VMSTAT(ro, -1); 1651 UPDATE_VMSTAT(ro, -1);
1657 1652
1658 if ( (new_access_rights & SOS_VM_MAP_PROT_E 1653 if ( (new_access_rights & SOS_VM_MAP_PROT_EXEC)
1659 && !(prev_access_rights & SOS_VM_MAP_P 1654 && !(prev_access_rights & SOS_VM_MAP_PROT_EXEC))
1660 { 1655 {
1661 UPDATE_VMSTAT(code, +1); 1656 UPDATE_VMSTAT(code, +1);
1662 } 1657 }
1663 else if ( !(new_access_rights & SOS_VM_MAP_ 1658 else if ( !(new_access_rights & SOS_VM_MAP_PROT_EXEC)
1664 && (prev_access_rights & SOS_VM_M 1659 && (prev_access_rights & SOS_VM_MAP_PROT_EXEC))
1665 { 1660 {
1666 UPDATE_VMSTAT(code, -1); 1661 UPDATE_VMSTAT(code, -1);
1667 } 1662 }
1668 1663
1669 if (new_access_rights && !prev_access_right 1664 if (new_access_rights && !prev_access_rights)
1670 UPDATE_VMSTAT(overall, +1); 1665 UPDATE_VMSTAT(overall, +1);
1671 else if (!new_access_rights && prev_access_ 1666 else if (!new_access_rights && prev_access_rights)
1672 UPDATE_VMSTAT(overall, -1); 1667 UPDATE_VMSTAT(overall, -1);
1673 1668
1674 } 1669 }