Diff markup
001 001
002 002
003 003
004 004
005 005
006 006
007 007
008 008
009 009
010 010
011 011
012 012
013 013
014 014
015 015
016 016
017 017
018 018
019 #include <sos/assert.h> 019 #include <sos/assert.h>
020 #include <sos/list.h> 020 #include <sos/list.h>
021 #include <sos/physmem.h> 021 #include <sos/physmem.h>
022 #include <sos/kmem_slab.h> 022 #include <sos/kmem_slab.h>
023 #include <drivers/bochs.h> 023 #include <drivers/bochs.h>
024 #include <hwcore/mm_context.h> 024 #include <hwcore/mm_context.h>
025 #include <hwcore/paging.h> 025 #include <hwcore/paging.h>
026 #include <drivers/zero.h> 026 #include <drivers/zero.h>
027 027
028 #include "umem_vmm.h" 028 #include "umem_vmm.h"
029 029
030 030
031 struct sos_umem_vmm_as 031 struct sos_umem_vmm_as
032 { 032 {
033 033
034 struct sos_process * process; 034 struct sos_process * process;
035 035
036 036
037 struct sos_mm_context * mm_context; 037 struct sos_mm_context * mm_context;
038 038
039 039
040 struct sos_umem_vmm_vr * list_vr; 040 struct sos_umem_vmm_vr * list_vr;
041 041
042 042
043 sos_uaddr_t heap_start; 043 sos_uaddr_t heap_start;
044 sos_size_t heap_size; 044 sos_size_t heap_size;
045 045
046 046
047 sos_size_t phys_total; 047 sos_size_t phys_total;
048 struct vm_usage 048 struct vm_usage
049 { 049 {
050 sos_size_t overall; 050 sos_size_t overall;
051 sos_size_t ro, rw, code 051 sos_size_t ro, rw, code ;
052 } vm_total, vm_shrd; 052 } vm_total, vm_shrd;
053 053
054 054
055 sos_size_t pgflt_cow; 055 sos_size_t pgflt_cow;
056 sos_size_t pgflt_page_in; 056 sos_size_t pgflt_page_in;
057 sos_size_t pgflt_invalid; 057 sos_size_t pgflt_invalid;
058 }; 058 };
059 059
060 060
061 struct sos_umem_vmm_vr 061 struct sos_umem_vmm_vr
062 { 062 {
063 063
064 struct sos_umem_vmm_as *address_space; 064 struct sos_umem_vmm_as *address_space;
065 065
066 066
067 sos_uaddr_t start; 067 sos_uaddr_t start;
068 sos_size_t size; 068 sos_size_t size;
069 069
070 070
071 071
072 sos_ui32_t access_rights; 072 sos_ui32_t access_rights;
073 073
074 074
075 075
076 076
077 sos_ui32_t flags; 077 sos_ui32_t flags;
078 078
079 079
080 080
081 081
082 082
083 struct sos_umem_vmm_vr_ops *ops; 083 struct sos_umem_vmm_vr_ops *ops;
084 084
085 085
086 struct sos_umem_vmm_mapped_resource *mapped_ 086 struct sos_umem_vmm_mapped_resource *mapped_resource;
087 sos_luoffset_t offset_in_resource; 087 sos_luoffset_t offset_in_resource;
088 088
089 089
090 090
091 struct sos_umem_vmm_vr *prev_in_as, *next_in 091 struct sos_umem_vmm_vr *prev_in_as, *next_in_as;
092 092
093 093
094 094
095 struct sos_umem_vmm_vr *prev_in_mapped_resou 095 struct sos_umem_vmm_vr *prev_in_mapped_resource, *next_in_mapped_resource;
096 }; 096 };
097 097
098 098
099 099
100 100
101 101
102 static struct sos_kslab_cache * cache_of_as; 102 static struct sos_kslab_cache * cache_of_as;
103 static struct sos_kslab_cache * cache_of_vr; 103 static struct sos_kslab_cache * cache_of_vr;
104 104
105 105
106 106
107 void sos_dump_as(const struct sos_umem_vmm_as 107 void sos_dump_as(const struct sos_umem_vmm_as * as, const char *str)
108 { 108 {
109 struct sos_umem_vmm_vr *vr; 109 struct sos_umem_vmm_vr *vr;
110 int nb_vr; 110 int nb_vr;
111 111
112 sos_bochs_printf("AS %p - %s:\n", as, str); 112 sos_bochs_printf("AS %p - %s:\n", as, str);
113 sos_bochs_printf(" physical mem: %x\n", 113 sos_bochs_printf(" physical mem: %x\n",
114 as->phys_total); 114 as->phys_total);
115 sos_bochs_printf(" VM (all/ro+rw/exec) tot 115 sos_bochs_printf(" VM (all/ro+rw/exec) tot:%x/%x+%x/%x shrd:%x/%x+%x/%x\n",
116 as->vm_total.overall, 116 as->vm_total.overall,
117 as->vm_total.ro, as->vm_tot 117 as->vm_total.ro, as->vm_total.rw, as->vm_total.code,
118 as->vm_shrd.overall, 118 as->vm_shrd.overall,
119 as->vm_shrd.ro, as->vm_shrd 119 as->vm_shrd.ro, as->vm_shrd.rw, as->vm_shrd.code);
120 sos_bochs_printf(" pgflt cow=%d pgin=%d in 120 sos_bochs_printf(" pgflt cow=%d pgin=%d inv=%d\n",
121 as->pgflt_cow, as->pgflt_pa 121 as->pgflt_cow, as->pgflt_page_in, as->pgflt_invalid);
122 list_foreach_named(as->list_vr, vr, nb_vr, p 122 list_foreach_named(as->list_vr, vr, nb_vr, prev_in_as, next_in_as)
123 { 123 {
124 sos_bochs_printf(" VR[%d]=%x: [%x,%x[ ( 124 sos_bochs_printf(" VR[%d]=%x: [%x,%x[ (sz=%x) mr=(%x)+%llx %c%c%c fl=%x\n",
125 nb_vr, (unsigned)vr, 125 nb_vr, (unsigned)vr,
126 vr->start, vr->start + 126 vr->start, vr->start + vr->size, vr->size,
127 (unsigned)vr->mapped_re 127 (unsigned)vr->mapped_resource,
128 vr->offset_in_resource, 128 vr->offset_in_resource,
129 (vr->access_rights & SO 129 (vr->access_rights & SOS_VM_MAP_PROT_READ)?'r':'-',
130 (vr->access_rights & SO 130 (vr->access_rights & SOS_VM_MAP_PROT_WRITE)?'w':'-',
131 (vr->access_rights & SO 131 (vr->access_rights & SOS_VM_MAP_PROT_EXEC)?'x':'-',
132 (unsigned)vr->flags); 132 (unsigned)vr->flags);
133 } 133 }
134 sos_bochs_printf("FIN (%s)\n", str); 134 sos_bochs_printf("FIN (%s)\n", str);
135 } 135 }
136 136
137 137
138 138
139 139
140 140
141 141
142 sos_paddr_t sos_zero_physpage = 0 !! 142 sos_paddr_t sos_zero_page = 0 ;
143 sos_vaddr_t sos_zero_kernelpage = 0 <<
144 143
145 144
146 145
147 146
148 147
149 148
150 149
151 150
152 151
153 static struct sos_umem_vmm_vr * 152 static struct sos_umem_vmm_vr *
154 find_enclosing_or_next_vr(struct sos_umem_vmm_ 153 find_enclosing_or_next_vr(struct sos_umem_vmm_as * as,
155 sos_uaddr_t uaddr); 154 sos_uaddr_t uaddr);
156 155
157 156
158 157
159 158
160 159
161 160
162 static struct sos_umem_vmm_vr * 161 static struct sos_umem_vmm_vr *
163 find_first_intersecting_vr(struct sos_umem_vmm 162 find_first_intersecting_vr(struct sos_umem_vmm_as * as,
164 sos_uaddr_t start_u 163 sos_uaddr_t start_uaddr, sos_size_t size);
165 164
166 165
167 166
168 167
169 168
170 169
171 170
172 171
173 172
174 static sos_uaddr_t 173 static sos_uaddr_t
175 find_first_free_interval(struct sos_umem_vmm_a 174 find_first_free_interval(struct sos_umem_vmm_as * as,
176 sos_uaddr_t hint_uadd 175 sos_uaddr_t hint_uaddr, sos_size_t size);
177 176
178 177
179 178
180 179
181 static void 180 static void
182 as_account_change_of_vr_protection(struct sos_ 181 as_account_change_of_vr_protection(struct sos_umem_vmm_as * as,
183 sos_bool_t 182 sos_bool_t is_shared,
184 sos_size_t 183 sos_size_t size,
185 sos_ui32_t 184 sos_ui32_t prev_access_rights,
186 sos_ui32_t 185 sos_ui32_t new_access_rights);
187 186
188 187
189 sos_ret_t sos_umem_vmm_subsystem_setup() 188 sos_ret_t sos_umem_vmm_subsystem_setup()
190 { 189 {
>> 190 sos_vaddr_t vaddr_zero_page;
>> 191
191 192
192 193
193 sos_zero_kernelpage = sos_kmem_vmm_alloc(1, !! 194 vaddr_zero_page = sos_kmem_vmm_alloc(1, SOS_KMEM_VMM_MAP);
194 if (sos_zero_kernelpage == (sos_vaddr_t)NULL !! 195 if (vaddr_zero_page == (sos_vaddr_t)NULL)
195 return -SOS_ENOMEM; 196 return -SOS_ENOMEM;
196 memset((void*)sos_zero_kernelpage, 0x0, SOS_ !! 197 memset((void*)vaddr_zero_page, 0x0, SOS_PAGE_SIZE);
197 198
198 199
199 sos_zero_physpage = sos_paging_get_paddr(sos !! 200 sos_zero_page = sos_paging_get_paddr(vaddr_zero_page);
200 SOS_ASSERT_FATAL(NULL != (void*)sos_zero_phy !! 201 SOS_ASSERT_FATAL(NULL != (void*)sos_zero_page);
201 sos_physmem_ref_physpage_at(sos_zero_physpag !! 202 sos_physmem_ref_physpage_at(sos_zero_page);
>> 203
>> 204
>> 205
>> 206 sos_paging_unmap(vaddr_zero_page);
202 207
203 208
204 cache_of_as 209 cache_of_as
205 = sos_kmem_cache_create("Address space str 210 = sos_kmem_cache_create("Address space structures",
206 sizeof(struct sos_ 211 sizeof(struct sos_umem_vmm_as),
207 1, 0, 212 1, 0,
208 SOS_KSLAB_CREATE_M 213 SOS_KSLAB_CREATE_MAP
209 | SOS_KSLAB_CREATE 214 | SOS_KSLAB_CREATE_ZERO);
210 if (! cache_of_as) 215 if (! cache_of_as)
211 { 216 {
212 sos_physmem_unref_physpage(sos_zero_phys !! 217 sos_physmem_unref_physpage(sos_zero_page);
213 return -SOS_ENOMEM; 218 return -SOS_ENOMEM;
214 } 219 }
215 220
216 cache_of_vr 221 cache_of_vr
217 = sos_kmem_cache_create("Virtual Region st 222 = sos_kmem_cache_create("Virtual Region structures",
218 sizeof(struct sos_ 223 sizeof(struct sos_umem_vmm_vr),
219 1, 0, 224 1, 0,
220 SOS_KSLAB_CREATE_M 225 SOS_KSLAB_CREATE_MAP
221 | SOS_KSLAB_CREATE 226 | SOS_KSLAB_CREATE_ZERO);
222 if (! cache_of_vr) 227 if (! cache_of_vr)
223 { 228 {
224 sos_physmem_unref_physpage(sos_zero_phys !! 229 sos_physmem_unref_physpage(sos_zero_page);
225 sos_kmem_cache_destroy(cache_of_as); 230 sos_kmem_cache_destroy(cache_of_as);
226 return -SOS_ENOMEM; 231 return -SOS_ENOMEM;
227 } 232 }
228 233
229 return SOS_OK; 234 return SOS_OK;
230 } 235 }
231 236
232 237
233 struct sos_umem_vmm_as * 238 struct sos_umem_vmm_as *
234 sos_umem_vmm_create_empty_as(struct sos_proces 239 sos_umem_vmm_create_empty_as(struct sos_process *owner)
235 { 240 {
236 struct sos_umem_vmm_as * as 241 struct sos_umem_vmm_as * as
237 = (struct sos_umem_vmm_as *) sos_kmem_cach 242 = (struct sos_umem_vmm_as *) sos_kmem_cache_alloc(cache_of_as, 0);
238 if (! as) 243 if (! as)
239 return NULL; 244 return NULL;
240 245
241 as->mm_context = sos_mm_context_create(); 246 as->mm_context = sos_mm_context_create();
242 if (NULL == as->mm_context) 247 if (NULL == as->mm_context)
243 { 248 {
244 249
245 sos_kmem_cache_free((sos_vaddr_t)as); 250 sos_kmem_cache_free((sos_vaddr_t)as);
246 return NULL; 251 return NULL;
247 } 252 }
248 253
249 as->process = owner; 254 as->process = owner;
250 return as; 255 return as;
251 } 256 }
252 257
253 258
254 struct sos_umem_vmm_as * 259 struct sos_umem_vmm_as *
255 sos_umem_vmm_duplicate_current_thread_as(struc 260 sos_umem_vmm_duplicate_current_thread_as(struct sos_process *owner)
256 { 261 {
257 __label__ undo_creation; 262 __label__ undo_creation;
258 struct sos_umem_vmm_as * my_as; 263 struct sos_umem_vmm_as * my_as;
259 struct sos_umem_vmm_vr * model_vr; 264 struct sos_umem_vmm_vr * model_vr;
260 int nb_vr; 265 int nb_vr;
261 266
262 struct sos_umem_vmm_as * new_as 267 struct sos_umem_vmm_as * new_as
263 = (struct sos_umem_vmm_as *) sos_kmem_cach 268 = (struct sos_umem_vmm_as *) sos_kmem_cache_alloc(cache_of_as, 0);
264 if (! new_as) 269 if (! new_as)
265 return NULL; 270 return NULL;
266 271
267 my_as = sos_process_get_address_space(sos_th 272 my_as = sos_process_get_address_space(sos_thread_get_current()->process);
268 new_as->process = owner; 273 new_as->process = owner;
269 list_init_named(new_as->list_vr, prev_in_as, 274 list_init_named(new_as->list_vr, prev_in_as, next_in_as);
270 275
271 276
272 277
273 278
274 279
275 280
276 SOS_ASSERT_FATAL(SOS_OK 281 SOS_ASSERT_FATAL(SOS_OK
277 == sos_thread_prepare_user_ 282 == sos_thread_prepare_user_space_access(my_as,
278 283 (sos_vaddr_t)
279 284 NULL));
280 285
281 286
282 list_foreach_named(my_as->list_vr, model_vr, 287 list_foreach_named(my_as->list_vr, model_vr, nb_vr, prev_in_as, next_in_as)
283 { 288 {
284 struct sos_umem_vmm_vr * vr; 289 struct sos_umem_vmm_vr * vr;
285 290
286 291
287 if ( !(model_vr->flags & SOS_VR_MAP_SHAR 292 if ( !(model_vr->flags & SOS_VR_MAP_SHARED)
288 && (model_vr->access_rights & SOS_V 293 && (model_vr->access_rights & SOS_VM_MAP_PROT_WRITE) )
289 { 294 {
290 295
291 296
292 SOS_ASSERT_FATAL(SOS_OK 297 SOS_ASSERT_FATAL(SOS_OK
293 == sos_paging_prepa 298 == sos_paging_prepare_COW(model_vr->start,
294 299 model_vr->size));
295 } 300 }
296 301
297 302
298 vr = (struct sos_umem_vmm_vr *) sos_kmem 303 vr = (struct sos_umem_vmm_vr *) sos_kmem_cache_alloc(cache_of_vr, 0);
299 if (! vr) 304 if (! vr)
300 goto undo_creation; 305 goto undo_creation;
301 memcpy(vr, model_vr, sizeof(*vr)); 306 memcpy(vr, model_vr, sizeof(*vr));
302 vr->address_space = new_as; 307 vr->address_space = new_as;
303 308
304 309
305 if (vr->ops && vr->ops->ref) 310 if (vr->ops && vr->ops->ref)
306 vr->ops->ref(vr); 311 vr->ops->ref(vr);
307 312
308 313
309 list_add_tail_named(new_as->list_vr, vr, 314 list_add_tail_named(new_as->list_vr, vr, prev_in_as, next_in_as);
310 315
311 316
312 list_add_tail_named(model_vr->mapped_res 317 list_add_tail_named(model_vr->mapped_resource->list_vr, vr,
313 prev_in_mapped_resou 318 prev_in_mapped_resource,
314 next_in_mapped_resou 319 next_in_mapped_resource);
315 } 320 }
316 321
317 322
318 new_as->mm_context = sos_mm_context_duplicat 323 new_as->mm_context = sos_mm_context_duplicate(my_as->mm_context);
319 if (NULL == new_as->mm_context) 324 if (NULL == new_as->mm_context)
320 goto undo_creation; 325 goto undo_creation;
321 326
322 327
323 new_as->heap_start = my_as->heap_start; 328 new_as->heap_start = my_as->heap_start;
324 new_as->heap_size = my_as->heap_size; 329 new_as->heap_size = my_as->heap_size;
325 new_as->phys_total = my_as->phys_total; 330 new_as->phys_total = my_as->phys_total;
326 memcpy(& new_as->vm_total, & my_as->vm_total 331 memcpy(& new_as->vm_total, & my_as->vm_total, sizeof(struct vm_usage));
327 memcpy(& new_as->vm_shrd, & my_as->vm_shrd, 332 memcpy(& new_as->vm_shrd, & my_as->vm_shrd, sizeof(struct vm_usage));
328 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_us 333 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_user_space_access());
329 return new_as; 334 return new_as;
330 335
331 336
332 undo_creation: 337 undo_creation:
333 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_us 338 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_user_space_access());
334 sos_umem_vmm_delete_as(new_as); 339 sos_umem_vmm_delete_as(new_as);
335 return NULL; 340 return NULL;
336 } 341 }
337 342
338 343
339 sos_ret_t 344 sos_ret_t
340 sos_umem_vmm_delete_as(struct sos_umem_vmm_as 345 sos_umem_vmm_delete_as(struct sos_umem_vmm_as * as)
341 { 346 {
342 while(! list_is_empty_named(as->list_vr, pre 347 while(! list_is_empty_named(as->list_vr, prev_in_as, next_in_as))
343 { 348 {
344 struct sos_umem_vmm_vr * vr; 349 struct sos_umem_vmm_vr * vr;
345 vr = list_get_head_named(as->list_vr, pr 350 vr = list_get_head_named(as->list_vr, prev_in_as, next_in_as);
346 351
347 352
348 list_pop_head_named(as->list_vr, prev_in 353 list_pop_head_named(as->list_vr, prev_in_as, next_in_as);
349 list_delete_named(vr->mapped_resource->l 354 list_delete_named(vr->mapped_resource->list_vr, vr,
350 prev_in_mapped_resourc 355 prev_in_mapped_resource,
351 next_in_mapped_resourc 356 next_in_mapped_resource);
352 357
353 358
354 359
355 if (vr->ops) 360 if (vr->ops)
356 { 361 {
357 if (vr->ops->unmap) 362 if (vr->ops->unmap)
358 vr->ops->unmap(vr, vr->start, vr-> 363 vr->ops->unmap(vr, vr->start, vr->size);
359 if (vr->ops->unref) 364 if (vr->ops->unref)
360 vr->ops->unref(vr); 365 vr->ops->unref(vr);
361 } 366 }
362 367
363 sos_kmem_cache_free((sos_vaddr_t)vr); 368 sos_kmem_cache_free((sos_vaddr_t)vr);
364 } 369 }
365 370
366 371
367 if (as->mm_context) 372 if (as->mm_context)
368 sos_mm_context_unref(as->mm_context); 373 sos_mm_context_unref(as->mm_context);
369 374
370 375
371 sos_kmem_cache_free((sos_vaddr_t)as); 376 sos_kmem_cache_free((sos_vaddr_t)as);
372 377
373 return SOS_OK; 378 return SOS_OK;
374 } 379 }
375 380
376 381
377 struct sos_process * 382 struct sos_process *
378 sos_umem_vmm_get_process(struct sos_umem_vmm_a 383 sos_umem_vmm_get_process(struct sos_umem_vmm_as * as)
379 { 384 {
380 return as->process; 385 return as->process;
381 } 386 }
382 387
383 388
384 struct sos_mm_context * 389 struct sos_mm_context *
385 sos_umem_vmm_get_mm_context(struct sos_umem_vm 390 sos_umem_vmm_get_mm_context(struct sos_umem_vmm_as * as)
386 { 391 {
387 return as->mm_context; 392 return as->mm_context;
388 } 393 }
389 394
390 395
391 struct sos_umem_vmm_vr * 396 struct sos_umem_vmm_vr *
392 sos_umem_vmm_get_vr_at_address(struct sos_umem 397 sos_umem_vmm_get_vr_at_address(struct sos_umem_vmm_as * as,
393 sos_uaddr_t uad 398 sos_uaddr_t uaddr)
394 { 399 {
395 struct sos_umem_vmm_vr * vr; 400 struct sos_umem_vmm_vr * vr;
396 vr = find_enclosing_or_next_vr(as, uaddr); 401 vr = find_enclosing_or_next_vr(as, uaddr);
397 if (! vr) 402 if (! vr)
398 return NULL; 403 return NULL;
399 404
400 405
401 if (uaddr < vr->start) 406 if (uaddr < vr->start)
402 return NULL; 407 return NULL;
403 408
404 return vr; 409 return vr;
405 } 410 }
406 411
407 412
408 struct sos_umem_vmm_as * 413 struct sos_umem_vmm_as *
409 sos_umem_vmm_get_as_of_vr(struct sos_umem_vmm_ 414 sos_umem_vmm_get_as_of_vr(struct sos_umem_vmm_vr * vr)
410 { 415 {
411 return vr->address_space; 416 return vr->address_space;
412 } 417 }
413 418
414 419
415 struct sos_umem_vmm_vr_ops * 420 struct sos_umem_vmm_vr_ops *
416 sos_umem_vmm_get_ops_of_vr(struct sos_umem_vmm 421 sos_umem_vmm_get_ops_of_vr(struct sos_umem_vmm_vr * vr)
417 { 422 {
418 return vr->ops; 423 return vr->ops;
419 } 424 }
420 425
421 426
422 sos_ui32_t sos_umem_vmm_get_prot_of_vr(struct 427 sos_ui32_t sos_umem_vmm_get_prot_of_vr(struct sos_umem_vmm_vr * vr)
423 { 428 {
424 return vr->access_rights; 429 return vr->access_rights;
425 } 430 }
426 431
427 432
428 sos_ui32_t sos_umem_vmm_get_flags_of_vr(struct 433 sos_ui32_t sos_umem_vmm_get_flags_of_vr(struct sos_umem_vmm_vr * vr)
429 { 434 {
430 return vr->flags; 435 return vr->flags;
431 } 436 }
432 437
433 438
434 struct sos_umem_vmm_mapped_resource * 439 struct sos_umem_vmm_mapped_resource *
435 sos_umem_vmm_get_mapped_resource_of_vr(struct 440 sos_umem_vmm_get_mapped_resource_of_vr(struct sos_umem_vmm_vr * vr)
436 { 441 {
437 return vr->mapped_resource; 442 return vr->mapped_resource;
438 } 443 }
439 444
440 445
441 sos_uaddr_t sos_umem_vmm_get_start_of_vr(struc 446 sos_uaddr_t sos_umem_vmm_get_start_of_vr(struct sos_umem_vmm_vr * vr)
442 { 447 {
443 return vr->start; 448 return vr->start;
444 } 449 }
445 450
446 451
447 sos_size_t sos_umem_vmm_get_size_of_vr(struct 452 sos_size_t sos_umem_vmm_get_size_of_vr(struct sos_umem_vmm_vr * vr)
448 { 453 {
449 return vr->size; 454 return vr->size;
450 } 455 }
451 456
452 457
453 sos_luoffset_t sos_umem_vmm_get_offset_in_reso 458 sos_luoffset_t sos_umem_vmm_get_offset_in_resource(struct sos_umem_vmm_vr * vr)
454 { 459 {
455 return vr->offset_in_resource; 460 return vr->offset_in_resource;
456 } 461 }
457 462
458 463
459 sos_ret_t 464 sos_ret_t
460 sos_umem_vmm_set_ops_of_vr(struct sos_umem_vmm 465 sos_umem_vmm_set_ops_of_vr(struct sos_umem_vmm_vr * vr,
461 struct sos_umem_vmm 466 struct sos_umem_vmm_vr_ops * ops)
462 { 467 {
463 468
464 SOS_ASSERT_FATAL(NULL == vr->ops); 469 SOS_ASSERT_FATAL(NULL == vr->ops);
465 470
466 vr->ops = ops; 471 vr->ops = ops;
467 return SOS_OK; 472 return SOS_OK;
468 } 473 }
469 474
470 475
471 476
472 477
473 478
474 479
475 #define INTERNAL_MAP_CALLED_FROM_MREMAP (1 << 480 #define INTERNAL_MAP_CALLED_FROM_MREMAP (1 << 8)
476 481
477 sos_ret_t 482 sos_ret_t
478 sos_umem_vmm_map(struct sos_umem_vmm_as * as, 483 sos_umem_vmm_map(struct sos_umem_vmm_as * as,
479 sos_uaddr_t * uaddr 484 sos_uaddr_t * uaddr, sos_size_t size,
480 sos_ui32_t access_rights, 485 sos_ui32_t access_rights,
481 sos_ui32_t flags, 486 sos_ui32_t flags,
482 struct sos_umem_vmm_mapped_re 487 struct sos_umem_vmm_mapped_resource * resource,
483 sos_luoffset_t offset_in_reso 488 sos_luoffset_t offset_in_resource)
484 { 489 {
485 __label__ return_mmap; 490 __label__ return_mmap;
486 sos_uaddr_t hint_uaddr; 491 sos_uaddr_t hint_uaddr;
487 struct sos_umem_vmm_vr *prev_vr, *next_vr, * 492 struct sos_umem_vmm_vr *prev_vr, *next_vr, *vr, *preallocated_vr;
488 sos_bool_t merge_with_preceding, merge_with_ 493 sos_bool_t merge_with_preceding, merge_with_next, used_preallocated_vr;
489 sos_bool_t internal_map_called_from_mremap 494 sos_bool_t internal_map_called_from_mremap
490 = (flags & INTERNAL_MAP_CALLED_FROM_MREMAP 495 = (flags & INTERNAL_MAP_CALLED_FROM_MREMAP);
491 496
492 sos_ret_t retval = SOS_OK; 497 sos_ret_t retval = SOS_OK;
493 used_preallocated_vr = FALSE; 498 used_preallocated_vr = FALSE;
494 hint_uaddr = *uaddr; 499 hint_uaddr = *uaddr;
495 500
496 501
497 *uaddr = (sos_vaddr_t)NULL; 502 *uaddr = (sos_vaddr_t)NULL;
498 503
499 if (! resource) 504 if (! resource)
500 return -SOS_EINVAL; 505 return -SOS_EINVAL;
501 if (! resource->mmap) 506 if (! resource->mmap)
502 return -SOS_EPERM; 507 return -SOS_EPERM;
503 508
504 if (! SOS_IS_PAGE_ALIGNED(hint_uaddr)) 509 if (! SOS_IS_PAGE_ALIGNED(hint_uaddr))
505 return -SOS_EINVAL; 510 return -SOS_EINVAL;
506 511
507 if (size <= 0) 512 if (size <= 0)
508 return -SOS_EINVAL; 513 return -SOS_EINVAL;
509 size = SOS_PAGE_ALIGN_SUP(size); 514 size = SOS_PAGE_ALIGN_SUP(size);
510 515
511 if (flags & SOS_VR_MAP_SHARED) 516 if (flags & SOS_VR_MAP_SHARED)
512 { 517 {
513 518
514 if ( ( (access_rights & SOS_VM_MAP_PROT_ 519 if ( ( (access_rights & SOS_VM_MAP_PROT_READ)
515 && !(resource->allowed_access_rig 520 && !(resource->allowed_access_rights & SOS_VM_MAP_PROT_READ) )
516 || ( (access_rights & SOS_VM_MAP_PR 521 || ( (access_rights & SOS_VM_MAP_PROT_WRITE)
517 && !(resource->allowed_access_ 522 && !(resource->allowed_access_rights & SOS_VM_MAP_PROT_WRITE) )
518 || ( (access_rights & SOS_VM_MAP_PR 523 || ( (access_rights & SOS_VM_MAP_PROT_EXEC)
519 && !(resource->allowed_access_ 524 && !(resource->allowed_access_rights & SOS_VM_MAP_PROT_EXEC)) )
520 return -SOS_EPERM; 525 return -SOS_EPERM;
521 } 526 }
522 527
523 528
524 if ( !internal_map_called_from_mremap 529 if ( !internal_map_called_from_mremap
525 && ( resource->flags & SOS_MAPPED_RESOU 530 && ( resource->flags & SOS_MAPPED_RESOURCE_ANONYMOUS ) )
526 531
527 { 532 {
528 533
529 } 534 }
530 535
531 536
532 else if (offset_in_resource + size <= offset 537 else if (offset_in_resource + size <= offset_in_resource)
533 return -SOS_EINVAL; 538 return -SOS_EINVAL;
534 539
535 540
536 access_rights &= (SOS_VM_MAP_PROT_READ 541 access_rights &= (SOS_VM_MAP_PROT_READ
537 | SOS_VM_MAP_PROT_WRITE 542 | SOS_VM_MAP_PROT_WRITE
538 | SOS_VM_MAP_PROT_EXEC); 543 | SOS_VM_MAP_PROT_EXEC);
539 flags &= (SOS_VR_MAP_SHARED 544 flags &= (SOS_VR_MAP_SHARED
540 | SOS_VR_MAP_FIXED); 545 | SOS_VR_MAP_FIXED);
541 546
542 547
543 548
544 549
545 preallocated_vr 550 preallocated_vr
546 = (struct sos_umem_vmm_vr *)sos_kmem_cache 551 = (struct sos_umem_vmm_vr *)sos_kmem_cache_alloc(cache_of_vr, 0);
547 if (! preallocated_vr) 552 if (! preallocated_vr)
548 return -SOS_ENOMEM; 553 return -SOS_ENOMEM;
549 554
550 555
551 if (flags & SOS_VR_MAP_FIXED) 556 if (flags & SOS_VR_MAP_FIXED)
552 { 557 {
553 558
554 559
555 560
556 561
557 562
558 if (hint_uaddr < SOS_PAGING_BASE_USER_AD 563 if (hint_uaddr < SOS_PAGING_BASE_USER_ADDRESS)
559 { retval = -SOS_EINVAL; goto return_mm 564 { retval = -SOS_EINVAL; goto return_mmap; }
560 if (hint_uaddr > SOS_PAGING_TOP_USER_ADD 565 if (hint_uaddr > SOS_PAGING_TOP_USER_ADDRESS - size)
561 { retval = -SOS_EINVAL; goto return_mm 566 { retval = -SOS_EINVAL; goto return_mmap; }
562 567
563 568
564 retval = sos_umem_vmm_unmap(as, hint_uad 569 retval = sos_umem_vmm_unmap(as, hint_uaddr, size);
565 if (SOS_OK != retval) 570 if (SOS_OK != retval)
566 { goto return_mmap; } 571 { goto return_mmap; }
567 } 572 }
568 else 573 else
569 { 574 {
570 575
571 576
572 577
573 578
574 579
575 hint_uaddr = find_first_free_interval(as 580 hint_uaddr = find_first_free_interval(as, hint_uaddr, size);
576 if (! hint_uaddr) 581 if (! hint_uaddr)
577 { retval = -SOS_ENOMEM; goto return_mm 582 { retval = -SOS_ENOMEM; goto return_mmap; }
578 } 583 }
579 584
580 585
581 586
582 587
583 if ( !internal_map_called_from_mremap 588 if ( !internal_map_called_from_mremap
584 && (resource->flags & SOS_MAPPED_RESOUR 589 && (resource->flags & SOS_MAPPED_RESOURCE_ANONYMOUS ) )
585 offset_in_resource = hint_uaddr; 590 offset_in_resource = hint_uaddr;
586 591
587 592
588 593
589 next_vr = find_enclosing_or_next_vr(as, hint 594 next_vr = find_enclosing_or_next_vr(as, hint_uaddr);
590 if (next_vr) 595 if (next_vr)
591 { 596 {
592 597
593 prev_vr = next_vr->prev_in_as; 598 prev_vr = next_vr->prev_in_as;
594 599
595 600
596 if (prev_vr->start > hint_uaddr) 601 if (prev_vr->start > hint_uaddr)
597 prev_vr = NULL; 602 prev_vr = NULL;
598 } 603 }
599 else 604 else
600 { 605 {
601 606
602 prev_vr = list_get_tail_named(as->list_v 607 prev_vr = list_get_tail_named(as->list_vr, prev_in_as, next_in_as);
603 } 608 }
604 609
605 610
606 merge_with_preceding 611 merge_with_preceding
607 = ( (NULL != prev_vr) 612 = ( (NULL != prev_vr)
608 && (prev_vr->mapped_resource == resour 613 && (prev_vr->mapped_resource == resource)
609 && (prev_vr->offset_in_resource + prev 614 && (prev_vr->offset_in_resource + prev_vr->size == offset_in_resource)
610 && (prev_vr->start + prev_vr->size == 615 && (prev_vr->start + prev_vr->size == hint_uaddr)
611 && (prev_vr->flags == flags) 616 && (prev_vr->flags == flags)
612 && (prev_vr->access_rights == access_r 617 && (prev_vr->access_rights == access_rights) );
613 618
614 619
615 merge_with_next 620 merge_with_next
616 = ( (NULL != next_vr) 621 = ( (NULL != next_vr)
617 && (next_vr->mapped_resource == resour 622 && (next_vr->mapped_resource == resource)
618 && (offset_in_resource + size == next_ 623 && (offset_in_resource + size == next_vr->offset_in_resource)
619 && (hint_uaddr + size == next_vr->star 624 && (hint_uaddr + size == next_vr->start)
620 && (next_vr->flags == flags) 625 && (next_vr->flags == flags)
621 && (next_vr->access_rights == access_r 626 && (next_vr->access_rights == access_rights) );
622 627
623 if (merge_with_preceding && merge_with_next) 628 if (merge_with_preceding && merge_with_next)
624 { 629 {
625 630
626 vr = prev_vr; 631 vr = prev_vr;
627 vr->size += size + next_vr->size; 632 vr->size += size + next_vr->size;
628 633
629 634
630 list_delete_named(as->list_vr, next_vr, 635 list_delete_named(as->list_vr, next_vr, prev_in_as, next_in_as);
631 list_delete_named(next_vr->mapped_resour 636 list_delete_named(next_vr->mapped_resource->list_vr, next_vr,
632 prev_in_mapped_resourc 637 prev_in_mapped_resource, next_in_mapped_resource);
633 638
634 if (next_vr->ops && next_vr->ops->unref) 639 if (next_vr->ops && next_vr->ops->unref)
635 next_vr->ops->unref(next_vr); 640 next_vr->ops->unref(next_vr);
636 641
637 sos_kmem_vmm_free((sos_vaddr_t) next_vr) 642 sos_kmem_vmm_free((sos_vaddr_t) next_vr);
638 } 643 }
639 else if (merge_with_preceding) 644 else if (merge_with_preceding)
640 { 645 {
641 646
642 vr = prev_vr; 647 vr = prev_vr;
643 vr->size += size; 648 vr->size += size;
644 } 649 }
645 else if (merge_with_next) 650 else if (merge_with_next)
646 { 651 {
647 652
648 vr = next_vr; 653 vr = next_vr;
649 vr->start -= size; 654 vr->start -= size;
650 vr->size += size; 655 vr->size += size;
651 } 656 }
652 else 657 else
653 { 658 {
654 659
655 660
656 vr = preallocated_vr; 661 vr = preallocated_vr;
657 used_preallocated_vr = TRUE; 662 used_preallocated_vr = TRUE;
658 663
659 vr->start = hint_uaddr; 664 vr->start = hint_uaddr;
660 vr->size = size; 665 vr->size = size;
661 vr->access_rights = access_rights; 666 vr->access_rights = access_rights;
662 vr->flags = flags; 667 vr->flags = flags;
663 vr->mapped_resource = resource; 668 vr->mapped_resource = resource;
664 vr->offset_in_resource = offset_in_resou 669 vr->offset_in_resource = offset_in_resource;
665 670
666 671
667 vr->address_space = as; 672 vr->address_space = as;
668 if (prev_vr) 673 if (prev_vr)
669 list_insert_after_named(as->list_vr, p 674 list_insert_after_named(as->list_vr, prev_vr, vr,
670 prev_in_as, ne 675 prev_in_as, next_in_as);
671 else 676 else
672 list_add_head_named(as->list_vr, vr, p 677 list_add_head_named(as->list_vr, vr, prev_in_as, next_in_as);
673 list_add_tail_named(vr->mapped_resource- 678 list_add_tail_named(vr->mapped_resource->list_vr, vr,
674 prev_in_mapped_resou 679 prev_in_mapped_resource,
675 next_in_mapped_resou 680 next_in_mapped_resource);
676 681
677 682
678 if (resource && resource->mmap) 683 if (resource && resource->mmap)
679 { 684 {
680 retval = resource->mmap(vr); 685 retval = resource->mmap(vr);
681 if (SOS_OK != retval) 686 if (SOS_OK != retval)
682 { 687 {
683 retval = sos_umem_vmm_unmap(as, 688 retval = sos_umem_vmm_unmap(as, vr->start, vr->size);
684 goto return_mmap; 689 goto return_mmap;
685 } 690 }
686 691
687 692
688 SOS_ASSERT_FATAL(vr->ops && vr->ops- 693 SOS_ASSERT_FATAL(vr->ops && vr->ops->page_in);
689 } 694 }
690 695
691 if (vr->ops && vr->ops->ref) 696 if (vr->ops && vr->ops->ref)
692 vr->ops->ref(vr); 697 vr->ops->ref(vr);
693 } 698 }
694 699
695 700
696 *uaddr = hint_uaddr; 701 *uaddr = hint_uaddr;
697 as_account_change_of_vr_protection(as, vr->f 702 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
698 size, 0, 703 size, 0, vr->access_rights);
699 retval = SOS_OK; 704 retval = SOS_OK;
700 705
701 return_mmap: 706 return_mmap:
702 if (! used_preallocated_vr) 707 if (! used_preallocated_vr)
703 sos_kmem_vmm_free((sos_vaddr_t)preallocate 708 sos_kmem_vmm_free((sos_vaddr_t)preallocated_vr);
704 709
705 return retval; 710 return retval;
706 } 711 }
707 712
708 713
709 sos_ret_t 714 sos_ret_t
710 sos_umem_vmm_unmap(struct sos_umem_vmm_as * as 715 sos_umem_vmm_unmap(struct sos_umem_vmm_as * as,
711 sos_uaddr_t uaddr, sos_size 716 sos_uaddr_t uaddr, sos_size_t size)
712 { 717 {
713 struct sos_umem_vmm_vr *vr, *preallocated_vr 718 struct sos_umem_vmm_vr *vr, *preallocated_vr;
714 sos_bool_t need_to_setup_mmu; 719 sos_bool_t need_to_setup_mmu;
715 sos_bool_t used_preallocated_vr; 720 sos_bool_t used_preallocated_vr;
716 721
717 if (! SOS_IS_PAGE_ALIGNED(uaddr)) 722 if (! SOS_IS_PAGE_ALIGNED(uaddr))
718 return -SOS_EINVAL; 723 return -SOS_EINVAL;
719 if (size <= 0) 724 if (size <= 0)
720 return -SOS_EINVAL; 725 return -SOS_EINVAL;
721 size = SOS_PAGE_ALIGN_SUP(size); 726 size = SOS_PAGE_ALIGN_SUP(size);
722 727
723 728
724 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS) 729 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS)
725 return -SOS_EINVAL; 730 return -SOS_EINVAL;
726 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS - si 731 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS - size)
727 return -SOS_EINVAL; 732 return -SOS_EINVAL;
728 733
729 734
730 735
731 736
732 737
733 738
734 used_preallocated_vr = FALSE; 739 used_preallocated_vr = FALSE;
735 preallocated_vr 740 preallocated_vr
736 = (struct sos_umem_vmm_vr *)sos_kmem_cache 741 = (struct sos_umem_vmm_vr *)sos_kmem_cache_alloc(cache_of_vr, 0);
737 if (! preallocated_vr) 742 if (! preallocated_vr)
738 return -SOS_ENOMEM; 743 return -SOS_ENOMEM;
739 744
740 745
741 vr = find_first_intersecting_vr(as, uaddr, s 746 vr = find_first_intersecting_vr(as, uaddr, size);
742 747
743 748
744 while (NULL != vr) 749 while (NULL != vr)
745 { 750 {
746 751
747 752
748 if (vr->start + vr->size <= uaddr) 753 if (vr->start + vr->size <= uaddr)
749 754
750 break; 755 break;
751 756
752 757
753 if (uaddr + size <= vr->start) 758 if (uaddr + size <= vr->start)
754 759
755 break; 760 break;
756 761
757 762
758 if ((vr->start >= uaddr) 763 if ((vr->start >= uaddr)
759 && (vr->start + vr->size <= uaddr + 764 && (vr->start + vr->size <= uaddr + size))
760 { 765 {
761 struct sos_umem_vmm_vr *next_vr; 766 struct sos_umem_vmm_vr *next_vr;
762 767
763 768
764 if (vr->ops && vr->ops->unmap) 769 if (vr->ops && vr->ops->unmap)
765 vr->ops->unmap(vr, vr->start, vr-> 770 vr->ops->unmap(vr, vr->start, vr->size);
766 771
767 772
768 next_vr = vr->next_in_as; 773 next_vr = vr->next_in_as;
769 if (next_vr == vr) 774 if (next_vr == vr)
770 next_vr = NULL; 775 next_vr = NULL;
771 list_delete_named(as->list_vr, vr, p 776 list_delete_named(as->list_vr, vr, prev_in_as, next_in_as);
772 777
773 778
774 list_delete_named(vr->mapped_resourc 779 list_delete_named(vr->mapped_resource->list_vr, vr,
775 prev_in_mapped_res 780 prev_in_mapped_resource,
776 next_in_mapped_res 781 next_in_mapped_resource);
777 782
778 if (vr->ops && vr->ops->unref) 783 if (vr->ops && vr->ops->unref)
779 vr->ops->unref(vr); 784 vr->ops->unref(vr);
780 785
781 as_account_change_of_vr_protection(a 786 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
782 v 787 vr->size, vr->access_rights, 0);
783 sos_kmem_vmm_free((sos_vaddr_t)vr); 788 sos_kmem_vmm_free((sos_vaddr_t)vr);
784 789
785 790
786 vr = next_vr; 791 vr = next_vr;
787 continue; 792 continue;
788 } 793 }
789 794
790 795
791 else if ( (vr->start < uaddr) 796 else if ( (vr->start < uaddr)
792 && (vr->start + vr->size > uad 797 && (vr->start + vr->size > uaddr + size) )
793 { 798 {
794 799
795 800
796 801
797 used_preallocated_vr = TRUE; 802 used_preallocated_vr = TRUE;
798 memcpy(preallocated_vr, vr, sizeof(* 803 memcpy(preallocated_vr, vr, sizeof(*vr));
799 804
800 805
801 preallocated_vr->start = uaddr + siz 806 preallocated_vr->start = uaddr + size;
802 preallocated_vr->size = vr->start + 807 preallocated_vr->size = vr->start + vr->size - (uaddr + size);
803 preallocated_vr->offset_in_resource 808 preallocated_vr->offset_in_resource += uaddr + size - vr->start;
804 vr->size 809 vr->size = uaddr - vr->start;
805 810
806 811
807 list_insert_after_named(as->list_vr, 812 list_insert_after_named(as->list_vr, vr, preallocated_vr,
808 prev_in_as, 813 prev_in_as, next_in_as);
809 list_add_tail_named(vr->mapped_resou 814 list_add_tail_named(vr->mapped_resource->list_vr, preallocated_vr,
810 prev_in_mapped_r 815 prev_in_mapped_resource,
811 next_in_mapped_r 816 next_in_mapped_resource);
812 817
813 818
814 if (vr->ops && vr->ops->unmap) 819 if (vr->ops && vr->ops->unmap)
815 vr->ops->unmap(vr, uaddr, size); 820 vr->ops->unmap(vr, uaddr, size);
816 if (preallocated_vr->ops && prealloc 821 if (preallocated_vr->ops && preallocated_vr->ops->ref)
817 preallocated_vr->ops->ref(prealloc 822 preallocated_vr->ops->ref(preallocated_vr);
818 823
819 824
820 as_account_change_of_vr_protection(a 825 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
821 s 826 size, vr->access_rights, 0);
822 827
823 828
824 break; 829 break;
825 } 830 }
826 831
827 832
828 else if (uaddr <= vr->start) 833 else if (uaddr <= vr->start)
829 { 834 {
830 sos_size_t translation = uaddr + siz 835 sos_size_t translation = uaddr + size - vr->start;
831 836
832 837
833 vr->size -= translatio 838 vr->size -= translation;
834 vr->offset_in_resource += translatio 839 vr->offset_in_resource += translation;
835 vr->start += translatio 840 vr->start += translation;
836 841
837 842
838 if (vr->ops && vr->ops->unmap) 843 if (vr->ops && vr->ops->unmap)
839 vr->ops->unmap(vr, uaddr + size, 844 vr->ops->unmap(vr, uaddr + size,
840 translation); 845 translation);
841 846
842 847
843 as_account_change_of_vr_protection(a 848 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
844 t 849 translation,
845 v 850 vr->access_rights, 0);
846 851
847 852
848 853
849 break; 854 break;
850 } 855 }
851 856
852 857
853 else if (uaddr + size >= vr->start + vr- 858 else if (uaddr + size >= vr->start + vr->size)
854 { 859 {
855 sos_size_t unmapped_size = vr->start 860 sos_size_t unmapped_size = vr->start + vr->size - uaddr;
856 861
857 862
858 vr->size = uaddr - vr->start; 863 vr->size = uaddr - vr->start;
859 864
860 865
861 if (vr->ops && vr->ops->unmap) 866 if (vr->ops && vr->ops->unmap)
862 vr->ops->unmap(vr, uaddr, unmapped 867 vr->ops->unmap(vr, uaddr, unmapped_size);
863 868
864 869
865 as_account_change_of_vr_protection(a 870 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
866 u 871 unmapped_size,
867 v 872 vr->access_rights, 0);
868 873
869 vr = vr->next_in_as; 874 vr = vr->next_in_as;
870 continue; 875 continue;
871 } 876 }
872 877
873 sos_display_fatal_error("BUG uaddr=%x sz 878 sos_display_fatal_error("BUG uaddr=%x sz=%x vr_start=%x, vr_sz=%x",
874 uaddr, size, vr- 879 uaddr, size, vr->start, vr->size);
875 } 880 }
876 881
877 need_to_setup_mmu = (sos_thread_get_current( 882 need_to_setup_mmu = (sos_thread_get_current()->squatted_mm_context
878 != as->mm_context); 883 != as->mm_context);
879 if (need_to_setup_mmu) 884 if (need_to_setup_mmu)
880 SOS_ASSERT_FATAL(SOS_OK 885 SOS_ASSERT_FATAL(SOS_OK
881 == sos_thread_prepare_use 886 == sos_thread_prepare_user_space_access(as,
882 887 (sos_vaddr_t)
883 888 NULL));
884 { 889 {
885 sos_size_t sz_unmapped = sos_paging_unmap_ 890 sos_size_t sz_unmapped = sos_paging_unmap_interval(uaddr, size);
886 SOS_ASSERT_FATAL(sz_unmapped >= 0); 891 SOS_ASSERT_FATAL(sz_unmapped >= 0);
887 as->phys_total -= sz_unmapped; 892 as->phys_total -= sz_unmapped;
888 } 893 }
889 if (need_to_setup_mmu) 894 if (need_to_setup_mmu)
890 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_ 895 SOS_ASSERT_FATAL(SOS_OK == sos_thread_end_user_space_access());
891 896
892 if (! used_preallocated_vr) 897 if (! used_preallocated_vr)
893 sos_kmem_vmm_free((sos_vaddr_t)preallocate 898 sos_kmem_vmm_free((sos_vaddr_t)preallocated_vr);
894 899
895 return SOS_OK; 900 return SOS_OK;
896 } 901 }
897 902
898 903
899 sos_ret_t 904 sos_ret_t
900 sos_umem_vmm_chprot(struct sos_umem_vmm_as * a 905 sos_umem_vmm_chprot(struct sos_umem_vmm_as * as,
901 sos_uaddr_t uaddr, sos_siz 906 sos_uaddr_t uaddr, sos_size_t size,
902 sos_ui32_t new_access_righ 907 sos_ui32_t new_access_rights)
903 { 908 {
904 struct sos_umem_vmm_vr *start_vr, *vr, 909 struct sos_umem_vmm_vr *start_vr, *vr,
905 *preallocated_middle_vr, *preallocated_rig 910 *preallocated_middle_vr, *preallocated_right_vr;
906 sos_bool_t used_preallocated_middle_vr, used 911 sos_bool_t used_preallocated_middle_vr, used_preallocated_right_vr;
907 912
908 if (! SOS_IS_PAGE_ALIGNED(uaddr)) 913 if (! SOS_IS_PAGE_ALIGNED(uaddr))
909 return -SOS_EINVAL; 914 return -SOS_EINVAL;
910 if (size <= 0) 915 if (size <= 0)
911 return -SOS_EINVAL; 916 return -SOS_EINVAL;
912 size = SOS_PAGE_ALIGN_SUP(size); 917 size = SOS_PAGE_ALIGN_SUP(size);
913 918
914 919
915 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS) 920 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS)
916 return -SOS_EINVAL; 921 return -SOS_EINVAL;
917 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS - si 922 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS - size)
918 return -SOS_EINVAL; 923 return -SOS_EINVAL;
919 924
920 925
921 926
922 used_preallocated_middle_vr = FALSE; 927 used_preallocated_middle_vr = FALSE;
923 used_preallocated_right_vr = FALSE; 928 used_preallocated_right_vr = FALSE;
924 preallocated_middle_vr 929 preallocated_middle_vr
925 = (struct sos_umem_vmm_vr *)sos_kmem_cache 930 = (struct sos_umem_vmm_vr *)sos_kmem_cache_alloc(cache_of_vr, 0);
926 if (! preallocated_middle_vr) 931 if (! preallocated_middle_vr)
927 return -SOS_ENOMEM; 932 return -SOS_ENOMEM;
928 preallocated_right_vr 933 preallocated_right_vr
929 = (struct sos_umem_vmm_vr *)sos_kmem_cache 934 = (struct sos_umem_vmm_vr *)sos_kmem_cache_alloc(cache_of_vr, 0);
930 if (! preallocated_right_vr) 935 if (! preallocated_right_vr)
931 { 936 {
932 sos_kmem_vmm_free((sos_vaddr_t)prealloca 937 sos_kmem_vmm_free((sos_vaddr_t)preallocated_middle_vr);
933 return -SOS_ENOMEM; 938 return -SOS_ENOMEM;
934 } 939 }
935 940
936 941
937 start_vr = find_first_intersecting_vr(as, ua 942 start_vr = find_first_intersecting_vr(as, uaddr, size);
938 if (NULL == start_vr) 943 if (NULL == start_vr)
939 return SOS_OK; 944 return SOS_OK;
940 945
941 946
942 947
943 vr = start_vr; 948 vr = start_vr;
944 while (TRUE) 949 while (TRUE)
945 { 950 {
946 951
947 952
948 if (vr->start + vr->size <= uaddr) 953 if (vr->start + vr->size <= uaddr)
949 954
950 break; 955 break;
951 956
952 957
953 if (uaddr + size < vr->start) 958 if (uaddr + size < vr->start)
954 959
955 break; 960 break;
956 961
957 if (vr->flags & SOS_VR_MAP_SHARED) 962 if (vr->flags & SOS_VR_MAP_SHARED)
958 { 963 {
959 964
960 965
961 if ( ( (new_access_rights & SOS_VM_M 966 if ( ( (new_access_rights & SOS_VM_MAP_PROT_READ)
962 && !(vr->mapped_resource->all 967 && !(vr->mapped_resource->allowed_access_rights
963 & SOS_VM_MAP_PROT_READ) 968 & SOS_VM_MAP_PROT_READ) )
964 || ( (new_access_rights & SOS_V 969 || ( (new_access_rights & SOS_VM_MAP_PROT_WRITE)
965 && !(vr->mapped_resource-> 970 && !(vr->mapped_resource->allowed_access_rights
966 & SOS_VM_MAP_PROT_WRI 971 & SOS_VM_MAP_PROT_WRITE) )
967 || ( (new_access_rights & SOS_V 972 || ( (new_access_rights & SOS_VM_MAP_PROT_EXEC)
968 && !(vr->mapped_resource-> 973 && !(vr->mapped_resource->allowed_access_rights
969 & SOS_VM_MAP_PROT_EXE 974 & SOS_VM_MAP_PROT_EXEC) ) )
970 return -SOS_EPERM; 975 return -SOS_EPERM;
971 } 976 }
972 977
973 vr = vr->next_in_as; 978 vr = vr->next_in_as;
974 } 979 }
975 980
976 981
977 982
978 vr = start_vr; 983 vr = start_vr;
979 while (TRUE) 984 while (TRUE)
980 { 985 {
981 986
982 987
983 988
984 if (vr->start + vr->size <= uaddr) 989 if (vr->start + vr->size <= uaddr)
985 990
986 break; 991 break;
987 992
988 993
989 if (uaddr + size <= vr->start) 994 if (uaddr + size <= vr->start)
990 995
991 break; 996 break;
992 997
993 998
994 if (vr->access_rights == new_access_righ 999 if (vr->access_rights == new_access_rights)
995 1000
996 { 1001 {
997 vr = vr->next_in_as; 1002 vr = vr->next_in_as;
998 continue; 1003 continue;
999 } 1004 }
1000 1005
1001 1006
1002 if ((vr->start >= uaddr) 1007 if ((vr->start >= uaddr)
1003 && (vr->start + vr->size <= uaddr + 1008 && (vr->start + vr->size <= uaddr + size))
1004 { 1009 {
1005 1010
1006 as_account_change_of_vr_protection( 1011 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1007 1012 vr->size, vr->access_rights,
1008 1013 new_access_rights);
1009 vr->access_rights = new_access_righ 1014 vr->access_rights = new_access_rights;
1010 1015
1011 if (vr->flags & SOS_VR_MAP_SHARED) 1016 if (vr->flags & SOS_VR_MAP_SHARED)
1012 1017
1013 1018
1014 sos_paging_set_prot_of_interval(v 1019 sos_paging_set_prot_of_interval(vr->start, vr->size,
1015 n 1020 new_access_rights);
1016 else 1021 else
1017 1022
1018 { 1023 {
1019 1024
1020 1025
1021 1026
1022 1027
1023 1028
1024 if (! (new_access_rights & SOS_ 1029 if (! (new_access_rights & SOS_VM_MAP_PROT_WRITE))
1025 sos_paging_set_prot_of_interv 1030 sos_paging_set_prot_of_interval(vr->start, vr->size,
1026 1031 new_access_rights);
1027 } 1032 }
1028 1033
1029 vr = vr->next_in_as; 1034 vr = vr->next_in_as;
1030 continue; 1035 continue;
1031 } 1036 }
1032 1037
1033 1038
1034 else if ( (vr->start < uaddr) 1039 else if ( (vr->start < uaddr)
1035 && (vr->start + vr->size > ua 1040 && (vr->start + vr->size > uaddr + size) )
1036 { 1041 {
1037 1042
1038 1043
1039 1044
1040 SOS_ASSERT_FATAL(! used_preallocate 1045 SOS_ASSERT_FATAL(! used_preallocated_middle_vr);
1041 SOS_ASSERT_FATAL(! used_preallocate 1046 SOS_ASSERT_FATAL(! used_preallocated_right_vr);
1042 used_preallocated_middle_vr = TRUE; 1047 used_preallocated_middle_vr = TRUE;
1043 memcpy(preallocated_middle_vr, vr, 1048 memcpy(preallocated_middle_vr, vr, sizeof(*vr));
1044 used_preallocated_right_vr = TRUE; 1049 used_preallocated_right_vr = TRUE;
1045 memcpy(preallocated_right_vr, vr, s 1050 memcpy(preallocated_right_vr, vr, sizeof(*vr));
1046 1051
1047 1052
1048 preallocated_middle_vr->start = uad 1053 preallocated_middle_vr->start = uaddr;
1049 preallocated_middle_vr->size = siz 1054 preallocated_middle_vr->size = size;
1050 preallocated_right_vr->start = uad 1055 preallocated_right_vr->start = uaddr + size;
1051 preallocated_right_vr->size = vr- 1056 preallocated_right_vr->size = vr->start + vr->size
1052 - 1057 - (uaddr + size);
1053 preallocated_middle_vr->offset_in_r 1058 preallocated_middle_vr->offset_in_resource
1054 += uaddr - vr->start; 1059 += uaddr - vr->start;
1055 preallocated_right_vr->offset_in_re 1060 preallocated_right_vr->offset_in_resource
1056 += uaddr + size - vr->start; 1061 += uaddr + size - vr->start;
1057 vr->size = uaddr - vr->start; 1062 vr->size = uaddr - vr->start;
1058 1063
1059 1064
1060 preallocated_middle_vr->access_righ 1065 preallocated_middle_vr->access_rights = new_access_rights;
1061 as_account_change_of_vr_protection( 1066 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1062 1067 size, vr->access_rights,
1063 1068 new_access_rights);
1064 1069
1065 1070
1066 list_insert_after_named(as->list_vr 1071 list_insert_after_named(as->list_vr, vr, preallocated_middle_vr,
1067 prev_in_as, 1072 prev_in_as, next_in_as);
1068 list_insert_after_named(as->list_vr 1073 list_insert_after_named(as->list_vr, preallocated_middle_vr,
1069 preallocate 1074 preallocated_right_vr,
1070 prev_in_as, 1075 prev_in_as, next_in_as);
1071 1076
1072 list_add_tail_named(vr->mapped_reso 1077 list_add_tail_named(vr->mapped_resource->list_vr,
1073 preallocated_mi 1078 preallocated_middle_vr,
1074 prev_in_mapped_ 1079 prev_in_mapped_resource,
1075 next_in_mapped_ 1080 next_in_mapped_resource);
1076 list_add_tail_named(vr->mapped_reso 1081 list_add_tail_named(vr->mapped_resource->list_vr,
1077 preallocated_ri 1082 preallocated_right_vr,
1078 prev_in_mapped_ 1083 prev_in_mapped_resource,
1079 next_in_mapped_ 1084 next_in_mapped_resource);
1080 1085
1081 1086
1082 if (!(preallocated_middle_vr->flags 1087 if (!(preallocated_middle_vr->flags & SOS_VR_MAP_SHARED)
1083 && (new_access_rights & SOS_VM_ 1088 && (new_access_rights & SOS_VM_MAP_PROT_WRITE))
1084 1089
1085 sos_paging_prepare_COW(preallocat 1090 sos_paging_prepare_COW(preallocated_middle_vr->start,
1086 preallocat 1091 preallocated_middle_vr->size);
1087 else 1092 else
1088 sos_paging_set_prot_of_interval(p 1093 sos_paging_set_prot_of_interval(preallocated_middle_vr->start,
1089 p 1094 preallocated_middle_vr->size,
1090 n 1095 new_access_rights);
1091 1096
1092 if (preallocated_right_vr->ops && p 1097 if (preallocated_right_vr->ops && preallocated_right_vr->ops->ref)
1093 preallocated_right_vr->ops->ref(p 1098 preallocated_right_vr->ops->ref(preallocated_right_vr);
1094 if (preallocated_middle_vr->ops && 1099 if (preallocated_middle_vr->ops && preallocated_middle_vr->ops->ref)
1095 preallocated_middle_vr->ops->ref( 1100 preallocated_middle_vr->ops->ref(preallocated_middle_vr);
1096 1101
1097 1102
1098 break; 1103 break;
1099 } 1104 }
1100 1105
1101 1106
1102 else if (uaddr <= vr->start) 1107 else if (uaddr <= vr->start)
1103 { 1108 {
1104 1109
1105 sos_uoffset_t offset_in_region = ua 1110 sos_uoffset_t offset_in_region = uaddr + size - vr->start;
1106 1111
1107 1112
1108 SOS_ASSERT_FATAL(! used_preallocate 1113 SOS_ASSERT_FATAL(! used_preallocated_middle_vr);
1109 used_preallocated_middle_vr = TRUE; 1114 used_preallocated_middle_vr = TRUE;
1110 memcpy(preallocated_middle_vr, vr, 1115 memcpy(preallocated_middle_vr, vr, sizeof(*vr));
1111 1116
1112 1117
1113 preallocated_middle_vr->start += of 1118 preallocated_middle_vr->start += offset_in_region;
1114 preallocated_middle_vr->size -= of 1119 preallocated_middle_vr->size -= offset_in_region;
1115 vr->size = of 1120 vr->size = offset_in_region;
1116 preallocated_middle_vr->offset_in_r 1121 preallocated_middle_vr->offset_in_resource += offset_in_region;
1117 1122
1118 1123
1119 as_account_change_of_vr_protection( 1124 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1120 1125 vr->size,
1121 1126 vr->access_rights,
1122 1127 new_access_rights);
1123 vr->access_rights = new_access_righ 1128 vr->access_rights = new_access_rights;
1124 1129
1125 1130
1126 list_insert_after_named(as->list_vr 1131 list_insert_after_named(as->list_vr, vr,
1127 preallocate 1132 preallocated_middle_vr,
1128 prev_in_as, 1133 prev_in_as, next_in_as);
1129 list_add_tail_named(vr->mapped_reso 1134 list_add_tail_named(vr->mapped_resource->list_vr,
1130 preallocated_mi 1135 preallocated_middle_vr,
1131 prev_in_mapped_ 1136 prev_in_mapped_resource,
1132 next_in_mapped_ 1137 next_in_mapped_resource);
1133 1138
1134 1139
1135 if (!(vr->flags & SOS_VR_MAP_SHARED 1140 if (!(vr->flags & SOS_VR_MAP_SHARED)
1136 && (new_access_rights & SOS_VM_ 1141 && (new_access_rights & SOS_VM_MAP_PROT_WRITE))
1137 1142
1138 sos_paging_prepare_COW(vr->start, 1143 sos_paging_prepare_COW(vr->start, vr->size);
1139 else 1144 else
1140 sos_paging_set_prot_of_interval(v 1145 sos_paging_set_prot_of_interval(vr->start, vr->size,
1141 n 1146 new_access_rights);
1142 1147
1143 if (preallocated_middle_vr->ops && 1148 if (preallocated_middle_vr->ops && preallocated_middle_vr->ops->ref)
1144 preallocated_middle_vr->ops->ref( 1149 preallocated_middle_vr->ops->ref(preallocated_middle_vr);
1145 1150
1146 1151
1147 1152
1148 break; 1153 break;
1149 } 1154 }
1150 1155
1151 1156
1152 else if (uaddr + size >= vr->start + vr 1157 else if (uaddr + size >= vr->start + vr->size)
1153 { 1158 {
1154 1159
1155 sos_uoffset_t offset_in_region = ua 1160 sos_uoffset_t offset_in_region = uaddr - vr->start;
1156 1161
1157 1162
1158 SOS_ASSERT_FATAL(! used_preallocate 1163 SOS_ASSERT_FATAL(! used_preallocated_right_vr);
1159 used_preallocated_right_vr = TRUE; 1164 used_preallocated_right_vr = TRUE;
1160 memcpy(preallocated_right_vr, vr, s 1165 memcpy(preallocated_right_vr, vr, sizeof(*vr));
1161 1166
1162 1167
1163 preallocated_right_vr->start 1168 preallocated_right_vr->start += offset_in_region;
1164 preallocated_right_vr->size 1169 preallocated_right_vr->size -= offset_in_region;
1165 vr->size 1170 vr->size = offset_in_region;
1166 preallocated_right_vr->offset_in_re 1171 preallocated_right_vr->offset_in_resource += offset_in_region;
1167 1172
1168 1173
1169 as_account_change_of_vr_protection( 1174 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1170 1175 preallocated_right_vr->size,
1171 1176 vr->access_rights,
1172 1177 new_access_rights);
1173 preallocated_right_vr->access_right 1178 preallocated_right_vr->access_rights = new_access_rights;
1174 1179
1175 1180
1176 list_insert_after_named(as->list_vr 1181 list_insert_after_named(as->list_vr, vr,
1177 preallocate 1182 preallocated_right_vr,
1178 prev_in_as, 1183 prev_in_as, next_in_as);
1179 list_add_tail_named(vr->mapped_reso 1184 list_add_tail_named(vr->mapped_resource->list_vr,
1180 preallocated_ri 1185 preallocated_right_vr,
1181 prev_in_mapped_ 1186 prev_in_mapped_resource,
1182 next_in_mapped_ 1187 next_in_mapped_resource);
1183 1188
1184 1189
1185 if (!(preallocated_right_vr->flags 1190 if (!(preallocated_right_vr->flags & SOS_VR_MAP_SHARED)
1186 && (new_access_rights & SOS_VM_ 1191 && (new_access_rights & SOS_VM_MAP_PROT_WRITE))
1187 1192
1188 sos_paging_prepare_COW(preallocat 1193 sos_paging_prepare_COW(preallocated_right_vr->start,
1189 preallocat 1194 preallocated_right_vr->size);
1190 else 1195 else
1191 sos_paging_set_prot_of_interval(p 1196 sos_paging_set_prot_of_interval(preallocated_right_vr->start,
1192 p 1197 preallocated_right_vr->size,
1193 n 1198 new_access_rights);
1194 1199
1195 if (preallocated_right_vr->ops && p 1200 if (preallocated_right_vr->ops && preallocated_right_vr->ops->ref)
1196 preallocated_right_vr->ops->ref(p 1201 preallocated_right_vr->ops->ref(preallocated_right_vr);
1197 1202
1198 vr = vr->next_in_as; 1203 vr = vr->next_in_as;
1199 continue; 1204 continue;
1200 } 1205 }
1201 1206
1202 sos_display_fatal_error("BUG"); 1207 sos_display_fatal_error("BUG");
1203 } 1208 }
1204 1209
1205 if (! used_preallocated_middle_vr) 1210 if (! used_preallocated_middle_vr)
1206 sos_kmem_vmm_free((sos_vaddr_t)preallocat 1211 sos_kmem_vmm_free((sos_vaddr_t)preallocated_middle_vr);
1207 if (! used_preallocated_right_vr) 1212 if (! used_preallocated_right_vr)
1208 sos_kmem_vmm_free((sos_vaddr_t)preallocat 1213 sos_kmem_vmm_free((sos_vaddr_t)preallocated_right_vr);
1209 1214
1210 return SOS_OK; 1215 return SOS_OK;
1211 } 1216 }
1212 1217
1213 1218
1214 sos_ret_t 1219 sos_ret_t
1215 sos_umem_vmm_resize(struct sos_umem_vmm_as * 1220 sos_umem_vmm_resize(struct sos_umem_vmm_as * as,
1216 sos_uaddr_t old_uaddr, so 1221 sos_uaddr_t old_uaddr, sos_size_t old_size,
1217 sos_uaddr_t *new_uaddr, s 1222 sos_uaddr_t *new_uaddr, sos_size_t new_size,
1218 sos_ui32_t flags) 1223 sos_ui32_t flags)
1219 { 1224 {
1220 sos_luoffset_t new_offset_in_resource; 1225 sos_luoffset_t new_offset_in_resource;
1221 sos_bool_t must_move_vr = FALSE; 1226 sos_bool_t must_move_vr = FALSE;
1222 struct sos_umem_vmm_vr *vr, *prev_vr, *next 1227 struct sos_umem_vmm_vr *vr, *prev_vr, *next_vr;
1223 1228
1224 1229
1225 if (*new_uaddr < SOS_PAGING_BASE_USER_ADDRE 1230 if (*new_uaddr < SOS_PAGING_BASE_USER_ADDRESS)
1226 return -SOS_EINVAL; 1231 return -SOS_EINVAL;
1227 if (*new_uaddr > SOS_PAGING_TOP_USER_ADDRES 1232 if (*new_uaddr > SOS_PAGING_TOP_USER_ADDRESS - new_size)
1228 return -SOS_EINVAL; 1233 return -SOS_EINVAL;
1229 1234
1230 old_uaddr = SOS_PAGE_ALIGN_INF(old_uaddr); 1235 old_uaddr = SOS_PAGE_ALIGN_INF(old_uaddr);
1231 old_size = SOS_PAGE_ALIGN_SUP(old_size); 1236 old_size = SOS_PAGE_ALIGN_SUP(old_size);
1232 if (! SOS_IS_PAGE_ALIGNED(*new_uaddr)) 1237 if (! SOS_IS_PAGE_ALIGNED(*new_uaddr))
1233 return -SOS_EINVAL; 1238 return -SOS_EINVAL;
1234 if (new_size <= 0) 1239 if (new_size <= 0)
1235 return -SOS_EINVAL; 1240 return -SOS_EINVAL;
1236 new_size = SOS_PAGE_ALIGN_SUP(new_size); 1241 new_size = SOS_PAGE_ALIGN_SUP(new_size);
1237 1242
1238 1243
1239 vr = find_first_intersecting_vr(as, old_uad 1244 vr = find_first_intersecting_vr(as, old_uaddr, old_size);
1240 if (! vr) 1245 if (! vr)
1241 return -SOS_EINVAL; 1246 return -SOS_EINVAL;
1242 1247
1243 1248
1244 if ( (vr->start > old_uaddr) 1249 if ( (vr->start > old_uaddr)
1245 || (vr->start + vr->size < old_uaddr + 1250 || (vr->start + vr->size < old_uaddr + old_size) )
1246 return -SOS_EINVAL; 1251 return -SOS_EINVAL;
1247 1252
1248 1253
1249 1254
1250 prev_vr = vr->prev_in_as; 1255 prev_vr = vr->prev_in_as;
1251 if (prev_vr->start >= vr->start) 1256 if (prev_vr->start >= vr->start)
1252 prev_vr = NULL; 1257 prev_vr = NULL;
1253 next_vr = vr->prev_in_as; 1258 next_vr = vr->prev_in_as;
1254 if (next_vr->start <= vr->start) 1259 if (next_vr->start <= vr->start)
1255 next_vr = NULL; 1260 next_vr = NULL;
1256 1261
1257 1262
1258 1263
1259 1264
1260 1265
1261 1266
1262 1267
1263 if ( (*new_uaddr < vr->start) 1268 if ( (*new_uaddr < vr->start)
1264 && (vr->start - *new_uaddr > vr->offse 1269 && (vr->start - *new_uaddr > vr->offset_in_resource) )
1265 return -SOS_EINVAL; 1270 return -SOS_EINVAL;
1266 1271
1267 1272
1268 if (vr->start > *new_uaddr) 1273 if (vr->start > *new_uaddr)
1269 new_offset_in_resource 1274 new_offset_in_resource
1270 = vr->offset_in_resource 1275 = vr->offset_in_resource
1271 - (vr->start - *new_uaddr); 1276 - (vr->start - *new_uaddr);
1272 else 1277 else
1273 new_offset_in_resource 1278 new_offset_in_resource
1274 = vr->offset_in_resource 1279 = vr->offset_in_resource
1275 + (*new_uaddr - vr->start); 1280 + (*new_uaddr - vr->start);
1276 1281
1277 1282
1278 1283
1279 if (prev_vr && (prev_vr->start + prev_vr->s 1284 if (prev_vr && (prev_vr->start + prev_vr->size > *new_uaddr))
1280 must_move_vr |= TRUE; 1285 must_move_vr |= TRUE;
1281 if (next_vr && (next_vr->start < *new_uaddr 1286 if (next_vr && (next_vr->start < *new_uaddr + new_size))
1282 must_move_vr |= TRUE; 1287 must_move_vr |= TRUE;
1283 1288
1284 1289
1285 if (*new_uaddr < SOS_PAGING_BASE_USER_ADDRE 1290 if (*new_uaddr < SOS_PAGING_BASE_USER_ADDRESS)
1286 must_move_vr |= TRUE; 1291 must_move_vr |= TRUE;
1287 if (*new_uaddr > SOS_PAGING_TOP_USER_ADDRES 1292 if (*new_uaddr > SOS_PAGING_TOP_USER_ADDRESS - new_size)
1288 must_move_vr |= TRUE; 1293 must_move_vr |= TRUE;
1289 1294
1290 1295
1291 if ( must_move_vr && !(flags & SOS_VR_REMAP 1296 if ( must_move_vr && !(flags & SOS_VR_REMAP_MAYMOVE) )
1292 return -SOS_EINVAL; 1297 return -SOS_EINVAL;
1293 1298
1294 1299
1295 1300
1296 if (must_move_vr) 1301 if (must_move_vr)
1297 { 1302 {
1298 sos_uaddr_t uaddr, result_uaddr; 1303 sos_uaddr_t uaddr, result_uaddr;
1299 sos_ret_t retval; 1304 sos_ret_t retval;
1300 1305
1301 result_uaddr = *new_uaddr; 1306 result_uaddr = *new_uaddr;
1302 retval = sos_umem_vmm_map(as, & result_ 1307 retval = sos_umem_vmm_map(as, & result_uaddr, new_size,
1303 vr->access_ri 1308 vr->access_rights,
1304 vr->flags | I 1309 vr->flags | INTERNAL_MAP_CALLED_FROM_MREMAP,
1305 vr->mapped_re 1310 vr->mapped_resource,
1306 new_offset_in 1311 new_offset_in_resource);
1307 if (SOS_OK != retval) 1312 if (SOS_OK != retval)
1308 return retval; 1313 return retval;
1309 1314
1310 1315
1311 for (uaddr = vr->start ; 1316 for (uaddr = vr->start ;
1312 uaddr < vr->start + vr->size ; 1317 uaddr < vr->start + vr->size ;
1313 uaddr += SOS_PAGE_SIZE) 1318 uaddr += SOS_PAGE_SIZE)
1314 { 1319 {
1315 sos_paddr_t paddr; 1320 sos_paddr_t paddr;
1316 sos_ui32_t prot; 1321 sos_ui32_t prot;
1317 sos_uaddr_t vaddr; 1322 sos_uaddr_t vaddr;
1318 1323
1319 if (uaddr < *new_uaddr) 1324 if (uaddr < *new_uaddr)
1320 continue; 1325 continue;
1321 if (uaddr > *new_uaddr + new_size) 1326 if (uaddr > *new_uaddr + new_size)
1322 continue; 1327 continue;
1323 1328
1324 1329
1325 1330
1326 if (vr->start >= *new_uaddr) 1331 if (vr->start >= *new_uaddr)
1327 vaddr = result_uaddr 1332 vaddr = result_uaddr
1328 + (uaddr - vr->start) 1333 + (uaddr - vr->start)
1329 + (vr->start - *new_uaddr); 1334 + (vr->start - *new_uaddr);
1330 else 1335 else
1331 vaddr = result_uaddr 1336 vaddr = result_uaddr
1332 + (uaddr - vr->start) 1337 + (uaddr - vr->start)
1333 - (*new_uaddr - vr->start); 1338 - (*new_uaddr - vr->start);
1334 1339
1335 paddr = sos_paging_get_paddr(uaddr) 1340 paddr = sos_paging_get_paddr(uaddr);
1336 if (! paddr) 1341 if (! paddr)
1337 1342
1338 continue; 1343 continue;
1339 1344
1340 prot = sos_paging_get_prot(uaddr); 1345 prot = sos_paging_get_prot(uaddr);
1341 SOS_ASSERT_FATAL(prot); 1346 SOS_ASSERT_FATAL(prot);
1342 1347
1343 1348
1344 retval = sos_paging_map(paddr, vadd 1349 retval = sos_paging_map(paddr, vaddr, TRUE, prot);
1345 if (SOS_OK != retval) 1350 if (SOS_OK != retval)
1346 { 1351 {
1347 sos_umem_vmm_unmap(as, result_u 1352 sos_umem_vmm_unmap(as, result_uaddr, new_size);
1348 return retval; 1353 return retval;
1349 } 1354 }
1350 } 1355 }
1351 1356
1352 retval = sos_umem_vmm_unmap(as, vr->sta 1357 retval = sos_umem_vmm_unmap(as, vr->start, vr->size);
1353 if (SOS_OK != retval) 1358 if (SOS_OK != retval)
1354 { 1359 {
1355 sos_umem_vmm_unmap(as, result_uaddr 1360 sos_umem_vmm_unmap(as, result_uaddr, new_size);
1356 return retval; 1361 return retval;
1357 } 1362 }
1358 1363
1359 *new_uaddr = result_uaddr; 1364 *new_uaddr = result_uaddr;
1360 return retval; 1365 return retval;
1361 } 1366 }
1362 1367
1363 1368
1364 1369
1365 1370
1366 if (*new_uaddr + new_size < vr->start + vr- 1371 if (*new_uaddr + new_size < vr->start + vr->size)
1367 sos_umem_vmm_unmap(as, *new_uaddr + new_s 1372 sos_umem_vmm_unmap(as, *new_uaddr + new_size,
1368 vr->start + vr->size - 1373 vr->start + vr->size - (*new_uaddr + new_size));
1369 else 1374 else
1370 { 1375 {
1371 as_account_change_of_vr_protection(as, 1376 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1372 *new 1377 *new_uaddr + new_size
1373 - 1378 - (vr->start + vr->size),
1374 0, v 1379 0, vr->access_rights);
1375 vr->size += *new_uaddr + new_size - (vr 1380 vr->size += *new_uaddr + new_size - (vr->start + vr->size);
1376 } 1381 }
1377 1382
1378 if (*new_uaddr > vr->start) 1383 if (*new_uaddr > vr->start)
1379 sos_umem_vmm_unmap(as, vr->start, *new_ua 1384 sos_umem_vmm_unmap(as, vr->start, *new_uaddr - vr->start);
1380 else 1385 else
1381 { 1386 {
1382 as_account_change_of_vr_protection(as, 1387 as_account_change_of_vr_protection(as, vr->flags & SOS_VR_MAP_SHARED,
1383 vr-> 1388 vr->start - *new_uaddr,
1384 0, v 1389 0, vr->access_rights);
1385 vr->size += vr->start - *new_uaddr; 1390 vr->size += vr->start - *new_uaddr;
1386 vr->start = *new_uaddr; 1391 vr->start = *new_uaddr;
1387 vr->offset_in_resource = new_offset_in_ 1392 vr->offset_in_resource = new_offset_in_resource;
1388 } 1393 }
1389 1394
1390 SOS_ASSERT_FATAL(vr->start == *new_uaddr); 1395 SOS_ASSERT_FATAL(vr->start == *new_uaddr);
1391 SOS_ASSERT_FATAL(vr->size == new_size); 1396 SOS_ASSERT_FATAL(vr->size == new_size);
1392 SOS_ASSERT_FATAL(vr->offset_in_resource == 1397 SOS_ASSERT_FATAL(vr->offset_in_resource == new_offset_in_resource);
1393 1398
1394 return SOS_OK; 1399 return SOS_OK;
1395 } 1400 }
1396 1401
1397 1402
1398 sos_ret_t sos_umem_vmm_try_resolve_page_fault 1403 sos_ret_t sos_umem_vmm_try_resolve_page_fault(sos_uaddr_t uaddr,
1399 1404 sos_bool_t write_access,
1400 1405 sos_bool_t user_access)
1401 { 1406 {
1402 struct sos_process *process = sos_threa 1407 struct sos_process *process = sos_thread_get_current()->process;
1403 struct sos_umem_vmm_as *as; 1408 struct sos_umem_vmm_as *as;
1404 struct sos_umem_vmm_vr *vr; 1409 struct sos_umem_vmm_vr *vr;
1405 1410
1406 if (! process) 1411 if (! process)
1407 return -SOS_EFAULT; 1412 return -SOS_EFAULT;
1408 1413
1409 as = sos_process_get_address_space(process) 1414 as = sos_process_get_address_space(process);
1410 if (! as) 1415 if (! as)
1411 return -SOS_EFAULT; 1416 return -SOS_EFAULT;
1412 1417
1413 vr = find_first_intersecting_vr(as, uaddr, 1418 vr = find_first_intersecting_vr(as, uaddr, 1);
1414 if (! vr) 1419 if (! vr)
1415 return -SOS_EFAULT; 1420 return -SOS_EFAULT;
1416 1421
1417 1422
1418 if (write_access && !(vr->access_rights & S 1423 if (write_access && !(vr->access_rights & SOS_VM_MAP_PROT_WRITE))
1419 return -SOS_EFAULT; 1424 return -SOS_EFAULT;
1420 1425
1421 1426
1422 if (write_access && !(vr->flags & SOS_VR_MA 1427 if (write_access && !(vr->flags & SOS_VR_MAP_SHARED))
1423 { 1428 {
1424 if (SOS_OK == sos_paging_try_resolve_CO 1429 if (SOS_OK == sos_paging_try_resolve_COW(uaddr))
1425 { 1430 {
1426 as->pgflt_cow ++; 1431 as->pgflt_cow ++;
1427 return SOS_OK; 1432 return SOS_OK;
1428 } 1433 }
1429 } 1434 }
1430 1435
1431 1436
1432 if (SOS_OK != vr->ops->page_in(vr, uaddr, w 1437 if (SOS_OK != vr->ops->page_in(vr, uaddr, write_access))
1433 { 1438 {
1434 as->pgflt_invalid ++; 1439 as->pgflt_invalid ++;
1435 return -SOS_EFAULT; 1440 return -SOS_EFAULT;
1436 } 1441 }
1437 1442
1438 as->phys_total += SOS_PAGE_SIZE; 1443 as->phys_total += SOS_PAGE_SIZE;
1439 as->pgflt_page_in ++; 1444 as->pgflt_page_in ++;
1440 1445
1441 1446
1442 if (!(vr->flags & SOS_VR_MAP_SHARED)) 1447 if (!(vr->flags & SOS_VR_MAP_SHARED))
1443 { 1448 {
1444 sos_paging_prepare_COW(SOS_PAGE_ALIGN_I 1449 sos_paging_prepare_COW(SOS_PAGE_ALIGN_INF(uaddr),
1445 SOS_PAGE_SIZE); 1450 SOS_PAGE_SIZE);
1446 } 1451 }
1447 1452
1448 return SOS_OK; 1453 return SOS_OK;
1449 } 1454 }
1450 1455
1451 1456
1452 sos_ret_t 1457 sos_ret_t
1453 sos_umem_vmm_init_heap(struct sos_umem_vmm_as 1458 sos_umem_vmm_init_heap(struct sos_umem_vmm_as * as,
1454 sos_uaddr_t heap_start 1459 sos_uaddr_t heap_start)
1455 { 1460 {
1456 SOS_ASSERT_FATAL(! as->heap_start); 1461 SOS_ASSERT_FATAL(! as->heap_start);
1457 1462
1458 as->heap_start = heap_start; 1463 as->heap_start = heap_start;
1459 as->heap_size = 0; 1464 as->heap_size = 0;
1460 return SOS_OK; 1465 return SOS_OK;
1461 } 1466 }
1462 1467
1463 1468
1464 sos_uaddr_t 1469 sos_uaddr_t
1465 sos_umem_vmm_brk(struct sos_umem_vmm_as * as, 1470 sos_umem_vmm_brk(struct sos_umem_vmm_as * as,
1466 sos_uaddr_t new_top_uaddr) 1471 sos_uaddr_t new_top_uaddr)
1467 { 1472 {
1468 sos_uaddr_t new_start; 1473 sos_uaddr_t new_start;
1469 sos_size_t new_size; 1474 sos_size_t new_size;
1470 SOS_ASSERT_FATAL(as->heap_start); 1475 SOS_ASSERT_FATAL(as->heap_start);
1471 1476
1472 if (! new_top_uaddr) 1477 if (! new_top_uaddr)
1473 return as->heap_start + as->heap_size; 1478 return as->heap_start + as->heap_size;
1474 1479
1475 if (new_top_uaddr == as->heap_start + as->h 1480 if (new_top_uaddr == as->heap_start + as->heap_size)
1476 return as->heap_start + as->heap_size; 1481 return as->heap_start + as->heap_size;
1477 1482
1478 if (new_top_uaddr < as->heap_start) 1483 if (new_top_uaddr < as->heap_start)
1479 return (sos_uaddr_t)NULL; 1484 return (sos_uaddr_t)NULL;
1480 1485
1481 new_top_uaddr = SOS_PAGE_ALIGN_SUP(new_top_ 1486 new_top_uaddr = SOS_PAGE_ALIGN_SUP(new_top_uaddr);
1482 new_start = as->heap_start; 1487 new_start = as->heap_start;
1483 new_size = new_top_uaddr - as->heap_start; 1488 new_size = new_top_uaddr - as->heap_start;
1484 1489
1485 1490
1486 if (! as->heap_size) 1491 if (! as->heap_size)
1487 { 1492 {
1488 if (SOS_OK != sos_dev_zero_map(as, & as 1493 if (SOS_OK != sos_dev_zero_map(as, & as->heap_start,
1489 new_size 1494 new_size,
1490 SOS_VM_M 1495 SOS_VM_MAP_PROT_READ
1491 | SOS_VM 1496 | SOS_VM_MAP_PROT_WRITE,
1492 0 1497 0 ))
1493 return (sos_uaddr_t)NULL; 1498 return (sos_uaddr_t)NULL;
1494 1499
1495 as->heap_size = new_size; 1500 as->heap_size = new_size;
1496 return as->heap_start + as->heap_size; 1501 return as->heap_start + as->heap_size;
1497 } 1502 }
1498 1503
1499 1504
1500 if (new_size <= 0) 1505 if (new_size <= 0)
1501 { 1506 {
1502 if (SOS_OK != sos_umem_vmm_unmap(as, 1507 if (SOS_OK != sos_umem_vmm_unmap(as,
1503 as->he 1508 as->heap_start, as->heap_size))
1504 return (sos_uaddr_t)NULL; 1509 return (sos_uaddr_t)NULL;
1505 } 1510 }
1506 else 1511 else
1507 { 1512 {
1508 if (SOS_OK != sos_umem_vmm_resize(as, 1513 if (SOS_OK != sos_umem_vmm_resize(as,
1509 as->h 1514 as->heap_start, as->heap_size,
1510 & new 1515 & new_start, new_size,
1511 0)) 1516 0))
1512 return (sos_uaddr_t)NULL; 1517 return (sos_uaddr_t)NULL;
1513 } 1518 }
1514 1519
1515 SOS_ASSERT_FATAL(new_start == as->heap_star 1520 SOS_ASSERT_FATAL(new_start == as->heap_start);
1516 as->heap_size = new_size; 1521 as->heap_size = new_size;
1517 return new_top_uaddr; 1522 return new_top_uaddr;
1518 } 1523 }
1519 1524
1520 1525
1521 static struct sos_umem_vmm_vr * 1526 static struct sos_umem_vmm_vr *
1522 find_enclosing_or_next_vr(struct sos_umem_vmm 1527 find_enclosing_or_next_vr(struct sos_umem_vmm_as * as,
1523 sos_uaddr_t uaddr) 1528 sos_uaddr_t uaddr)
1524 { 1529 {
1525 struct sos_umem_vmm_vr *vr; 1530 struct sos_umem_vmm_vr *vr;
1526 int nb_vr; 1531 int nb_vr;
1527 1532
1528 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS) 1533 if (uaddr < SOS_PAGING_BASE_USER_ADDRESS)
1529 return NULL; 1534 return NULL;
1530 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS) 1535 if (uaddr > SOS_PAGING_TOP_USER_ADDRESS)
1531 return NULL; 1536 return NULL;
1532 1537
1533 list_foreach_named(as->list_vr, vr, nb_vr, 1538 list_foreach_named(as->list_vr, vr, nb_vr, prev_in_as, next_in_as)
1534 { 1539 {
1535 1540
1536 1541
1537 if (uaddr <= vr->start + (vr->size - 1) 1542 if (uaddr <= vr->start + (vr->size - 1))
1538 return vr; 1543 return vr;
1539 } 1544 }
1540 1545
1541 return NULL; 1546 return NULL;
1542 } 1547 }
1543 1548
1544 1549
1545 static struct sos_umem_vmm_vr * 1550 static struct sos_umem_vmm_vr *
1546 find_first_intersecting_vr(struct sos_umem_vm 1551 find_first_intersecting_vr(struct sos_umem_vmm_as * as,
1547 sos_uaddr_t start_ 1552 sos_uaddr_t start_uaddr, sos_size_t size)
1548 { 1553 {
1549 struct sos_umem_vmm_vr * vr; 1554 struct sos_umem_vmm_vr * vr;
1550 vr = find_enclosing_or_next_vr(as, start_ua 1555 vr = find_enclosing_or_next_vr(as, start_uaddr);
1551 if (! vr) 1556 if (! vr)
1552 return NULL; 1557 return NULL;
1553 1558
1554 if (start_uaddr + size <= vr->start) 1559 if (start_uaddr + size <= vr->start)
1555 return NULL; 1560 return NULL;
1556 1561
1557 return vr; 1562 return vr;
1558 } 1563 }
1559 1564
1560 1565
1561 static sos_uaddr_t 1566 static sos_uaddr_t
1562 find_first_free_interval(struct sos_umem_vmm_ 1567 find_first_free_interval(struct sos_umem_vmm_as * as,
1563 sos_uaddr_t hint_uad 1568 sos_uaddr_t hint_uaddr, sos_size_t size)
1564 { 1569 {
1565 struct sos_umem_vmm_vr * initial_vr, * vr; 1570 struct sos_umem_vmm_vr * initial_vr, * vr;
1566 1571
1567 if (hint_uaddr < SOS_PAGING_BASE_USER_ADDRE 1572 if (hint_uaddr < SOS_PAGING_BASE_USER_ADDRESS)
1568 hint_uaddr = SOS_PAGING_BASE_USER_ADDRESS 1573 hint_uaddr = SOS_PAGING_BASE_USER_ADDRESS;
1569 1574
1570 if (hint_uaddr > SOS_PAGING_TOP_USER_ADDRES 1575 if (hint_uaddr > SOS_PAGING_TOP_USER_ADDRESS - size + 1)
1571 return (sos_uaddr_t)NULL; 1576 return (sos_uaddr_t)NULL;
1572 1577
1573 initial_vr = vr = find_enclosing_or_next_vr 1578 initial_vr = vr = find_enclosing_or_next_vr(as, hint_uaddr);
1574 if (! vr) 1579 if (! vr)
1575 1580
1576 return hint_uaddr; 1581 return hint_uaddr;
1577 1582
1578 1583
1579 do 1584 do
1580 { 1585 {
1581 1586
1582 if (hint_uaddr + size <= vr->start) 1587 if (hint_uaddr + size <= vr->start)
1583 1588
1584 return hint_uaddr; 1589 return hint_uaddr;
1585 1590
1586 1591
1587 1592
1588 if (vr->next_in_as->start >= hint_uaddr 1593 if (vr->next_in_as->start >= hint_uaddr)
1589 1594
1590 hint_uaddr = vr->start + vr->size; 1595 hint_uaddr = vr->start + vr->size;
1591 else 1596 else
1592 { 1597 {
1593 1598
1594 1599
1595 1600
1596 if (hint_uaddr <= SOS_PAGING_TOP_US 1601 if (hint_uaddr <= SOS_PAGING_TOP_USER_ADDRESS - size)
1597 return hint_uaddr; 1602 return hint_uaddr;
1598 1603
1599 hint_uaddr = SOS_PAGING_BASE_USER_A 1604 hint_uaddr = SOS_PAGING_BASE_USER_ADDRESS;
1600 } 1605 }
1601 1606
1602 1607
1603 vr = vr->next_in_as; 1608 vr = vr->next_in_as;
1604 } 1609 }
1605 while (vr != initial_vr); 1610 while (vr != initial_vr);
1606 1611
1607 1612
1608 1613
1609 1614
1610 return (sos_uaddr_t)NULL; 1615 return (sos_uaddr_t)NULL;
1611 } 1616 }
1612 1617
1613 1618
1614 static void 1619 static void
1615 as_account_change_of_vr_protection(struct sos 1620 as_account_change_of_vr_protection(struct sos_umem_vmm_as * as,
1616 sos_bool_t 1621 sos_bool_t is_shared,
1617 sos_size_t 1622 sos_size_t size,
1618 sos_ui32_t 1623 sos_ui32_t prev_access_rights,
1619 sos_ui32_t 1624 sos_ui32_t new_access_rights)
1620 { 1625 {
1621 if (prev_access_rights == new_access_rights 1626 if (prev_access_rights == new_access_rights)
1622 return; 1627 return;
1623 1628
1624 #define _UPDATE_VMSTAT(field,is_increment) \ 1629 #define _UPDATE_VMSTAT(field,is_increment) \
1625 ({ if (is_increment > 0) \ 1630 ({ if (is_increment > 0) \
1626 as->field += size; \ 1631 as->field += size; \
1627 else \ 1632 else \
1628 { SOS_ASSERT_FATAL(as->field >= size); 1633 { SOS_ASSERT_FATAL(as->field >= size); as->field -= size; } })
1629 #define UPDATE_VMSTAT(field,is_increment) \ 1634 #define UPDATE_VMSTAT(field,is_increment) \
1630 ({ if (is_shared) _UPDATE_VMSTAT(vm_shrd.fi 1635 ({ if (is_shared) _UPDATE_VMSTAT(vm_shrd.field, is_increment); \
1631 _UPDATE_VMSTAT(vm_total.field, is_increm 1636 _UPDATE_VMSTAT(vm_total.field, is_increment); \
1632 SOS_ASSERT_FATAL(as->vm_total.field >= a 1637 SOS_ASSERT_FATAL(as->vm_total.field >= as->vm_shrd.field); })
1633 1638
1634 if ( (new_access_rights & SOS_VM_MAP_PROT_W 1639 if ( (new_access_rights & SOS_VM_MAP_PROT_WRITE)
1635 && !(prev_access_rights & SOS_VM_MAP_P 1640 && !(prev_access_rights & SOS_VM_MAP_PROT_WRITE))
1636 { 1641 {
1637 UPDATE_VMSTAT(rw, +1); 1642 UPDATE_VMSTAT(rw, +1);
1638 if (prev_access_rights & SOS_VM_MAP_PRO 1643 if (prev_access_rights & SOS_VM_MAP_PROT_READ)
1639 UPDATE_VMSTAT(ro, -1); 1644 UPDATE_VMSTAT(ro, -1);
1640 } 1645 }
1641 else if ( !(new_access_rights & SOS_VM_MAP_ 1646 else if ( !(new_access_rights & SOS_VM_MAP_PROT_WRITE)
1642 && (prev_access_rights & SOS_VM_M 1647 && (prev_access_rights & SOS_VM_MAP_PROT_WRITE))
1643 { 1648 {
1644 if (new_access_rights & SOS_VM_MAP_PROT 1649 if (new_access_rights & SOS_VM_MAP_PROT_READ)
1645 UPDATE_VMSTAT(ro, +1); 1650 UPDATE_VMSTAT(ro, +1);
1646 UPDATE_VMSTAT(rw, -1); 1651 UPDATE_VMSTAT(rw, -1);
1647 } 1652 }
1648 else if (new_access_rights & SOS_VM_MAP_PRO 1653 else if (new_access_rights & SOS_VM_MAP_PROT_READ)
1649 UPDATE_VMSTAT(ro, +1); 1654 UPDATE_VMSTAT(ro, +1);
1650 else if (!(new_access_rights & SOS_VM_MAP_P 1655 else if (!(new_access_rights & SOS_VM_MAP_PROT_READ))
1651 UPDATE_VMSTAT(ro, -1); 1656 UPDATE_VMSTAT(ro, -1);
1652 1657
1653 if ( (new_access_rights & SOS_VM_MAP_PROT_E 1658 if ( (new_access_rights & SOS_VM_MAP_PROT_EXEC)
1654 && !(prev_access_rights & SOS_VM_MAP_P 1659 && !(prev_access_rights & SOS_VM_MAP_PROT_EXEC))
1655 { 1660 {
1656 UPDATE_VMSTAT(code, +1); 1661 UPDATE_VMSTAT(code, +1);
1657 } 1662 }
1658 else if ( !(new_access_rights & SOS_VM_MAP_ 1663 else if ( !(new_access_rights & SOS_VM_MAP_PROT_EXEC)
1659 && (prev_access_rights & SOS_VM_M 1664 && (prev_access_rights & SOS_VM_MAP_PROT_EXEC))
1660 { 1665 {
1661 UPDATE_VMSTAT(code, -1); 1666 UPDATE_VMSTAT(code, -1);
1662 } 1667 }
1663 1668
1664 if (new_access_rights && !prev_access_right 1669 if (new_access_rights && !prev_access_rights)
1665 UPDATE_VMSTAT(overall, +1); 1670 UPDATE_VMSTAT(overall, +1);
1666 else if (!new_access_rights && prev_access_ 1671 else if (!new_access_rights && prev_access_rights)
1667 UPDATE_VMSTAT(overall, -1); 1672 UPDATE_VMSTAT(overall, -1);
1668 1673
1669 } 1674 }