Diff markup
001 001
002 002
003 003
004 004
005 005
006 006
007 007
008 008
009 009
010 010
011 011
012 012
013 013
014 014
015 015
016 016
017 017
018 #include <sos/physmem.h> 018 #include <sos/physmem.h>
019 #include <sos/klibc.h> 019 #include <sos/klibc.h>
020 #include <sos/assert.h> 020 #include <sos/assert.h>
021 021
>> 022 #include "mm_context.h"
>> 023
022 #include "paging.h" 024 #include "paging.h"
023 025
>> 026
>> 027
>> 028
>> 029
>> 030
>> 031
>> 032
>> 033
>> 034
>> 035
>> 036
>> 037
>> 038
>> 039
>> 040
>> 041
>> 042
>> 043
>> 044
024 045
025 046
026 struct x86_pde 047 struct x86_pde
027 { 048 {
028 sos_ui32_t present :1; 049 sos_ui32_t present :1;
029 sos_ui32_t write :1; 050 sos_ui32_t write :1;
030 sos_ui32_t user :1; 051 sos_ui32_t user :1;
031 sos_ui32_t write_through :1; 052 sos_ui32_t write_through :1;
032 sos_ui32_t cache_disabled :1; 053 sos_ui32_t cache_disabled :1;
033 sos_ui32_t accessed :1; 054 sos_ui32_t accessed :1;
034 sos_ui32_t zero :1; 055 sos_ui32_t zero :1;
035 sos_ui32_t page_size :1; 056 sos_ui32_t page_size :1;
036 sos_ui32_t global_page :1; 057 sos_ui32_t global_page :1;
037 sos_ui32_t custom :3; 058 sos_ui32_t custom :3;
038 sos_ui32_t pt_paddr :20; 059 sos_ui32_t pt_paddr :20;
039 } __attribute__ ((packed)); 060 } __attribute__ ((packed));
040 061
041 062
>> 063
>> 064 typedef union {
>> 065 struct x86_pde pde;
>> 066 sos_ui32_t ui32;
>> 067 } x86_pde_val_t;
>> 068
>> 069
042 070
043 071
044 struct x86_pte 072 struct x86_pte
045 { 073 {
046 sos_ui32_t present :1; 074 sos_ui32_t present :1;
047 sos_ui32_t write :1; 075 sos_ui32_t write :1;
048 sos_ui32_t user :1; 076 sos_ui32_t user :1;
049 sos_ui32_t write_through :1; 077 sos_ui32_t write_through :1;
050 sos_ui32_t cache_disabled :1; 078 sos_ui32_t cache_disabled :1;
051 sos_ui32_t accessed :1; 079 sos_ui32_t accessed :1;
052 sos_ui32_t dirty :1; 080 sos_ui32_t dirty :1;
053 sos_ui32_t zero :1; 081 sos_ui32_t zero :1;
054 sos_ui32_t global_page :1; 082 sos_ui32_t global_page :1;
055 083
056 sos_ui32_t custom :3; 084 sos_ui32_t custom :3;
057 sos_ui32_t paddr :20; 085 sos_ui32_t paddr :20;
058 } __attribute__ ((packed)); 086 } __attribute__ ((packed));
059 087
060 088
>> 089
>> 090 typedef union {
>> 091 struct x86_pte pte;
>> 092 sos_ui32_t ui32;
>> 093 } x86_pte_val_t;
>> 094
>> 095
061 096
062 097
063 struct x86_pdbr 098 struct x86_pdbr
064 { 099 {
065 sos_ui32_t zero1 :3; 100 sos_ui32_t zero1 :3;
066 sos_ui32_t write_through :1; 101 sos_ui32_t write_through :1;
067 sos_ui32_t cache_disabled :1; 102 sos_ui32_t cache_disabled :1;
068 sos_ui32_t zero2 :7; 103 sos_ui32_t zero2 :7;
069 sos_ui32_t pd_paddr :20; 104 sos_ui32_t pd_paddr :20;
070 } __attribute__ ((packed)); 105 } __attribute__ ((packed));
071 106
072 107
073 108
074 109
075 110
076 111
077 112
078 #define invlpg(vaddr) \ 113 #define invlpg(vaddr) \
079 do { \ 114 do { \
080 __asm__ __volatile__("invlpg %0"::"m"(* 115 __asm__ __volatile__("invlpg %0"::"m"(*((unsigned *)(vaddr)))); \
081 } while(0) 116 } while(0)
082 117
083 118
084 119
085 120
086 121
087 122
088 #define flush_tlb() \ 123 #define flush_tlb() \
089 do { \ 124 do { \
090 unsigned long tmpreg; \ 125 unsigned long tmpreg; \
091 asm volatile("movl %%cr3,%0\n\tmovl %0 126 asm volatile("movl %%cr3,%0\n\tmovl %0,%%cr3" :"=r" \
092 (tmpreg) : :"memory"); \ 127 (tmpreg) : :"memory"); \
093 } while (0) 128 } while (0)
094 129
095 130
096 131
097 132
098 133
099 134
100 #define virt_to_pd_index(vaddr) \ 135 #define virt_to_pd_index(vaddr) \
101 (((unsigned)(vaddr)) >> 22) 136 (((unsigned)(vaddr)) >> 22)
102 137
103 138
104 139
105 140
106 141
107 142
108 #define virt_to_pt_index(vaddr) \ 143 #define virt_to_pt_index(vaddr) \
109 ( (((unsigned)(vaddr)) >> 12) & 0x3ff ) 144 ( (((unsigned)(vaddr)) >> 12) & 0x3ff )
110 145
111 146
112 147
113 148
114 149
115 150
116 #define virt_to_page_offset(vaddr) \ 151 #define virt_to_page_offset(vaddr) \
117 (((unsigned)(vaddr)) & SOS_PAGE_MASK) 152 (((unsigned)(vaddr)) & SOS_PAGE_MASK)
118 153
119 154
120 155
121 156
122 157
123 158
124 159
125 static sos_ret_t paging_setup_map_helper(struc 160 static sos_ret_t paging_setup_map_helper(struct x86_pde * pd,
126 sos_p 161 sos_paddr_t ppage,
127 sos_v 162 sos_vaddr_t vaddr)
128 { 163 {
129 164
130 165
131 unsigned index_in_pd = virt_to_pd_index(vadd 166 unsigned index_in_pd = virt_to_pd_index(vaddr);
132 unsigned index_in_pt = virt_to_pt_index(vadd 167 unsigned index_in_pt = virt_to_pt_index(vaddr);
133 168
134 169
135 struct x86_pte * pt; 170 struct x86_pte * pt;
136 if (pd[index_in_pd].present) 171 if (pd[index_in_pd].present)
137 { 172 {
138 pt = (struct x86_pte*) (pd[index_in_pd]. 173 pt = (struct x86_pte*) (pd[index_in_pd].pt_paddr << 12);
139 174
140 !! 175
141 !! 176
142 !! 177
143 !! 178
144 !! 179 if (pt[index_in_pt].present)
145 <<
146 if (! pt[index_in_pt].present) <<
147 sos_physmem_ref_physpage_at((sos_paddr <<
148 <<
149 <<
150 else <<
151 SOS_ASSERT_FATAL(FALSE); 180 SOS_ASSERT_FATAL(FALSE);
152 } 181 }
153 else 182 else
154 { 183 {
155 184
156 pt = (struct x86_pte*) sos_physmem_ref_p 185 pt = (struct x86_pte*) sos_physmem_ref_physpage_new(FALSE);
157 if (! pt) 186 if (! pt)
158 return -SOS_ENOMEM; 187 return -SOS_ENOMEM;
159 188
160 memset((void*)pt, 0x0, SOS_PAGE_SIZE); 189 memset((void*)pt, 0x0, SOS_PAGE_SIZE);
161 190
162 pd[index_in_pd].present = TRUE; 191 pd[index_in_pd].present = TRUE;
163 pd[index_in_pd].write = 1; 192 pd[index_in_pd].write = 1;
164 193
165 194
166 195
167 196
168 pd[index_in_pd].pt_paddr = ((sos_paddr_t 197 pd[index_in_pd].pt_paddr = ((sos_paddr_t)pt) >> 12;
169 } 198 }
170 199
171 200
172 201
173 pt[index_in_pt].present = 1; 202 pt[index_in_pt].present = 1;
174 pt[index_in_pt].write = 1; 203 pt[index_in_pt].write = 1;
175 204
176 205
177 206
178 pt[index_in_pt].user = 0; 207 pt[index_in_pt].user = 0;
179 pt[index_in_pt].paddr = ppage >> 12; 208 pt[index_in_pt].paddr = ppage >> 12;
180 209
>> 210
>> 211
>> 212 sos_physmem_inc_physpage_occupation((sos_paddr_t)pt);
>> 213
181 return SOS_OK; 214 return SOS_OK;
182 } 215 }
183 216
184 217
185 sos_ret_t sos_paging_setup(sos_paddr_t identit !! 218 sos_ret_t sos_paging_subsystem_setup(sos_paddr_t identity_mapping_base,
186 sos_paddr_t identit !! 219 sos_paddr_t identity_mapping_top)
187 { 220 {
188 221
189 struct x86_pdbr cr3; 222 struct x86_pdbr cr3;
190 223
191 224
192 struct x86_pde * pd 225 struct x86_pde * pd
193 = (struct x86_pde*) sos_physmem_ref_physpa 226 = (struct x86_pde*) sos_physmem_ref_physpage_new(FALSE);
194 227
195 228
196 sos_paddr_t paddr; 229 sos_paddr_t paddr;
197 230
198 231
199 232
200 memset((void*)pd, 233 memset((void*)pd,
201 0x0, 234 0x0,
202 SOS_PAGE_SIZE); 235 SOS_PAGE_SIZE);
203 236
204 237
205 for (paddr = identity_mapping_base ; 238 for (paddr = identity_mapping_base ;
206 paddr < identity_mapping_top ; 239 paddr < identity_mapping_top ;
207 paddr += SOS_PAGE_SIZE) 240 paddr += SOS_PAGE_SIZE)
208 { 241 {
209 if (paging_setup_map_helper(pd, paddr, p 242 if (paging_setup_map_helper(pd, paddr, paddr))
210 return -SOS_ENOMEM; 243 return -SOS_ENOMEM;
211 } 244 }
212 245
213 246
214 for (paddr = BIOS_N_VIDEO_START ; 247 for (paddr = BIOS_N_VIDEO_START ;
215 paddr < BIOS_N_VIDEO_END ; 248 paddr < BIOS_N_VIDEO_END ;
216 paddr += SOS_PAGE_SIZE) 249 paddr += SOS_PAGE_SIZE)
217 { 250 {
218 if (paging_setup_map_helper(pd, paddr, p 251 if (paging_setup_map_helper(pd, paddr, paddr))
219 return -SOS_ENOMEM; 252 return -SOS_ENOMEM;
220 } 253 }
221 254
222 255
223 256
224 pd[virt_to_pd_index(SOS_PAGING_MIRROR_VADDR) 257 pd[virt_to_pd_index(SOS_PAGING_MIRROR_VADDR)].present = TRUE;
225 pd[virt_to_pd_index(SOS_PAGING_MIRROR_VADDR) 258 pd[virt_to_pd_index(SOS_PAGING_MIRROR_VADDR)].write = 1;
226 pd[virt_to_pd_index(SOS_PAGING_MIRROR_VADDR) 259 pd[virt_to_pd_index(SOS_PAGING_MIRROR_VADDR)].user = 0;
227 pd[virt_to_pd_index(SOS_PAGING_MIRROR_VADDR) 260 pd[virt_to_pd_index(SOS_PAGING_MIRROR_VADDR)].pt_paddr
228 = ((sos_paddr_t)pd)>>12; 261 = ((sos_paddr_t)pd)>>12;
229 262
230 263
231 264
232 memset(& cr3, 0x0, sizeof(struct x86_pdbr)); 265 memset(& cr3, 0x0, sizeof(struct x86_pdbr));
233 cr3.pd_paddr = ((sos_paddr_t)pd) >> 12; 266 cr3.pd_paddr = ((sos_paddr_t)pd) >> 12;
234 267
235 268
236 269
237 270
238 asm volatile ("movl %0,%%cr3\n\t" 271 asm volatile ("movl %0,%%cr3\n\t"
239 "movl %%cr0,%%eax\n\t" 272 "movl %%cr0,%%eax\n\t"
240 "orl $0x80010000, %%eax\n\t" 273 "orl $0x80010000, %%eax\n\t"
241 "movl %%eax,%%cr0\n\t" 274 "movl %%eax,%%cr0\n\t"
242 "jmp 1f\n\t" 275 "jmp 1f\n\t"
243 "1:\n\t" 276 "1:\n\t"
244 "movl $2f, %%eax\n\t" 277 "movl $2f, %%eax\n\t"
245 "jmp *%%eax\n\t" 278 "jmp *%%eax\n\t"
246 "2:\n\t" ::"r"(cr3):"memory"," 279 "2:\n\t" ::"r"(cr3):"memory","eax");
247 280
248 281
249 282
250 283
251 284
252 285
253 286
254 287
255 288
256 return SOS_OK; 289 return SOS_OK;
257 } 290 }
258 291
259 292
260 293
261 294
262 sos_ret_t sos_paging_map(sos_paddr_t ppage_pad 295 sos_ret_t sos_paging_map(sos_paddr_t ppage_paddr,
263 sos_vaddr_t vpage_vad 296 sos_vaddr_t vpage_vaddr,
264 sos_bool_t is_user_pa 297 sos_bool_t is_user_page,
265 int flags) !! 298 sos_ui32_t flags)
266 { 299 {
267 300
268 301
269 unsigned index_in_pd = virt_to_pd_index(vpag 302 unsigned index_in_pd = virt_to_pd_index(vpage_vaddr);
270 unsigned index_in_pt = virt_to_pt_index(vpag 303 unsigned index_in_pt = virt_to_pt_index(vpage_vaddr);
271 304
272 305
273 struct x86_pde *pd = (struct x86_pde*) 306 struct x86_pde *pd = (struct x86_pde*)
274 (SOS_PAGING_MIRROR_VADDR 307 (SOS_PAGING_MIRROR_VADDR
275 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGI 308 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGING_MIRROR_VADDR));
276 309
277 310
278 struct x86_pte * pt = (struct x86_pte*) (SOS 311 struct x86_pte * pt = (struct x86_pte*) (SOS_PAGING_MIRROR_VADDR
279 + S 312 + SOS_PAGE_SIZE*index_in_pd);
280 313
>> 314 SOS_ASSERT_FATAL(SOS_IS_PAGE_ALIGNED(ppage_paddr));
>> 315 SOS_ASSERT_FATAL(SOS_IS_PAGE_ALIGNED(vpage_vaddr));
>> 316
>> 317
>> 318 flags &= ~SOS_VM_MAP_PROT_EXEC;
>> 319
281 320
282 if ((vpage_vaddr >= SOS_PAGING_MIRROR_VADDR) 321 if ((vpage_vaddr >= SOS_PAGING_MIRROR_VADDR)
283 && (vpage_vaddr < SOS_PAGING_MIRROR_VADD 322 && (vpage_vaddr < SOS_PAGING_MIRROR_VADDR + SOS_PAGING_MIRROR_SIZE))
284 return -SOS_EINVAL; 323 return -SOS_EINVAL;
285 324
286 325
287 if (! pd[index_in_pd].present) 326 if (! pd[index_in_pd].present)
288 { 327 {
>> 328 x86_pde_val_t u;
>> 329
289 330
290 sos_paddr_t pt_ppage 331 sos_paddr_t pt_ppage
291 = sos_physmem_ref_physpage_new(! (flag 332 = sos_physmem_ref_physpage_new(! (flags & SOS_VM_MAP_ATOMIC));
292 if (! pt_ppage) 333 if (! pt_ppage)
293 { 334 {
294 return -SOS_ENOMEM; 335 return -SOS_ENOMEM;
295 } 336 }
296 337
297 pd[index_in_pd].present = TRUE; !! 338
298 pd[index_in_pd].write = 1; !! 339 u.pde = (struct x86_pde){
299 !! 340 .present = TRUE,
300 pd[index_in_pd].user |= (is_user_pag !! 341 .write = 1,
301 pd[index_in_pd].pt_paddr = ((sos_paddr_t !! 342 .pt_paddr = ((sos_paddr_t)pt_ppage) >> 12
>> 343 };
>> 344
>> 345
>> 346 if (vpage_vaddr < SOS_PAGING_MIRROR_VADDR)
>> 347 {
>> 348
>> 349
>> 350
>> 351
>> 352 u.pde.user = 0;
>> 353
>> 354
>> 355 SOS_ASSERT_FATAL(SOS_OK ==
>> 356 sos_mm_context_synch_kernel_PDE(index_in_pd,
>> 357 u.ui32));
>> 358 }
>> 359 else
>> 360
>> 361
>> 362
>> 363 {
>> 364
>> 365
>> 366
>> 367
>> 368 u.pde.user = 1;
>> 369
>> 370
>> 371 pd[index_in_pd] = u.pde;
>> 372 }
302 373
303 374
304 375
305 376
306 377
307 378
308 invlpg(pt); 379 invlpg(pt);
309 380
310 381
311 memset((void*)pt, 0x0, SOS_PAGE_SIZE); 382 memset((void*)pt, 0x0, SOS_PAGE_SIZE);
312 } 383 }
313 384
314 !! 385
315 386
316 else if (! pt[index_in_pt].present) !! 387 if (! pt[index_in_pt].present)
317 sos_physmem_ref_physpage_at(pd[index_in_pd !! 388 sos_physmem_inc_physpage_occupation(pd[index_in_pd].pt_paddr << 12);
318 389
319 390
320 391
321 else 392 else
322 sos_physmem_unref_physpage(pt[index_in_pt] 393 sos_physmem_unref_physpage(pt[index_in_pt].paddr << 12);
323 394
324 395
325 pt[index_in_pt].present = TRUE; 396 pt[index_in_pt].present = TRUE;
326 pt[index_in_pt].write = (flags & SOS_VM_MA 397 pt[index_in_pt].write = (flags & SOS_VM_MAP_PROT_WRITE)?1:0;
327 pt[index_in_pt].user = (is_user_page)?1:0 398 pt[index_in_pt].user = (is_user_page)?1:0;
328 pt[index_in_pt].paddr = ppage_paddr >> 12; 399 pt[index_in_pt].paddr = ppage_paddr >> 12;
329 sos_physmem_ref_physpage_at(ppage_paddr); 400 sos_physmem_ref_physpage_at(ppage_paddr);
330 401
>> 402
331 403
332 404
333 405
334 406
335 407
336 invlpg(vpage_vaddr); 408 invlpg(vpage_vaddr);
337 409
338 return SOS_OK; 410 return SOS_OK;
339 } 411 }
340 412
341 413
342 sos_ret_t sos_paging_unmap(sos_vaddr_t vpage_v 414 sos_ret_t sos_paging_unmap(sos_vaddr_t vpage_vaddr)
343 { 415 {
344 sos_ret_t pt_unref_retval; !! 416 sos_ret_t pt_dec_occupation_retval;
345 417
346 418
347 419
348 unsigned index_in_pd = virt_to_pd_index(vpag 420 unsigned index_in_pd = virt_to_pd_index(vpage_vaddr);
349 unsigned index_in_pt = virt_to_pt_index(vpag 421 unsigned index_in_pt = virt_to_pt_index(vpage_vaddr);
350 422
351 423
352 struct x86_pde *pd = (struct x86_pde*) 424 struct x86_pde *pd = (struct x86_pde*)
353 (SOS_PAGING_MIRROR_VADDR 425 (SOS_PAGING_MIRROR_VADDR
354 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGI 426 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGING_MIRROR_VADDR));
355 427
356 428
357 struct x86_pte * pt = (struct x86_pte*) (SOS 429 struct x86_pte * pt = (struct x86_pte*) (SOS_PAGING_MIRROR_VADDR
358 + S 430 + SOS_PAGE_SIZE*index_in_pd);
359 431
>> 432 SOS_ASSERT_FATAL(SOS_IS_PAGE_ALIGNED(vpage_vaddr));
>> 433
360 434
361 if (! pd[index_in_pd].present) 435 if (! pd[index_in_pd].present)
362 return -SOS_EINVAL; 436 return -SOS_EINVAL;
363 if (! pt[index_in_pt].present) 437 if (! pt[index_in_pt].present)
364 return -SOS_EINVAL; 438 return -SOS_EINVAL;
365 439
366 440
367 if ((vpage_vaddr >= SOS_PAGING_MIRROR_VADDR) 441 if ((vpage_vaddr >= SOS_PAGING_MIRROR_VADDR)
368 && (vpage_vaddr < SOS_PAGING_MIRROR_VADD 442 && (vpage_vaddr < SOS_PAGING_MIRROR_VADDR + SOS_PAGING_MIRROR_SIZE))
369 return -SOS_EINVAL; 443 return -SOS_EINVAL;
370 444
371 445
372 sos_physmem_unref_physpage(pt[index_in_pt].p 446 sos_physmem_unref_physpage(pt[index_in_pt].paddr << 12);
373 447
374 448
375 memset(pt + index_in_pt, 0x0, sizeof(struct 449 memset(pt + index_in_pt, 0x0, sizeof(struct x86_pte));
376 450
377 451
378 invlpg(vpage_vaddr); 452 invlpg(vpage_vaddr);
379 453
380 454
381 pt_unref_retval = sos_physmem_unref_physpage !! 455 pt_dec_occupation_retval
382 SOS_ASSERT_FATAL(pt_unref_retval >= 0); !! 456 = sos_physmem_dec_physpage_occupation(pd[index_in_pd].pt_paddr << 12);
383 if (pt_unref_retval > 0) !! 457 SOS_ASSERT_FATAL(pt_dec_occupation_retval >= 0);
>> 458 if (pt_dec_occupation_retval > 0)
384 459
385 { 460 {
386 !! 461 x86_pde_val_t u;
387 memset(pd + index_in_pd, 0x0, sizeof(str !! 462
>> 463
>> 464
>> 465
>> 466
>> 467 sos_physmem_unref_physpage(pd[index_in_pd].pt_paddr << 12);
>> 468
>> 469
>> 470
>> 471
>> 472
>> 473
>> 474
>> 475 u.ui32 = 0;
>> 476
>> 477
>> 478 if (vpage_vaddr < SOS_PAGING_MIRROR_VADDR)
>> 479 {
>> 480
>> 481 SOS_ASSERT_FATAL(SOS_OK ==
>> 482 sos_mm_context_synch_kernel_PDE(index_in_pd,
>> 483 u.ui32));
>> 484 }
>> 485 else
>> 486
>> 487
>> 488
>> 489 {
>> 490
>> 491
>> 492 pd[index_in_pd] = u.pde;
>> 493 }
388 494
389 495
390 invlpg(pt); 496 invlpg(pt);
391 } 497 }
392 498
393 return SOS_OK; 499 return SOS_OK;
394 } 500 }
395 501
396 502
397 int sos_paging_get_prot(sos_vaddr_t vaddr) !! 503 sos_ret_t sos_paging_unmap_interval(sos_vaddr_t vaddr,
>> 504 sos_size_t size)
398 { 505 {
399 int retval; !! 506 sos_ret_t retval = 0;
>> 507
>> 508 if (! SOS_IS_PAGE_ALIGNED(vaddr))
>> 509 return -SOS_EINVAL;
>> 510 if (! SOS_IS_PAGE_ALIGNED(size))
>> 511 return -SOS_EINVAL;
>> 512
>> 513 for ( ;
>> 514 size >= SOS_PAGE_SIZE ;
>> 515 vaddr += SOS_PAGE_SIZE, size -= SOS_PAGE_SIZE)
>> 516 if (SOS_OK == sos_paging_unmap(vaddr))
>> 517 retval += SOS_PAGE_SIZE;
>> 518
>> 519 return retval;
>> 520 }
>> 521
>> 522
>> 523 sos_ui32_t sos_paging_get_prot(sos_vaddr_t vaddr)
>> 524 {
>> 525 sos_ui32_t retval;
400 526
401 527
402 528
403 unsigned index_in_pd = virt_to_pd_index(vadd 529 unsigned index_in_pd = virt_to_pd_index(vaddr);
404 unsigned index_in_pt = virt_to_pt_index(vadd 530 unsigned index_in_pt = virt_to_pt_index(vaddr);
405 531
406 532
407 struct x86_pde *pd = (struct x86_pde*) 533 struct x86_pde *pd = (struct x86_pde*)
408 (SOS_PAGING_MIRROR_VADDR 534 (SOS_PAGING_MIRROR_VADDR
409 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGI 535 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGING_MIRROR_VADDR));
410 536
411 537
412 struct x86_pte * pt = (struct x86_pte*) (SOS 538 struct x86_pte * pt = (struct x86_pte*) (SOS_PAGING_MIRROR_VADDR
413 + S 539 + SOS_PAGE_SIZE*index_in_pd);
414 540
415 541
416 if (! pd[index_in_pd].present) 542 if (! pd[index_in_pd].present)
417 return SOS_VM_MAP_PROT_NONE; 543 return SOS_VM_MAP_PROT_NONE;
418 if (! pt[index_in_pt].present) 544 if (! pt[index_in_pt].present)
419 return SOS_VM_MAP_PROT_NONE; 545 return SOS_VM_MAP_PROT_NONE;
420 546
421 547
422 retval = SOS_VM_MAP_PROT_READ; 548 retval = SOS_VM_MAP_PROT_READ;
423 if (pd[index_in_pd].write && pt[index_in_pt] 549 if (pd[index_in_pd].write && pt[index_in_pt].write)
424 retval |= SOS_VM_MAP_PROT_WRITE; 550 retval |= SOS_VM_MAP_PROT_WRITE;
425 551
426 return retval; 552 return retval;
427 } 553 }
428 554
429 555
>> 556 sos_ret_t sos_paging_set_prot(sos_vaddr_t vaddr,
>> 557 sos_ui32_t new_prot)
>> 558 {
>> 559
>> 560
>> 561 unsigned index_in_pd = virt_to_pd_index(vaddr);
>> 562 unsigned index_in_pt = virt_to_pt_index(vaddr);
>> 563
>> 564
>> 565 struct x86_pde *pd = (struct x86_pde*)
>> 566 (SOS_PAGING_MIRROR_VADDR
>> 567 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGING_MIRROR_VADDR));
>> 568
>> 569
>> 570 struct x86_pte * pt = (struct x86_pte*) (SOS_PAGING_MIRROR_VADDR
>> 571 + SOS_PAGE_SIZE*index_in_pd);
>> 572
>> 573
>> 574 new_prot &= ~SOS_VM_MAP_PROT_EXEC;
>> 575
>> 576
>> 577 if (new_prot & ~(SOS_VM_MAP_PROT_READ | SOS_VM_MAP_PROT_WRITE))
>> 578 return -SOS_EINVAL;
>> 579 if (! (new_prot & SOS_VM_MAP_PROT_READ))
>> 580
>> 581 return -SOS_ENOSUP;
>> 582
>> 583
>> 584 if (! pd[index_in_pd].present)
>> 585 return -SOS_EINVAL;
>> 586 if (! pt[index_in_pt].present)
>> 587 return -SOS_EINVAL;
>> 588
>> 589
>> 590 pt[index_in_pt].write = ((new_prot & SOS_VM_MAP_PROT_WRITE) != 0);
>> 591 invlpg(vaddr);
>> 592
>> 593 return SOS_OK;
>> 594 }
>> 595
>> 596
>> 597 sos_ret_t sos_paging_set_prot_of_interval(sos_vaddr_t vaddr,
>> 598 sos_size_t size,
>> 599 sos_ui32_t new_prot)
>> 600 {
>> 601 if (! SOS_IS_PAGE_ALIGNED(vaddr))
>> 602 return -SOS_EINVAL;
>> 603 if (! SOS_IS_PAGE_ALIGNED(size))
>> 604 return -SOS_EINVAL;
>> 605
>> 606 for ( ; size >= SOS_PAGE_SIZE ; vaddr += SOS_PAGE_SIZE, size -= SOS_PAGE_SIZE)
>> 607 sos_paging_set_prot(vaddr, new_prot);
>> 608
>> 609 return SOS_OK;
>> 610 }
>> 611
>> 612
>> 613 sos_bool_t sos_paging_is_dirty(sos_vaddr_t vaddr)
>> 614 {
>> 615
>> 616
>> 617 unsigned index_in_pd = virt_to_pd_index(vaddr);
>> 618 unsigned index_in_pt = virt_to_pt_index(vaddr);
>> 619
>> 620
>> 621 struct x86_pde *pd = (struct x86_pde*)
>> 622 (SOS_PAGING_MIRROR_VADDR
>> 623 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGING_MIRROR_VADDR));
>> 624
>> 625
>> 626 struct x86_pte * pt = (struct x86_pte*) (SOS_PAGING_MIRROR_VADDR
>> 627 + SOS_PAGE_SIZE*index_in_pd);
>> 628
>> 629
>> 630 if (! pd[index_in_pd].present)
>> 631 return FALSE;
>> 632 if (! pt[index_in_pt].present)
>> 633 return FALSE;
>> 634
>> 635 return (pt[index_in_pt].dirty != 0);
>> 636 }
>> 637
>> 638
>> 639 sos_ret_t sos_paging_set_dirty(sos_vaddr_t vaddr,
>> 640 sos_bool_t is_dirty)
>> 641 {
>> 642
>> 643
>> 644 unsigned index_in_pd = virt_to_pd_index(vaddr);
>> 645 unsigned index_in_pt = virt_to_pt_index(vaddr);
>> 646
>> 647
>> 648 struct x86_pde *pd = (struct x86_pde*)
>> 649 (SOS_PAGING_MIRROR_VADDR
>> 650 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGING_MIRROR_VADDR));
>> 651
>> 652
>> 653 struct x86_pte * pt = (struct x86_pte*) (SOS_PAGING_MIRROR_VADDR
>> 654 + SOS_PAGE_SIZE*index_in_pd);
>> 655
>> 656
>> 657 if (! pd[index_in_pd].present)
>> 658 return -SOS_EFAULT;
>> 659 if (! pt[index_in_pt].present)
>> 660 return -SOS_EFAULT;
>> 661
>> 662 pt[index_in_pt].dirty = is_dirty;
>> 663 return SOS_OK;
>> 664 }
>> 665
>> 666
430 sos_paddr_t sos_paging_get_paddr(sos_vaddr_t v 667 sos_paddr_t sos_paging_get_paddr(sos_vaddr_t vaddr)
431 { 668 {
432 669
433 670
434 unsigned index_in_pd = virt_to_pd_index(vadd 671 unsigned index_in_pd = virt_to_pd_index(vaddr);
435 unsigned index_in_pt = virt_to_pt_index(vadd 672 unsigned index_in_pt = virt_to_pt_index(vaddr);
436 unsigned offset_in_page = virt_to_page_offse 673 unsigned offset_in_page = virt_to_page_offset(vaddr);
437 674
438 675
439 struct x86_pde *pd = (struct x86_pde*) 676 struct x86_pde *pd = (struct x86_pde*)
440 (SOS_PAGING_MIRROR_VADDR 677 (SOS_PAGING_MIRROR_VADDR
441 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGI 678 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGING_MIRROR_VADDR));
442 679
443 680
444 struct x86_pte * pt = (struct x86_pte*) (SOS 681 struct x86_pte * pt = (struct x86_pte*) (SOS_PAGING_MIRROR_VADDR
445 + S 682 + SOS_PAGE_SIZE*index_in_pd);
446 683
447 684
448 if (! pd[index_in_pd].present) 685 if (! pd[index_in_pd].present)
449 return (sos_paddr_t)NULL; 686 return (sos_paddr_t)NULL;
450 if (! pt[index_in_pt].present) 687 if (! pt[index_in_pt].present)
451 return (sos_paddr_t)NULL; 688 return (sos_paddr_t)NULL;
452 689
453 return (pt[index_in_pt].paddr << 12) + offse 690 return (pt[index_in_pt].paddr << 12) + offset_in_page;
454 } 691 }
455 692
>> 693
>> 694
>> 695
>> 696
>> 697
>> 698
>> 699 sos_paddr_t sos_paging_get_current_PD_paddr(void)
>> 700 {
>> 701 struct x86_pdbr pdbr;
>> 702 asm volatile("movl %%cr3, %0\n": "=r"(pdbr));
>> 703 return (pdbr.pd_paddr << 12);
>> 704 }
>> 705
>> 706
>> 707 sos_ret_t sos_paging_set_current_PD_paddr(sos_paddr_t paddr_PD)
>> 708 {
>> 709 struct x86_pdbr pdbr;
>> 710
>> 711 SOS_ASSERT_FATAL(paddr_PD != 0);
>> 712 SOS_ASSERT_FATAL(SOS_IS_PAGE_ALIGNED(paddr_PD));
>> 713
>> 714
>> 715 memset(& pdbr, 0x0, sizeof(struct x86_pdbr));
>> 716 pdbr.pd_paddr = (paddr_PD >> 12);
>> 717
>> 718
>> 719 asm volatile ("movl %0,%%cr3\n" ::"r"(pdbr));
>> 720
>> 721 return SOS_OK;
>> 722 }
>> 723
>> 724
>> 725 sos_ret_t sos_paging_dispose(sos_vaddr_t vaddr_PD)
>> 726 {
>> 727 x86_pde_val_t *pd = (x86_pde_val_t*) vaddr_PD;
>> 728 x86_pte_val_t *pt;
>> 729 int index_in_pd;
>> 730
>> 731
>> 732
>> 733 pt = (x86_pte_val_t *)sos_kmem_vmm_alloc(1, 0);
>> 734 if (! pt)
>> 735 return -SOS_ENOMEM;
>> 736
>> 737
>> 738
>> 739
>> 740 for (index_in_pd = (SOS_PAGING_BASE_USER_ADDRESS >> 22) ;
>> 741 index_in_pd < 1024 ;
>> 742
>> 743
>> 744 index_in_pd ++)
>> 745 {
>> 746 sos_paddr_t paddr_pt = (pd[index_in_pd].pde.pt_paddr << 12);
>> 747 int index_in_pt;
>> 748
>> 749
>> 750 if (! pd[index_in_pd].pde.present)
>> 751 {
>> 752 pd[index_in_pd].ui32 = 0;
>> 753 continue;
>> 754 }
>> 755
>> 756
>> 757 SOS_ASSERT_FATAL(SOS_OK
>> 758 == sos_paging_map(paddr_pt,
>> 759 (sos_vaddr_t)pt, FALSE,
>> 760 SOS_VM_MAP_PROT_READ
>> 761 | SOS_VM_MAP_PROT_WRITE));
>> 762
>> 763
>> 764 for (index_in_pt = 0 ; index_in_pt < 1024 ; index_in_pt ++)
>> 765 {
>> 766
>> 767 if (! pt[index_in_pt].pte.present)
>> 768 {
>> 769 pt[index_in_pt].ui32 = 0;
>> 770 continue;
>> 771 }
>> 772
>> 773
>> 774 sos_physmem_unref_physpage(pt[index_in_pt].pte.paddr << 12);
>> 775
>> 776
>> 777 sos_physmem_dec_physpage_occupation(paddr_pt);
>> 778
>> 779
>> 780 pt[index_in_pt].ui32 = 0;
>> 781 }
>> 782
>> 783
>> 784 SOS_ASSERT_FATAL(SOS_OK == sos_paging_unmap((sos_vaddr_t)pt));
>> 785
>> 786
>> 787 pd[index_in_pd].ui32 = 0;
>> 788
>> 789
>> 790 sos_physmem_unref_physpage(paddr_pt);
>> 791 }
>> 792
>> 793
>> 794 SOS_ASSERT_FATAL(SOS_OK == sos_kmem_vmm_free((sos_vaddr_t)pt));
>> 795
>> 796 return SOS_OK;
>> 797 }
>> 798
>> 799
>> 800 sos_ret_t sos_paging_copy_kernel_space(sos_vaddr_t dest_vaddr_PD,
>> 801 sos_vaddr_t src_vaddr_PD)
>> 802 {
>> 803 x86_pde_val_t *src_pd = (x86_pde_val_t*) src_vaddr_PD;
>> 804 x86_pde_val_t *dest_pd = (x86_pde_val_t*) dest_vaddr_PD;
>> 805 sos_paddr_t dest_paddr_PD = sos_paging_get_paddr(dest_vaddr_PD);
>> 806 x86_pde_val_t mirror_pde;
>> 807 int index_in_pd;
>> 808
>> 809
>> 810 memset((void*)dest_vaddr_PD, 0x0, SOS_PAGE_SIZE);
>> 811
>> 812
>> 813
>> 814 for (index_in_pd = 0 ;
>> 815 index_in_pd < (SOS_PAGING_MIRROR_VADDR >> 22) ;
>> 816
>> 817
>> 818 index_in_pd ++)
>> 819 {
>> 820
>> 821 dest_pd[index_in_pd].ui32 = src_pd[index_in_pd].ui32;
>> 822
>> 823
>> 824
>> 825
>> 826
>> 827
>> 828
>> 829
>> 830
>> 831 }
>> 832
>> 833
>> 834 mirror_pde.ui32 = 0;
>> 835 mirror_pde.pde.present = TRUE;
>> 836 mirror_pde.pde.write = 1;
>> 837 mirror_pde.pde.user = 0;
>> 838 mirror_pde.pde.pt_paddr = (dest_paddr_PD >> 12);
>> 839 dest_pd[SOS_PAGING_MIRROR_VADDR >> 22].ui32 = mirror_pde.ui32;
>> 840
>> 841 return SOS_OK;
>> 842 }
>> 843
>> 844
>> 845 sos_ret_t sos_paging_copy_user_space(sos_vaddr_t dest_vaddr_PD,
>> 846 sos_vaddr_t src_vaddr_PD)
>> 847 {
>> 848 x86_pde_val_t *src_pd = (x86_pde_val_t*) src_vaddr_PD;
>> 849 x86_pde_val_t *dest_pd = (x86_pde_val_t*) dest_vaddr_PD;
>> 850 x86_pte_val_t *tmp_src_pt, *tmp_dest_pt;
>> 851 int index_in_pd;
>> 852
>> 853
>> 854
>> 855 tmp_src_pt = (x86_pte_val_t *)sos_kmem_vmm_alloc(1, 0);
>> 856 if (! tmp_src_pt)
>> 857 return -SOS_ENOMEM;
>> 858
>> 859 tmp_dest_pt = (x86_pte_val_t *)sos_kmem_vmm_alloc(1, 0);
>> 860 if (! tmp_dest_pt)
>> 861 {
>> 862 sos_kmem_vmm_free((sos_vaddr_t)tmp_dest_pt);
>> 863 return -SOS_ENOMEM;
>> 864 }
>> 865
>> 866
>> 867 for (index_in_pd = (SOS_PAGING_BASE_USER_ADDRESS >> 22) ;
>> 868 index_in_pd < 1024 ;
>> 869
>> 870
>> 871 index_in_pd ++)
>> 872 {
>> 873 sos_paddr_t paddr_dest_pt;
>> 874 int index_in_pt;
>> 875
>> 876
>> 877
>> 878
>> 879
>> 880
>> 881 dest_pd[index_in_pd].ui32 = src_pd[index_in_pd].ui32;
>> 882
>> 883
>> 884 if (! src_pd[index_in_pd].pde.present)
>> 885 continue;
>> 886
>> 887
>> 888 paddr_dest_pt = sos_physmem_ref_physpage_new(TRUE);
>> 889 if (NULL == (void*)paddr_dest_pt)
>> 890 {
>> 891 sos_paging_dispose((sos_vaddr_t)dest_vaddr_PD);
>> 892
>> 893
>> 894 sos_kmem_vmm_free((sos_vaddr_t)tmp_src_pt);
>> 895 sos_kmem_vmm_free((sos_vaddr_t)tmp_dest_pt);
>> 896 return -SOS_ENOMEM;
>> 897 }
>> 898
>> 899
>> 900 SOS_ASSERT_FATAL(SOS_OK
>> 901 == sos_paging_map(src_pd[index_in_pd].pde.pt_paddr << 12,
>> 902 (sos_vaddr_t)tmp_src_pt, FALSE,
>> 903 SOS_VM_MAP_PROT_READ));
>> 904 SOS_ASSERT_FATAL(SOS_OK
>> 905 == sos_paging_map(paddr_dest_pt,
>> 906 (sos_vaddr_t)tmp_dest_pt, FALSE,
>> 907 SOS_VM_MAP_PROT_READ
>> 908 | SOS_VM_MAP_PROT_WRITE));
>> 909
>> 910
>> 911
>> 912 for (index_in_pt = 0 ; index_in_pt < 1024 ; index_in_pt ++)
>> 913 {
>> 914
>> 915 tmp_dest_pt[index_in_pt].ui32 = tmp_src_pt[index_in_pt].ui32;
>> 916
>> 917
>> 918 if (! tmp_dest_pt[index_in_pt].pte.present)
>> 919 continue;
>> 920
>> 921
>> 922 tmp_dest_pt[index_in_pt].pte.accessed = 0;
>> 923 tmp_dest_pt[index_in_pt].pte.dirty = 0;
>> 924
>> 925
>> 926 sos_physmem_ref_physpage_at(tmp_src_pt[index_in_pt].pte.paddr << 12);
>> 927
>> 928
>> 929 sos_physmem_inc_physpage_occupation(paddr_dest_pt);
>> 930 }
>> 931
>> 932
>> 933 SOS_ASSERT_FATAL(SOS_OK == sos_paging_unmap((sos_vaddr_t)tmp_src_pt));
>> 934 SOS_ASSERT_FATAL(SOS_OK == sos_paging_unmap((sos_vaddr_t)tmp_dest_pt));
>> 935
>> 936
>> 937 dest_pd[index_in_pd].pde.pt_paddr = (paddr_dest_pt >> 12);
>> 938
>> 939
>> 940 dest_pd[index_in_pd].pde.accessed = 0;
>> 941 }
>> 942
>> 943
>> 944
>> 945 SOS_ASSERT_FATAL(SOS_OK == sos_kmem_vmm_free((sos_vaddr_t)tmp_src_pt));
>> 946 SOS_ASSERT_FATAL(SOS_OK == sos_kmem_vmm_free((sos_vaddr_t)tmp_dest_pt));
>> 947
>> 948 return SOS_OK;
>> 949 }
>> 950
>> 951
>> 952 sos_ret_t sos_paging_prepare_COW(sos_uaddr_t base_address,
>> 953 sos_size_t length)
>> 954 {
>> 955 SOS_ASSERT_FATAL(SOS_IS_PAGE_ALIGNED(base_address));
>> 956 SOS_ASSERT_FATAL(SOS_IS_PAGE_ALIGNED(length));
>> 957 SOS_ASSERT_FATAL(SOS_PAGING_BASE_USER_ADDRESS <= base_address);
>> 958
>> 959
>> 960
>> 961 for ( ;
>> 962 length > 0 ;
>> 963 length -= SOS_PAGE_SIZE, base_address += SOS_PAGE_SIZE)
>> 964 {
>> 965 sos_paging_set_prot(base_address,
>> 966 SOS_VM_MAP_PROT_READ);
>> 967 }
>> 968
>> 969 return SOS_OK;
>> 970 }
>> 971
>> 972
>> 973 sos_ret_t sos_paging_try_resolve_COW(sos_uaddr_t uaddr)
>> 974 {
>> 975 sos_ret_t refcnt;
>> 976
>> 977
>> 978
>> 979 unsigned index_in_pd = virt_to_pd_index(uaddr);
>> 980 unsigned index_in_pt = virt_to_pt_index(uaddr);
>> 981
>> 982
>> 983 struct x86_pde *pd = (struct x86_pde*)
>> 984 (SOS_PAGING_MIRROR_VADDR
>> 985 + SOS_PAGE_SIZE*virt_to_pd_index(SOS_PAGING_MIRROR_VADDR));
>> 986
>> 987
>> 988 struct x86_pte * pt = (struct x86_pte*) (SOS_PAGING_MIRROR_VADDR
>> 989 + SOS_PAGE_SIZE*index_in_pd);
>> 990
>> 991
>> 992 if (! pd[index_in_pd].present)
>> 993 return -SOS_EFAULT;
>> 994 if (! pt[index_in_pt].present)
>> 995 return -SOS_EFAULT;
>> 996
>> 997
>> 998 if (! pd[index_in_pd].write)
>> 999 return -SOS_EFAULT;
>> 1000
>> 1001
>> 1002
>> 1003 SOS_ASSERT_FATAL(! pt[index_in_pt].write);
>> 1004
>> 1005
>> 1006
>> 1007 refcnt = sos_physmem_get_physpage_refcount(pt[index_in_pt].paddr << 12);
>> 1008 SOS_ASSERT_FATAL(refcnt > 0);
>> 1009
>> 1010 if (refcnt == 1)
>> 1011 {
>> 1012
>> 1013
>> 1014 pt[index_in_pt].write = 1;
>> 1015 invlpg(pt[index_in_pt].paddr << 12);
>> 1016 }
>> 1017
>> 1018
>> 1019 else
>> 1020 {
>> 1021 sos_paddr_t new_ppage;
>> 1022 sos_vaddr_t vpage_src, tmp_dest;
>> 1023
>> 1024
>> 1025
>> 1026
>> 1027 tmp_dest = sos_kmem_vmm_alloc(1, SOS_KMEM_VMM_MAP);
>> 1028 if (! tmp_dest)
>> 1029 return -SOS_ENOMEM;
>> 1030
>> 1031
>> 1032 vpage_src = SOS_PAGE_ALIGN_INF(uaddr);
>> 1033 memcpy((void*)tmp_dest, (void*)vpage_src, SOS_PAGE_SIZE);
>> 1034
>> 1035
>> 1036
>> 1037
>> 1038 new_ppage = sos_paging_get_paddr(tmp_dest);
>> 1039 SOS_ASSERT_FATAL(new_ppage != (sos_vaddr_t)NULL);
>> 1040 if (SOS_OK != sos_paging_map(new_ppage, vpage_src,
>> 1041 TRUE,
>> 1042 SOS_VM_MAP_PROT_READ
>> 1043 | SOS_VM_MAP_PROT_WRITE))
>> 1044 {
>> 1045 sos_kmem_vmm_free(tmp_dest);
>> 1046 return -SOS_ENOMEM;
>> 1047 }
>> 1048
>> 1049
>> 1050
>> 1051 SOS_ASSERT_FATAL(SOS_OK == sos_kmem_vmm_free(tmp_dest));
>> 1052 }
>> 1053
>> 1054
>> 1055 return SOS_OK;
>> 1056 }