Attachment 'mc2_litmus-rt_2015.1-rtns18.patch'
Download 1 diff --git arch/arm/boot/compressed/Makefile arch/arm/boot/compressed/Makefile
2 index 6e1fb2b..e2284fe 100644
3 --- arch/arm/boot/compressed/Makefile
4 +++ arch/arm/boot/compressed/Makefile
5 @@ -107,6 +107,7 @@ ifeq ($(CONFIG_FUNCTION_TRACER),y)
6 ORIG_CFLAGS := $(KBUILD_CFLAGS)
7 KBUILD_CFLAGS = $(subst -pg, , $(ORIG_CFLAGS))
8 endif
9 +KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
10
11 ccflags-y := -fpic -mno-single-pic-base -fno-builtin -I$(obj)
12 asflags-y := -DZIMAGE
13 diff --git arch/arm/include/asm/unistd.h arch/arm/include/asm/unistd.h
14 index 0c462a9..7197bbe 100644
15 --- arch/arm/include/asm/unistd.h
16 +++ arch/arm/include/asm/unistd.h
17 @@ -19,7 +19,8 @@
18 * This may need to be greater than __NR_last_syscall+1 in order to
19 * account for the padding in the syscall table
20 */
21 -#define __NR_syscalls (388 + NR_litmus_syscalls)
22 +#define __NR_syscalls (388 + NR_litmus_syscalls + 3)
23 +
24
25 /*
26 * *NOTE*: This is a ghost syscall private to the kernel. Only the
27 diff --git arch/arm/kernel/calls.S arch/arm/kernel/calls.S
28 index f4738a8..04cc83b 100644
29 --- arch/arm/kernel/calls.S
30 +++ arch/arm/kernel/calls.S
31 @@ -409,6 +409,15 @@
32 CALL(sys_wait_for_ts_release)
33 CALL(sys_release_ts)
34 CALL(sys_null_call)
35 +/* 400 */ CALL(sys_get_current_budget)
36 + CALL(sys_reservation_create)
37 + CALL(sys_reservation_destroy)
38 + CALL(sys_set_mc2_task_param)
39 + CALL(sys_set_page_color)
40 +/* 405 */ CALL(sys_test_call)
41 + CALL(sys_run_test)
42 + CALL(sys_lock_buffer)
43 + CALL(sys_recolor_mem)
44
45 #ifndef syscalls_counted
46 .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls
47 diff --git arch/arm/kernel/irq.c arch/arm/kernel/irq.c
48 index 350f188..a9ba6e5 100644
49 --- arch/arm/kernel/irq.c
50 +++ arch/arm/kernel/irq.c
51 @@ -44,6 +44,9 @@
52 #include <asm/mach/irq.h>
53 #include <asm/mach/time.h>
54
55 +#include <litmus/cache_proc.h>
56 +#include <litmus/litmus.h>
57 +
58 unsigned long irq_err_count;
59
60 int arch_show_interrupts(struct seq_file *p, int prec)
61 @@ -66,7 +69,9 @@ int arch_show_interrupts(struct seq_file *p, int prec)
62 */
63 void handle_IRQ(unsigned int irq, struct pt_regs *regs)
64 {
65 + enter_irq_mode();
66 __handle_domain_irq(NULL, irq, false, regs);
67 + exit_irq_mode();
68 }
69
70 /*
71 diff --git arch/arm/mm/cache-l2x0.c arch/arm/mm/cache-l2x0.c
72 index e309c8f..969da4a 100644
73 --- arch/arm/mm/cache-l2x0.c
74 +++ arch/arm/mm/cache-l2x0.c
75 @@ -33,6 +33,8 @@
76 #include "cache-tauros3.h"
77 #include "cache-aurora-l2.h"
78
79 +#include <litmus/cache_proc.h>
80 +
81 struct l2c_init_data {
82 const char *type;
83 unsigned way_size_0;
84 @@ -651,6 +653,11 @@ static void __init l2c310_enable(void __iomem *base, u32 aux, unsigned num_lock)
85 */
86 aux |= L310_AUX_CTRL_NS_LOCKDOWN;
87
88 + /*
89 + * Always enable non-secure interrupt access control registers
90 + */
91 + aux |= L220_AUX_CTRL_NS_INT_CTRL;
92 +
93 l2c_enable(base, aux, num_lock);
94
95 /* Read back resulting AUX_CTRL value as it could have been altered. */
96 @@ -726,7 +733,6 @@ static void __init l2c310_fixup(void __iomem *base, u32 cache_id,
97
98 if (n) {
99 unsigned i;
100 -
101 pr_info("L2C-310 errat%s", n > 1 ? "a" : "um");
102 for (i = 0; i < n; i++)
103 pr_cont(" %s", errata[i]);
104 @@ -774,6 +780,11 @@ static const struct l2c_init_data l2c310_init_fns __initconst = {
105 },
106 };
107
108 +void l2c310_flush_all(void)
109 +{
110 + l2c210_flush_all();
111 +};
112 +
113 static int __init __l2c_init(const struct l2c_init_data *data,
114 u32 aux_val, u32 aux_mask, u32 cache_id)
115 {
116 @@ -876,6 +887,8 @@ static int __init __l2c_init(const struct l2c_init_data *data,
117 pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n",
118 data->type, cache_id, aux);
119
120 + litmus_setup_lockdown(l2x0_base, cache_id);
121 +
122 return 0;
123 }
124
125 diff --git arch/x86/syscalls/syscall_32.tbl arch/x86/syscalls/syscall_32.tbl
126 index 34680a5..6b80d98 100644
127 --- arch/x86/syscalls/syscall_32.tbl
128 +++ arch/x86/syscalls/syscall_32.tbl
129 @@ -377,3 +377,12 @@
130 368 i386 wait_for_ts_release sys_wait_for_ts_release
131 369 i386 release_ts sys_release_ts
132 370 i386 null_call sys_null_call
133 +371 i386 get_current_budget sys_get_current_budget
134 +372 i386 reservation_create sys_reservation_create
135 +373 i386 reservation_destroy sys_reservation_destroy
136 +374 i386 set_mc2_task_param sys_set_mc2_task_param
137 +375 i386 set_page_color sys_set_page_color
138 +376 i386 test_call sys_test_call
139 +377 i386 run_test sys_run_test
140 +378 i386 lock_buffer sys_lock_buffer
141 +379 i386 recolor_mem sys_recolor_mem
142 diff --git arch/x86/syscalls/syscall_64.tbl arch/x86/syscalls/syscall_64.tbl
143 index cbd1b6b..05aa913 100644
144 --- arch/x86/syscalls/syscall_64.tbl
145 +++ arch/x86/syscalls/syscall_64.tbl
146 @@ -342,6 +342,15 @@
147 360 common wait_for_ts_release sys_wait_for_ts_release
148 361 common release_ts sys_release_ts
149 362 common null_call sys_null_call
150 +363 common get_current_budget sys_get_current_budget
151 +364 common reservation_create sys_reservation_create
152 +365 common reservation_destroy sys_reservation_destroy
153 +366 common set_mc2_task_param sys_set_mc2_task_param
154 +367 common set_page_color sys_set_page_color
155 +368 common test_call sys_test_call
156 +369 common run_test sys_run_test
157 +370 common lock_buffer sys_lock_buffer
158 +371 common recolor_mem sys_recolor_mem
159
160 #
161 # x32-specific system call numbers start at 512 to avoid cache impact
162 diff --git drivers/media/usb/uvc/uvc_video.c drivers/media/usb/uvc/uvc_video.c
163 index 20ccc9d..9dd652c 100644
164 --- drivers/media/usb/uvc/uvc_video.c
165 +++ drivers/media/usb/uvc/uvc_video.c
166 @@ -26,6 +26,16 @@
167
168 #include "uvcvideo.h"
169
170 +#include <litmus/litmus.h>
171 +#include <litmus/trace.h>
172 +
173 +// default behavior is using Level-C/OS
174 +#ifdef CONFIG_UVC_ALLOC_LEVEL_AB
175 +#define UVC_FLAG (GFP_COLOR)
176 +#else
177 +#define UVC_FLAG (0)
178 +#endif
179 +
180 /* ------------------------------------------------------------------------
181 * UVC Controls
182 */
183 @@ -167,7 +177,7 @@ static int uvc_get_video_ctrl(struct uvc_streaming *stream,
184 query == UVC_GET_DEF)
185 return -EIO;
186
187 - data = kmalloc(size, GFP_KERNEL);
188 + data = kmalloc(size, GFP_KERNEL|UVC_FLAG);
189 if (data == NULL)
190 return -ENOMEM;
191
192 @@ -251,7 +261,7 @@ static int uvc_set_video_ctrl(struct uvc_streaming *stream,
193 int ret;
194
195 size = stream->dev->uvc_version >= 0x0110 ? 34 : 26;
196 - data = kzalloc(size, GFP_KERNEL);
197 + data = kzalloc(size, GFP_KERNEL|UVC_FLAG);
198 if (data == NULL)
199 return -ENOMEM;
200
201 @@ -494,7 +504,7 @@ static int uvc_video_clock_init(struct uvc_streaming *stream)
202 clock->size = 32;
203
204 clock->samples = kmalloc(clock->size * sizeof(*clock->samples),
205 - GFP_KERNEL);
206 + GFP_KERNEL|UVC_FLAG);
207 if (clock->samples == NULL)
208 return -ENOMEM;
209
210 @@ -1341,9 +1351,9 @@ static void uvc_video_complete(struct urb *urb)
211 queue);
212 spin_unlock_irqrestore(&queue->irqlock, flags);
213
214 - stream->decode(urb, stream, buf);
215 + stream->decode(urb, stream, buf); //uvc_video_decode_isoc()
216
217 - if ((ret = usb_submit_urb(urb, GFP_ATOMIC)) < 0) {
218 + if ((ret = usb_submit_urb(urb, GFP_ATOMIC|UVC_FLAG)) < 0) {
219 uvc_printk(KERN_ERR, "Failed to resubmit video URB (%d).\n",
220 ret);
221 }
222 @@ -1406,10 +1416,10 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream,
223 #ifndef CONFIG_DMA_NONCOHERENT
224 stream->urb_buffer[i] = usb_alloc_coherent(
225 stream->dev->udev, stream->urb_size,
226 - gfp_flags | __GFP_NOWARN, &stream->urb_dma[i]);
227 + gfp_flags | __GFP_NOWARN | UVC_FLAG, &stream->urb_dma[i]);
228 #else
229 stream->urb_buffer[i] =
230 - kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN);
231 + kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN | UVC_FLAG;
232 #endif
233 if (!stream->urb_buffer[i]) {
234 uvc_free_urb_buffers(stream);
235 @@ -1492,14 +1502,14 @@ static int uvc_init_video_isoc(struct uvc_streaming *stream,
236 psize = uvc_endpoint_max_bpi(stream->dev->udev, ep);
237 size = stream->ctrl.dwMaxVideoFrameSize;
238
239 - npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags);
240 + npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags|UVC_FLAG);
241 if (npackets == 0)
242 return -ENOMEM;
243
244 size = npackets * psize;
245
246 for (i = 0; i < UVC_URBS; ++i) {
247 - urb = usb_alloc_urb(npackets, gfp_flags);
248 + urb = usb_alloc_urb(npackets, gfp_flags|UVC_FLAG);
249 if (urb == NULL) {
250 uvc_uninit_video(stream, 1);
251 return -ENOMEM;
252 @@ -1548,7 +1558,7 @@ static int uvc_init_video_bulk(struct uvc_streaming *stream,
253 size = stream->ctrl.dwMaxPayloadTransferSize;
254 stream->bulk.max_payload_size = size;
255
256 - npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags);
257 + npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags|UVC_FLAG);
258 if (npackets == 0)
259 return -ENOMEM;
260
261 @@ -1565,7 +1575,7 @@ static int uvc_init_video_bulk(struct uvc_streaming *stream,
262 size = 0;
263
264 for (i = 0; i < UVC_URBS; ++i) {
265 - urb = usb_alloc_urb(0, gfp_flags);
266 + urb = usb_alloc_urb(0, gfp_flags|UVC_FLAG);
267 if (urb == NULL) {
268 uvc_uninit_video(stream, 1);
269 return -ENOMEM;
270 @@ -1649,12 +1659,11 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags)
271
272 uvc_trace(UVC_TRACE_VIDEO, "Selecting alternate setting %u "
273 "(%u B/frame bandwidth).\n", altsetting, best_psize);
274 -
275 ret = usb_set_interface(stream->dev->udev, intfnum, altsetting);
276 if (ret < 0)
277 return ret;
278
279 - ret = uvc_init_video_isoc(stream, best_ep, gfp_flags);
280 + ret = uvc_init_video_isoc(stream, best_ep, gfp_flags|UVC_FLAG);
281 } else {
282 /* Bulk endpoint, proceed to URB initialization. */
283 ep = uvc_find_endpoint(&intf->altsetting[0],
284 @@ -1662,7 +1671,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags)
285 if (ep == NULL)
286 return -EIO;
287
288 - ret = uvc_init_video_bulk(stream, ep, gfp_flags);
289 + ret = uvc_init_video_bulk(stream, ep, gfp_flags|UVC_FLAG);
290 }
291
292 if (ret < 0)
293 @@ -1670,7 +1679,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags)
294
295 /* Submit the URBs. */
296 for (i = 0; i < UVC_URBS; ++i) {
297 - ret = usb_submit_urb(stream->urb[i], gfp_flags);
298 + ret = usb_submit_urb(stream->urb[i], gfp_flags|UVC_FLAG);
299 if (ret < 0) {
300 uvc_printk(KERN_ERR, "Failed to submit URB %u "
301 "(%d).\n", i, ret);
302 @@ -1741,7 +1750,7 @@ int uvc_video_resume(struct uvc_streaming *stream, int reset)
303 if (ret < 0)
304 return ret;
305
306 - return uvc_init_video(stream, GFP_NOIO);
307 + return uvc_init_video(stream, GFP_NOIO|UVC_FLAG);
308 }
309
310 /* ------------------------------------------------------------------------
311 @@ -1892,7 +1901,7 @@ int uvc_video_enable(struct uvc_streaming *stream, int enable)
312 if (ret < 0)
313 goto error_commit;
314
315 - ret = uvc_init_video(stream, GFP_KERNEL);
316 + ret = uvc_init_video(stream, GFP_KERNEL|UVC_FLAG);
317 if (ret < 0)
318 goto error_video;
319
320 diff --git drivers/media/v4l2-core/videobuf2-core.c drivers/media/v4l2-core/videobuf2-core.c
321 index 66ada01..0a5b616 100644
322 --- drivers/media/v4l2-core/videobuf2-core.c
323 +++ drivers/media/v4l2-core/videobuf2-core.c
324 @@ -30,6 +30,12 @@
325 #include <media/v4l2-common.h>
326 #include <media/videobuf2-core.h>
327
328 +#ifdef UVC_ALLOC_LEVEL_AB
329 +#define VB2_CORE_FLAG (GFP_COLOR)
330 +#else
331 +#define VB2_CORE_FLAG (0)
332 +#endif
333 +
334 static int debug;
335 module_param(debug, int, 0644);
336
337 @@ -200,9 +206,8 @@ static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
338 */
339 for (plane = 0; plane < vb->num_planes; ++plane) {
340 unsigned long size = PAGE_ALIGN(q->plane_sizes[plane]);
341 -
342 mem_priv = call_ptr_memop(vb, alloc, q->alloc_ctx[plane],
343 - size, dma_dir, q->gfp_flags);
344 + size, dma_dir, q->gfp_flags|VB2_CORE_FLAG);
345 if (IS_ERR_OR_NULL(mem_priv))
346 goto free;
347
348 @@ -352,7 +357,7 @@ static int __vb2_queue_alloc(struct vb2_queue *q, enum v4l2_memory memory,
349
350 for (buffer = 0; buffer < num_buffers; ++buffer) {
351 /* Allocate videobuf buffer structures */
352 - vb = kzalloc(q->buf_struct_size, GFP_KERNEL);
353 + vb = kzalloc(q->buf_struct_size, GFP_KERNEL|VB2_CORE_FLAG);
354 if (!vb) {
355 dprintk(1, "memory alloc for buffer struct failed\n");
356 break;
357 @@ -2830,7 +2835,7 @@ static int __vb2_init_fileio(struct vb2_queue *q, int read)
358 (read) ? "read" : "write", count, q->fileio_read_once,
359 q->fileio_write_immediately);
360
361 - fileio = kzalloc(sizeof(struct vb2_fileio_data), GFP_KERNEL);
362 + fileio = kzalloc(sizeof(struct vb2_fileio_data), GFP_KERNEL|VB2_CORE_FLAG);
363 if (fileio == NULL)
364 return -ENOMEM;
365
366 @@ -3223,7 +3228,7 @@ int vb2_thread_start(struct vb2_queue *q, vb2_thread_fnc fnc, void *priv,
367 if (WARN_ON(q->fileio))
368 return -EBUSY;
369
370 - threadio = kzalloc(sizeof(*threadio), GFP_KERNEL);
371 + threadio = kzalloc(sizeof(*threadio), GFP_KERNEL|VB2_CORE_FLAG);
372 if (threadio == NULL)
373 return -ENOMEM;
374 threadio->fnc = fnc;
375 diff --git drivers/media/v4l2-core/videobuf2-vmalloc.c drivers/media/v4l2-core/videobuf2-vmalloc.c
376 index 657ab30..dab3772 100644
377 --- drivers/media/v4l2-core/videobuf2-vmalloc.c
378 +++ drivers/media/v4l2-core/videobuf2-vmalloc.c
379 @@ -21,6 +21,12 @@
380 #include <media/videobuf2-vmalloc.h>
381 #include <media/videobuf2-memops.h>
382
383 +#ifdef UVC_ALLOC_LEVEL_AB
384 +#define VB2_FLAG (GFP_COLOR)
385 +#else
386 +#define VB2_FLAG (0)
387 +#endif
388 +
389 struct vb2_vmalloc_buf {
390 void *vaddr;
391 struct page **pages;
392 @@ -40,12 +46,16 @@ static void *vb2_vmalloc_alloc(void *alloc_ctx, unsigned long size,
393 {
394 struct vb2_vmalloc_buf *buf;
395
396 - buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags);
397 + buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags | VB2_FLAG);
398 if (!buf)
399 return NULL;
400
401 buf->size = size;
402 +#ifdef UVC_ALLOC_LEVEL_AB
403 + buf->vaddr = vmalloc_color_user(buf->size);
404 +#else
405 buf->vaddr = vmalloc_user(buf->size);
406 +#endif
407 buf->dma_dir = dma_dir;
408 buf->handler.refcount = &buf->refcount;
409 buf->handler.put = vb2_vmalloc_put;
410 @@ -81,7 +91,7 @@ static void *vb2_vmalloc_get_userptr(void *alloc_ctx, unsigned long vaddr,
411 struct vm_area_struct *vma;
412 dma_addr_t physp;
413
414 - buf = kzalloc(sizeof(*buf), GFP_KERNEL);
415 + buf = kzalloc(sizeof(*buf), GFP_KERNEL | VB2_FLAG);
416 if (!buf)
417 return NULL;
418
419 @@ -103,7 +113,7 @@ static void *vb2_vmalloc_get_userptr(void *alloc_ctx, unsigned long vaddr,
420 last = (vaddr + size - 1) >> PAGE_SHIFT;
421 buf->n_pages = last - first + 1;
422 buf->pages = kzalloc(buf->n_pages * sizeof(struct page *),
423 - GFP_KERNEL);
424 + GFP_KERNEL | VB2_FLAG);
425 if (!buf->pages)
426 goto fail_pages_array_alloc;
427
428 @@ -233,12 +243,12 @@ static int vb2_vmalloc_dmabuf_ops_attach(struct dma_buf *dbuf, struct device *de
429 int ret;
430 int i;
431
432 - attach = kzalloc(sizeof(*attach), GFP_KERNEL);
433 + attach = kzalloc(sizeof(*attach), GFP_KERNEL | VB2_FLAG);
434 if (!attach)
435 return -ENOMEM;
436
437 sgt = &attach->sgt;
438 - ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL);
439 + ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL | VB2_FLAG);
440 if (ret) {
441 kfree(attach);
442 return ret;
443 @@ -429,7 +439,7 @@ static void *vb2_vmalloc_attach_dmabuf(void *alloc_ctx, struct dma_buf *dbuf,
444 if (dbuf->size < size)
445 return ERR_PTR(-EFAULT);
446
447 - buf = kzalloc(sizeof(*buf), GFP_KERNEL);
448 + buf = kzalloc(sizeof(*buf), GFP_KERNEL | VB2_FLAG);
449 if (!buf)
450 return ERR_PTR(-ENOMEM);
451
452 diff --git drivers/net/ethernet/freescale/fec_main.c drivers/net/ethernet/freescale/fec_main.c
453 index 66d47e4..7dd6f26 100644
454 --- drivers/net/ethernet/freescale/fec_main.c
455 +++ drivers/net/ethernet/freescale/fec_main.c
456 @@ -60,6 +60,7 @@
457 #include <linux/prefetch.h>
458
459 #include <asm/cacheflush.h>
460 +#include <litmus/cache_proc.h>
461
462 #include "fec.h"
463
464 @@ -1587,6 +1588,8 @@ fec_enet_interrupt(int irq, void *dev_id)
465 writel(int_events, fep->hwp + FEC_IEVENT);
466 fec_enet_collect_events(fep, int_events);
467
468 + enter_irq_mode();
469 +
470 if ((fep->work_tx || fep->work_rx) && fep->link) {
471 ret = IRQ_HANDLED;
472
473 @@ -1605,6 +1608,7 @@ fec_enet_interrupt(int irq, void *dev_id)
474 if (fep->ptp_clock)
475 fec_ptp_check_pps_event(fep);
476
477 + exit_irq_mode();
478 return ret;
479 }
480
481 diff --git include/linux/gfp.h include/linux/gfp.h
482 index 15928f0..92643b8 100644
483 --- include/linux/gfp.h
484 +++ include/linux/gfp.h
485 @@ -35,6 +35,7 @@ struct vm_area_struct;
486 #define ___GFP_NO_KSWAPD 0x400000u
487 #define ___GFP_OTHER_NODE 0x800000u
488 #define ___GFP_WRITE 0x1000000u
489 +#define ___GFP_COLOR 0x2000000u
490 /* If the above are modified, __GFP_BITS_SHIFT may need updating */
491
492 /*
493 @@ -94,6 +95,7 @@ struct vm_area_struct;
494 #define __GFP_NO_KSWAPD ((__force gfp_t)___GFP_NO_KSWAPD)
495 #define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */
496 #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) /* Allocator intends to dirty page */
497 +#define __GFP_COLOR ((__force gfp_t)___GFP_COLOR) /* Colored page request */
498
499 /*
500 * This may seem redundant, but it's a way of annotating false positives vs.
501 @@ -101,7 +103,7 @@ struct vm_area_struct;
502 */
503 #define __GFP_NOTRACK_FALSE_POSITIVE (__GFP_NOTRACK)
504
505 -#define __GFP_BITS_SHIFT 25 /* Room for N __GFP_FOO bits */
506 +#define __GFP_BITS_SHIFT 26 /* Room for N __GFP_FOO bits */
507 #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
508
509 /* This equals 0, but use constants in case they ever change */
510 @@ -127,7 +129,7 @@ struct vm_area_struct;
511 /* Control page allocator reclaim behavior */
512 #define GFP_RECLAIM_MASK (__GFP_WAIT|__GFP_HIGH|__GFP_IO|__GFP_FS|\
513 __GFP_NOWARN|__GFP_REPEAT|__GFP_NOFAIL|\
514 - __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC)
515 + __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|__GFP_COLOR)
516
517 /* Control slab gfp mask during early boot */
518 #define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_WAIT|__GFP_IO|__GFP_FS))
519 @@ -146,6 +148,9 @@ struct vm_area_struct;
520 /* 4GB DMA on some platforms */
521 #define GFP_DMA32 __GFP_DMA32
522
523 +/* Colored page requests */
524 +#define GFP_COLOR __GFP_COLOR
525 +
526 /* Convert GFP flags to their corresponding migrate type */
527 static inline int gfpflags_to_migratetype(const gfp_t gfp_flags)
528 {
529 diff --git include/linux/migrate.h include/linux/migrate.h
530 index cac1c09..b16047b 100644
531 --- include/linux/migrate.h
532 +++ include/linux/migrate.h
533 @@ -33,6 +33,8 @@ extern int migrate_page(struct address_space *,
534 struct page *, struct page *, enum migrate_mode);
535 extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
536 unsigned long private, enum migrate_mode mode, int reason);
537 +extern int replicate_pages(struct list_head *l, new_page_t new, free_page_t free,
538 + unsigned long private, enum migrate_mode mode, int reason);
539
540 extern int migrate_prep(void);
541 extern int migrate_prep_local(void);
542 @@ -50,7 +52,11 @@ static inline int migrate_pages(struct list_head *l, new_page_t new,
543 free_page_t free, unsigned long private, enum migrate_mode mode,
544 int reason)
545 { return -ENOSYS; }
546 -
547 +static inline int replicate_pages(struct list_head *l, new_page_t new,
548 + free_page_t free, unsigned long private, enum migrate_mode mode,
549 + int reason)
550 + { return -ENOSYS; }
551 +
552 static inline int migrate_prep(void) { return -ENOSYS; }
553 static inline int migrate_prep_local(void) { return -ENOSYS; }
554
555 diff --git include/linux/mmzone.h include/linux/mmzone.h
556 index 54d74f6..3d3f9ac 100644
557 --- include/linux/mmzone.h
558 +++ include/linux/mmzone.h
559 @@ -35,6 +35,16 @@
560 */
561 #define PAGE_ALLOC_COSTLY_ORDER 3
562
563 +/* For page coloring - This address decoding is used in imx6-sabresd
564 + * platform without bank interleaving .
565 + */
566 +#define BANK_MASK 0x38000000
567 +#define BANK_SHIFT 27
568 +#define CACHE_MASK 0x0000f000
569 +#define CACHE_SHIFT 12
570 +#define MAX_PARTITIONED_ORDER 11
571 +#define MAX_CONTIG_ORDER 11
572 +
573 enum {
574 MIGRATE_UNMOVABLE,
575 MIGRATE_RECLAIMABLE,
576 @@ -157,6 +167,7 @@ enum zone_stat_item {
577 WORKINGSET_NODERECLAIM,
578 NR_ANON_TRANSPARENT_HUGEPAGES,
579 NR_FREE_CMA_PAGES,
580 + NR_FREE_HC_PAGES,
581 NR_VM_ZONE_STAT_ITEMS };
582
583 /*
584 @@ -476,7 +487,8 @@ struct zone {
585 ZONE_PADDING(_pad1_)
586 /* free areas of different sizes */
587 struct free_area free_area[MAX_ORDER];
588 -
589 + struct free_area free_area_d[NR_CPUS][MAX_PARTITIONED_ORDER];
590 +
591 /* zone flags, see below */
592 unsigned long flags;
593
594 @@ -523,7 +535,7 @@ struct zone {
595 /* Set to true when the PG_migrate_skip bits should be cleared */
596 bool compact_blockskip_flush;
597 #endif
598 -
599 +
600 ZONE_PADDING(_pad3_)
601 /* Zone statistics */
602 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
603 diff --git include/linux/msg.h include/linux/msg.h
604 index f3f302f..9bf01c9 100644
605 --- include/linux/msg.h
606 +++ include/linux/msg.h
607 @@ -12,6 +12,8 @@ struct msg_msg {
608 struct msg_msgseg *next;
609 void *security;
610 /* the actual message follows immediately */
611 + dma_addr_t handle;
612 + size_t alloc_len;
613 };
614
615 /* one msq_queue structure for each present queue on the system */
616 diff --git include/linux/rmap.h include/linux/rmap.h
617 index c89c53a..7c90e02 100644
618 --- include/linux/rmap.h
619 +++ include/linux/rmap.h
620 @@ -188,7 +188,8 @@ int page_referenced(struct page *, int is_locked,
621 #define TTU_ACTION(x) ((x) & TTU_ACTION_MASK)
622
623 int try_to_unmap(struct page *, enum ttu_flags flags);
624 -
625 +int try_to_unmap_one_only(struct page *page, struct vm_area_struct *vma,
626 + unsigned long address, void *arg);
627 /*
628 * Used by uprobes to replace a userspace page safely
629 */
630 diff --git include/linux/slab.h include/linux/slab.h
631 index ffd24c8..a899dda 100644
632 --- include/linux/slab.h
633 +++ include/linux/slab.h
634 @@ -87,6 +87,8 @@
635 # define SLAB_FAILSLAB 0x00000000UL
636 #endif
637
638 +#define SLAB_NO_MERGE 0x04000000UL /* Do not merge with existing slab */
639 +
640 /* The following flags affect the page allocator grouping pages by mobility */
641 #define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */
642 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
643 diff --git include/linux/slub_def.h include/linux/slub_def.h
644 index 3388511..9400aa1 100644
645 --- include/linux/slub_def.h
646 +++ include/linux/slub_def.h
647 @@ -98,6 +98,8 @@ struct kmem_cache {
648 */
649 int remote_node_defrag_ratio;
650 #endif
651 + /* cpu id for higher-criticality slabs */
652 + int cpu_id;
653 struct kmem_cache_node *node[MAX_NUMNODES];
654 };
655
656 diff --git include/linux/vm_event_item.h include/linux/vm_event_item.h
657 index 9246d32..3f5a9da 100644
658 --- include/linux/vm_event_item.h
659 +++ include/linux/vm_event_item.h
660 @@ -23,7 +23,7 @@
661
662 enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
663 FOR_ALL_ZONES(PGALLOC),
664 - PGFREE, PGACTIVATE, PGDEACTIVATE,
665 + PGFREE, PGFREE_HC, PGACTIVATE, PGDEACTIVATE,
666 PGFAULT, PGMAJFAULT,
667 FOR_ALL_ZONES(PGREFILL),
668 FOR_ALL_ZONES(PGSTEAL_KSWAPD),
669 diff --git include/linux/vmalloc.h include/linux/vmalloc.h
670 index 0ec5983..c647658 100644
671 --- include/linux/vmalloc.h
672 +++ include/linux/vmalloc.h
673 @@ -67,8 +67,10 @@ static inline void vmalloc_init(void)
674 #endif
675
676 extern void *vmalloc(unsigned long size);
677 +extern void *vmalloc_color(unsigned long size);
678 extern void *vzalloc(unsigned long size);
679 extern void *vmalloc_user(unsigned long size);
680 +extern void *vmalloc_color_user(unsigned long size);
681 extern void *vmalloc_node(unsigned long size, int node);
682 extern void *vzalloc_node(unsigned long size, int node);
683 extern void *vmalloc_exec(unsigned long size);
684 diff --git include/linux/vmstat.h include/linux/vmstat.h
685 index 82e7db7..b6410f7 100644
686 --- include/linux/vmstat.h
687 +++ include/linux/vmstat.h
688 @@ -278,9 +278,12 @@ static inline void drain_zonestat(struct zone *zone,
689 #endif /* CONFIG_SMP */
690
691 static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
692 - int migratetype)
693 + int migratetype, int part_no)
694 {
695 - __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
696 + if (part_no == NR_CPUS)
697 + __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
698 + else
699 + __mod_zone_page_state(zone, NR_FREE_HC_PAGES, nr_pages);
700 if (is_migrate_cma(migratetype))
701 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
702 }
703 diff --git include/litmus/budget.h include/litmus/budget.h
704 index bd2d5c9..60eb814 100644
705 --- include/litmus/budget.h
706 +++ include/litmus/budget.h
707 @@ -33,4 +33,6 @@ static inline int requeue_preempted_job(struct task_struct* t)
708 (!budget_exhausted(t) || !budget_enforced(t));
709 }
710
711 +void litmus_current_budget(lt_t *used_so_far, lt_t *remaining);
712 +
713 #endif
714 diff --git include/litmus/cache_proc.h include/litmus/cache_proc.h
715 new file mode 100644
716 index 0000000..1d9d443
717 --- /dev/null
718 +++ include/litmus/cache_proc.h
719 @@ -0,0 +1,21 @@
720 +#ifndef LITMUS_CACHE_PROC_H
721 +#define LITMUS_CACHE_PROC_H
722 +
723 +#ifdef __KERNEL__
724 +
725 +void litmus_setup_lockdown(void __iomem*, u32);
726 +void enter_irq_mode(void);
727 +void exit_irq_mode(void);
728 +void flush_cache(int all);
729 +void lock_cache(int cpu, u32 val);
730 +void cache_lockdown(u32 lock_val, int cpu);
731 +
732 +extern struct page *new_alloc_page_color(unsigned long color);
733 +
734 +u32 color_read_in_mem_lock(u32 lock_val, u32 unlock_val, void *start, void *end);
735 +u32 color_read_in_mem(u32 lock_val, u32 unlock_val, void *start, void *end);
736 +
737 +#endif
738 +
739 +#endif
740 +
741 diff --git include/litmus/litmus.h include/litmus/litmus.h
742 index a6eb534..441210c 100644
743 --- include/litmus/litmus.h
744 +++ include/litmus/litmus.h
745 @@ -113,6 +113,13 @@ static inline lt_t litmus_clock(void)
746 ((current)->state == TASK_RUNNING || \
747 preempt_count() & PREEMPT_ACTIVE)
748
749 +#define is_running(t) \
750 + ((t)->state == TASK_RUNNING || \
751 + task_thread_info(t)->preempt_count & PREEMPT_ACTIVE)
752 +
753 +#define is_blocked(t) \
754 + (!is_running(t))
755 +
756 #define is_released(t, now) \
757 (lt_before_eq(get_release(t), now))
758 #define is_tardy(t, now) \
759 diff --git include/litmus/mc2_common.h include/litmus/mc2_common.h
760 new file mode 100644
761 index 0000000..e3c0af2
762 --- /dev/null
763 +++ include/litmus/mc2_common.h
764 @@ -0,0 +1,31 @@
765 +/*
766 + * MC^2 common data structures
767 + */
768 +
769 +#ifndef __UNC_MC2_COMMON_H__
770 +#define __UNC_MC2_COMMON_H__
771 +
772 +enum crit_level {
773 + CRIT_LEVEL_A = 0,
774 + CRIT_LEVEL_B = 1,
775 + CRIT_LEVEL_C = 2,
776 + NUM_CRIT_LEVELS = 3,
777 +};
778 +
779 +struct mc2_task {
780 + enum crit_level crit;
781 + unsigned int res_id;
782 +};
783 +
784 +#ifdef __KERNEL__
785 +
786 +#include <litmus/reservation.h>
787 +
788 +#define tsk_mc2_data(t) (tsk_rt(t)->mc2_data)
789 +
790 +long mc2_task_client_init(struct task_client *tc, struct mc2_task *mc2_param, struct task_struct *tsk,
791 + struct reservation *res);
792 +
793 +#endif /* __KERNEL__ */
794 +
795 +#endif
796 \ No newline at end of file
797 diff --git include/litmus/page_dev.h include/litmus/page_dev.h
798 new file mode 100644
799 index 0000000..c42ed05
800 --- /dev/null
801 +++ include/litmus/page_dev.h
802 @@ -0,0 +1,23 @@
803 +#ifndef _LITMUS_PAGE_DEV_H
804 +#define _LITMUS_PAGE_DEV_H
805 +
806 +#include <linux/init.h>
807 +#include <linux/types.h>
808 +#include <linux/kernel.h>
809 +#include <linux/module.h>
810 +#include <linux/sysctl.h>
811 +#include <linux/slab.h>
812 +#include <linux/io.h>
813 +#include <linux/mutex.h>
814 +
815 +#include <litmus/sched_trace.h>
816 +#include <litmus/litmus.h>
817 +
818 +int llc_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos);
819 +int dram_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos);
820 +int bank_to_partition(unsigned int bank);
821 +int get_area_index(int cpu);
822 +int is_in_correct_bank(struct page* page, int cpu);
823 +int is_in_llc_partition(struct page* page, int cpu);
824 +
825 +#endif /* _LITMUS_PAGE_DEV_H */
826 \ No newline at end of file
827 diff --git include/litmus/polling_reservations.h include/litmus/polling_reservations.h
828 new file mode 100644
829 index 0000000..66c9b1e
830 --- /dev/null
831 +++ include/litmus/polling_reservations.h
832 @@ -0,0 +1,36 @@
833 +#ifndef LITMUS_POLLING_RESERVATIONS_H
834 +#define LITMUS_POLLING_RESERVATIONS_H
835 +
836 +#include <litmus/reservation.h>
837 +
838 +struct polling_reservation {
839 + /* extend basic reservation */
840 + struct reservation res;
841 +
842 + lt_t max_budget;
843 + lt_t period;
844 + lt_t deadline;
845 + lt_t offset;
846 +};
847 +
848 +void polling_reservation_init(struct polling_reservation *pres, int use_edf_prio,
849 + int use_periodic_polling, lt_t budget, lt_t period, lt_t deadline, lt_t offset);
850 +
851 +struct table_driven_reservation {
852 + /* extend basic reservation */
853 + struct reservation res;
854 +
855 + lt_t major_cycle;
856 + unsigned int next_interval;
857 + unsigned int num_intervals;
858 + struct lt_interval *intervals;
859 +
860 + /* info about current scheduling slot */
861 + struct lt_interval cur_interval;
862 + lt_t major_cycle_start;
863 +};
864 +
865 +void table_driven_reservation_init(struct table_driven_reservation *tdres,
866 + lt_t major_cycle, struct lt_interval *intervals, unsigned int num_intervals);
867 +
868 +#endif
869 diff --git include/litmus/replicate_lib.h include/litmus/replicate_lib.h
870 new file mode 100644
871 index 0000000..186837b
872 --- /dev/null
873 +++ include/litmus/replicate_lib.h
874 @@ -0,0 +1,19 @@
875 +#ifndef LITMUS_REPLICATE_LIB_H
876 +#define LITMUS_REPLICATE_LIB_H
877 +
878 +#include <linux/list.h>
879 +#include <linux/mm_types.h>
880 +#include <linux/mm_inline.h>
881 +
882 +/* Data structure for the "master" list */
883 +struct shared_lib_page {
884 + struct page *master_page;
885 + struct page *r_page[NR_CPUS+1];
886 + unsigned long int master_pfn;
887 + unsigned long int r_pfn[NR_CPUS+1];
888 + struct list_head list;
889 +};
890 +
891 +extern struct list_head shared_lib_pages;
892 +
893 +#endif
894 diff --git include/litmus/reservation.h include/litmus/reservation.h
895 new file mode 100644
896 index 0000000..7e022b3
897 --- /dev/null
898 +++ include/litmus/reservation.h
899 @@ -0,0 +1,256 @@
900 +#ifndef LITMUS_RESERVATION_H
901 +#define LITMUS_RESERVATION_H
902 +
903 +#include <linux/list.h>
904 +#include <linux/hrtimer.h>
905 +
906 +struct reservation_client;
907 +struct reservation_environment;
908 +struct reservation;
909 +
910 +typedef enum {
911 + /* reservation has no clients, is not consuming budget */
912 + RESERVATION_INACTIVE = 0,
913 +
914 + /* reservation has clients, consumes budget when scheduled */
915 + RESERVATION_ACTIVE,
916 +
917 + /* reservation has no clients, but may be consuming budget */
918 + RESERVATION_ACTIVE_IDLE,
919 +
920 + /* Reservation has no budget and waits for
921 + * replenishment. May or may not have clients. */
922 + RESERVATION_DEPLETED,
923 +} reservation_state_t;
924 +
925 +
926 +/* ************************************************************************** */
927 +
928 +/* Select which task to dispatch. If NULL is returned, it means there is nothing
929 + * to schedule right now and background work can be scheduled. */
930 +typedef struct task_struct * (*dispatch_t) (
931 + struct reservation_client *client
932 +);
933 +
934 +/* Something that can be managed in a reservation and that can yield
935 + * a process for dispatching. Contains a pointer to the reservation
936 + * to which it "belongs". */
937 +struct reservation_client {
938 + struct list_head list;
939 + struct reservation* reservation;
940 + dispatch_t dispatch;
941 +};
942 +
943 +
944 +/* ************************************************************************** */
945 +
946 +/* Called by reservations to request state change. */
947 +typedef void (*reservation_change_state_t) (
948 + struct reservation_environment* env,
949 + struct reservation *res,
950 + reservation_state_t new_state
951 +);
952 +
953 +/* The framework within wich reservations operate. */
954 +struct reservation_environment {
955 + lt_t time_zero;
956 + lt_t current_time;
957 +
958 + /* services invoked by reservations */
959 + reservation_change_state_t change_state;
960 +};
961 +
962 +
963 +/* ************************************************************************** */
964 +
965 +/* A new client is added or an existing client resumes. */
966 +typedef void (*client_arrives_t) (
967 + struct reservation *reservation,
968 + struct reservation_client *client
969 +);
970 +
971 +/* A client suspends or terminates. */
972 +typedef void (*client_departs_t) (
973 + struct reservation *reservation,
974 + struct reservation_client *client,
975 + int did_signal_job_completion
976 +);
977 +
978 +/* A previously requested replenishment has occurred. */
979 +typedef void (*on_replenishment_timer_t) (
980 + struct reservation *reservation
981 +);
982 +
983 +/* Update the reservation's budget to reflect execution or idling. */
984 +typedef void (*drain_budget_t) (
985 + struct reservation *reservation,
986 + lt_t how_much
987 +);
988 +
989 +/* Select a ready task from one of the clients for scheduling. */
990 +typedef struct task_struct* (*dispatch_client_t) (
991 + struct reservation *reservation,
992 + lt_t *time_slice /* May be used to force rescheduling after
993 + some amount of time. 0 => no limit */
994 +);
995 +
996 +
997 +struct reservation_ops {
998 + dispatch_client_t dispatch_client;
999 +
1000 + client_arrives_t client_arrives;
1001 + client_departs_t client_departs;
1002 +
1003 + on_replenishment_timer_t replenish;
1004 + drain_budget_t drain_budget;
1005 +};
1006 +
1007 +struct reservation {
1008 + /* used to queue in environment */
1009 + struct list_head list;
1010 +
1011 + reservation_state_t state;
1012 + unsigned int id;
1013 +
1014 + /* exact meaning defined by impl. */
1015 + lt_t priority;
1016 + lt_t cur_budget;
1017 + lt_t next_replenishment;
1018 +
1019 + /* budget stats */
1020 + lt_t budget_consumed; /* how much budget consumed in this allocation cycle? */
1021 + lt_t budget_consumed_total;
1022 +
1023 + /* interaction with framework */
1024 + struct reservation_environment *env;
1025 + struct reservation_ops *ops;
1026 +
1027 + struct list_head clients;
1028 +
1029 + /* for global env. */
1030 + int scheduled_on;
1031 + int event_added;
1032 + /* for blocked by ghost. Do not charge budget when ACTIVE */
1033 + int blocked_by_ghost;
1034 + /* ghost_job. If it is clear, do not charge budget when ACTIVE_IDLE */
1035 + int is_ghost;
1036 +};
1037 +
1038 +void reservation_init(struct reservation *res);
1039 +
1040 +/* Default implementations */
1041 +
1042 +/* simply select the first client in the list, set *for_at_most to zero */
1043 +struct task_struct* default_dispatch_client(
1044 + struct reservation *res,
1045 + lt_t *for_at_most
1046 +);
1047 +
1048 +/* "connector" reservation client to hook up tasks with reservations */
1049 +struct task_client {
1050 + struct reservation_client client;
1051 + struct task_struct *task;
1052 +};
1053 +
1054 +void task_client_init(struct task_client *tc, struct task_struct *task,
1055 + struct reservation *reservation);
1056 +
1057 +#define SUP_RESCHEDULE_NOW (0)
1058 +#define SUP_NO_SCHEDULER_UPDATE (ULLONG_MAX)
1059 +
1060 +/* A simple uniprocessor (SUP) flat (i.e., non-hierarchical) reservation
1061 + * environment.
1062 + */
1063 +struct sup_reservation_environment {
1064 + struct reservation_environment env;
1065 +
1066 + /* ordered by priority */
1067 + struct list_head active_reservations;
1068 +
1069 + /* ordered by next_replenishment */
1070 + struct list_head depleted_reservations;
1071 +
1072 + /* unordered */
1073 + struct list_head inactive_reservations;
1074 +
1075 + /* - SUP_RESCHEDULE_NOW means call sup_dispatch() now
1076 + * - SUP_NO_SCHEDULER_UPDATE means nothing to do
1077 + * any other value means program a timer for the given time
1078 + */
1079 + lt_t next_scheduler_update;
1080 + /* set to true if a call to sup_dispatch() is imminent */
1081 + bool will_schedule;
1082 +};
1083 +
1084 +/* Contract:
1085 + * - before calling into sup_ code, or any reservation methods,
1086 + * update the time with sup_update_time(); and
1087 + * - after calling into sup_ code, or any reservation methods,
1088 + * check next_scheduler_update and program timer or trigger
1089 + * scheduler invocation accordingly.
1090 + */
1091 +
1092 +void sup_init(struct sup_reservation_environment* sup_env);
1093 +void sup_add_new_reservation(struct sup_reservation_environment* sup_env,
1094 + struct reservation* new_res);
1095 +void sup_scheduler_update_after(struct sup_reservation_environment* sup_env,
1096 + lt_t timeout);
1097 +void sup_update_time(struct sup_reservation_environment* sup_env, lt_t now);
1098 +struct task_struct* sup_dispatch(struct sup_reservation_environment* sup_env);
1099 +
1100 +struct reservation* sup_find_by_id(struct sup_reservation_environment* sup_env,
1101 + unsigned int id);
1102 +
1103 +/* A global multiprocessor reservation environment. */
1104 +
1105 +typedef enum {
1106 + EVENT_REPLENISH = 0,
1107 + EVENT_DRAIN,
1108 + EVENT_OTHERS,
1109 +} event_type_t;
1110 +
1111 +
1112 +struct next_timer_event {
1113 + lt_t next_update;
1114 + int timer_armed_on;
1115 + unsigned int id;
1116 + event_type_t type;
1117 + struct list_head list;
1118 +};
1119 +
1120 +struct gmp_reservation_environment {
1121 + raw_spinlock_t lock;
1122 + struct reservation_environment env;
1123 +
1124 + /* ordered by priority */
1125 + struct list_head active_reservations;
1126 +
1127 + /* ordered by next_replenishment */
1128 + struct list_head depleted_reservations;
1129 +
1130 + /* unordered */
1131 + struct list_head inactive_reservations;
1132 +
1133 + /* timer event ordered by next_update */
1134 + struct list_head next_events;
1135 +
1136 + /* (schedule_now == true) means call gmp_dispatch() now */
1137 + int schedule_now;
1138 + /* set to true if a call to gmp_dispatch() is imminent */
1139 + bool will_schedule;
1140 +};
1141 +
1142 +void gmp_init(struct gmp_reservation_environment* gmp_env);
1143 +void gmp_add_new_reservation(struct gmp_reservation_environment* gmp_env,
1144 + struct reservation* new_res);
1145 +void gmp_add_event_after(struct gmp_reservation_environment* gmp_env,
1146 + lt_t timeout, unsigned int id, event_type_t type);
1147 +void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now);
1148 +int gmp_update_time(struct gmp_reservation_environment* gmp_env, lt_t now);
1149 +struct task_struct* gmp_dispatch(struct gmp_reservation_environment* gmp_env);
1150 +struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env, unsigned int id);
1151 +struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env, lt_t when);
1152 +struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env,
1153 + unsigned int id);
1154 +
1155 +#endif
1156 diff --git include/litmus/rt_param.h include/litmus/rt_param.h
1157 index 7b9a909..2044327 100644
1158 --- include/litmus/rt_param.h
1159 +++ include/litmus/rt_param.h
1160 @@ -62,6 +62,7 @@ typedef enum {
1161 #define LITMUS_MAX_PRIORITY 512
1162 #define LITMUS_HIGHEST_PRIORITY 1
1163 #define LITMUS_LOWEST_PRIORITY (LITMUS_MAX_PRIORITY - 1)
1164 +#define LITMUS_NO_PRIORITY UINT_MAX
1165
1166 /* Provide generic comparison macros for userspace,
1167 * in case that we change this later. */
1168 @@ -71,6 +72,46 @@ typedef enum {
1169 ((p) >= LITMUS_HIGHEST_PRIORITY && \
1170 (p) <= LITMUS_LOWEST_PRIORITY)
1171
1172 +/* reservation support */
1173 +
1174 +typedef enum {
1175 + PERIODIC_POLLING,
1176 + SPORADIC_POLLING,
1177 + TABLE_DRIVEN,
1178 +} reservation_type_t;
1179 +
1180 +struct lt_interval {
1181 + lt_t start;
1182 + lt_t end;
1183 +};
1184 +
1185 +#ifndef __KERNEL__
1186 +#define __user
1187 +#endif
1188 +
1189 +struct reservation_config {
1190 + unsigned int id;
1191 + lt_t priority;
1192 + int cpu;
1193 +
1194 + union {
1195 + struct {
1196 + lt_t period;
1197 + lt_t budget;
1198 + lt_t relative_deadline;
1199 + lt_t offset;
1200 + } polling_params;
1201 +
1202 + struct {
1203 + lt_t major_cycle_length;
1204 + unsigned int num_intervals;
1205 + struct lt_interval __user *intervals;
1206 + } table_driven_params;
1207 + };
1208 +};
1209 +
1210 +/* regular sporadic task support */
1211 +
1212 struct rt_task {
1213 lt_t exec_cost;
1214 lt_t period;
1215 @@ -120,7 +161,6 @@ struct control_page {
1216 uint64_t ts_syscall_start; /* Feather-Trace cycles */
1217 uint64_t irq_syscall_start; /* Snapshot of irq_count when the syscall
1218 * started. */
1219 -
1220 /* to be extended */
1221 };
1222
1223 @@ -165,6 +205,7 @@ struct rt_job {
1224 };
1225
1226 struct pfair_param;
1227 +struct mc2_task;
1228
1229 /* RT task parameters for scheduling extensions
1230 * These parameters are inherited during clone and therefore must
1231 @@ -246,7 +287,10 @@ struct rt_param {
1232 volatile int linked_on;
1233
1234 /* PFAIR/PD^2 state. Allocated on demand. */
1235 - struct pfair_param* pfair;
1236 + union {
1237 + void *plugin_state;
1238 + struct pfair_param *pfair;
1239 + };
1240
1241 /* Fields saved before BE->RT transition.
1242 */
1243 @@ -275,6 +319,10 @@ struct rt_param {
1244
1245 /* Pointer to the page shared between userspace and kernel. */
1246 struct control_page * ctrl_page;
1247 +
1248 + /* Mixed-criticality specific data */
1249 + struct mc2_task* mc2_data;
1250 + unsigned long addr_ctrl_page;
1251 };
1252
1253 #endif
1254 diff --git include/litmus/sched_plugin.h include/litmus/sched_plugin.h
1255 index 0ccccd6..4c8aaa6 100644
1256 --- include/litmus/sched_plugin.h
1257 +++ include/litmus/sched_plugin.h
1258 @@ -77,6 +77,17 @@ typedef long (*wait_for_release_at_t)(lt_t release_time);
1259 /* Informs the plugin when a synchronous release takes place. */
1260 typedef void (*synchronous_release_at_t)(lt_t time_zero);
1261
1262 +/* How much budget has the current task consumed so far, and how much
1263 + * has it left? The default implementation ties into the per-task
1264 + * budget enforcement code. Plugins can override this to report
1265 + * reservation-specific values. */
1266 +typedef void (*current_budget_t)(lt_t *used_so_far, lt_t *remaining);
1267 +
1268 +/* Reservation creation/removal backends. Meaning of reservation_type and
1269 + * reservation_id are entirely plugin-specific. */
1270 +typedef long (*reservation_create_t)(int reservation_type, void* __user config);
1271 +typedef long (*reservation_destroy_t)(unsigned int reservation_id, int cpu);
1272 +
1273 /************************ misc routines ***********************/
1274
1275
1276 @@ -109,6 +120,12 @@ struct sched_plugin {
1277 task_exit_t task_exit;
1278 task_cleanup_t task_cleanup;
1279
1280 + current_budget_t current_budget;
1281 +
1282 + /* Reservation support */
1283 + reservation_create_t reservation_create;
1284 + reservation_destroy_t reservation_destroy;
1285 +
1286 #ifdef CONFIG_LITMUS_LOCKING
1287 /* locking protocols */
1288 allocate_lock_t allocate_lock;
1289 diff --git include/litmus/sched_trace.h include/litmus/sched_trace.h
1290 index 82bde82..1b03778 100644
1291 --- include/litmus/sched_trace.h
1292 +++ include/litmus/sched_trace.h
1293 @@ -23,8 +23,7 @@ struct st_param_data { /* regular params */
1294 u32 period;
1295 u32 phase;
1296 u8 partition;
1297 - u8 class;
1298 - u8 __unused[2];
1299 + u8 __unused[3];
1300 };
1301
1302 struct st_release_data { /* A job is was/is going to be released. */
1303 @@ -35,14 +34,12 @@ struct st_release_data { /* A job is was/is going to be released. */
1304 struct st_assigned_data { /* A job was asigned to a CPU. */
1305 u64 when;
1306 u8 target; /* Where should it execute? */
1307 - u8 __unused[7];
1308 + u8 __unused[3];
1309 };
1310
1311 struct st_switch_to_data { /* A process was switched to on a given CPU. */
1312 u64 when; /* When did this occur? */
1313 u32 exec_time; /* Time the current job has executed. */
1314 - u8 __unused[4];
1315 -
1316 };
1317
1318 struct st_switch_away_data { /* A process was switched away from on a given CPU. */
1319 @@ -52,11 +49,10 @@ struct st_switch_away_data { /* A process was switched away from on a given CPU.
1320
1321 struct st_completion_data { /* A job completed. */
1322 u64 when;
1323 - u8 forced:1; /* Set to 1 if job overran and kernel advanced to the
1324 + u64 forced:1; /* Set to 1 if job overran and kernel advanced to the
1325 * next task automatically; set to 0 otherwise.
1326 */
1327 - u8 __uflags:7;
1328 - u8 __unused[7];
1329 + u64 exec_time:63;
1330 };
1331
1332 struct st_block_data { /* A task blocks. */
1333 @@ -69,6 +65,16 @@ struct st_resume_data { /* A task resumes. */
1334 u64 __unused;
1335 };
1336
1337 +struct st_np_enter_data { /* A task becomes non-preemptable. */
1338 + u64 when;
1339 + u64 __unused;
1340 +};
1341 +
1342 +struct st_np_exit_data { /* A task becomes preemptable again. */
1343 + u64 when;
1344 + u64 __unused;
1345 +};
1346 +
1347 struct st_action_data {
1348 u64 when;
1349 u8 action;
1350 @@ -94,7 +100,10 @@ typedef enum {
1351 ST_BLOCK,
1352 ST_RESUME,
1353 ST_ACTION,
1354 - ST_SYS_RELEASE
1355 + ST_SYS_RELEASE,
1356 + ST_NP_ENTER,
1357 + ST_NP_EXIT,
1358 + ST_INVALID
1359 } st_event_record_type_t;
1360
1361 struct st_event_record {
1362 @@ -113,6 +122,8 @@ struct st_event_record {
1363 DATA(resume);
1364 DATA(action);
1365 DATA(sys_release);
1366 + DATA(np_enter);
1367 + DATA(np_exit);
1368 } data;
1369 };
1370
1371 diff --git include/litmus/trace.h include/litmus/trace.h
1372 index 6017872..24ca412 100644
1373 --- include/litmus/trace.h
1374 +++ include/litmus/trace.h
1375 @@ -118,6 +118,9 @@ feather_callback void save_cpu_task_latency(unsigned long event, unsigned long w
1376 #define TS_TICK_START(t) CPU_TTIMESTAMP(110, t)
1377 #define TS_TICK_END(t) CPU_TTIMESTAMP(111, t)
1378
1379 +#define TS_RELEASE_C_START CPU_DTIMESTAMP(108, TSK_RT)
1380 +#define TS_RELEASE_C_END CPU_DTIMESTAMP(109, TSK_RT)
1381 +
1382 #define TS_QUANTUM_BOUNDARY_START CPU_TIMESTAMP_CUR(112)
1383 #define TS_QUANTUM_BOUNDARY_END CPU_TIMESTAMP_CUR(113)
1384
1385 @@ -137,6 +140,17 @@ feather_callback void save_cpu_task_latency(unsigned long event, unsigned long w
1386 #define TS_SEND_RESCHED_START(c) MSG_TIMESTAMP_SENT(190, c)
1387 #define TS_SEND_RESCHED_END MSG_TIMESTAMP_RECEIVED(191)
1388
1389 -#define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when))
1390 +#define TS_ISR_START CPU_TIMESTAMP_CUR(192)
1391 +#define TS_ISR_END CPU_TIMESTAMP_CUR(193)
1392 +
1393 +#define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when))
1394 +#define TS_RELEASE_LATENCY_A(when) CPU_LTIMESTAMP(209, &(when))
1395 +#define TS_RELEASE_LATENCY_B(when) CPU_LTIMESTAMP(210, &(when))
1396 +#define TS_RELEASE_LATENCY_C(when) CPU_LTIMESTAMP(211, &(when))
1397 +
1398 +#define TS_SCHED_A_START CPU_DTIMESTAMP(212, TSK_UNKNOWN)
1399 +#define TS_SCHED_A_END(t) CPU_TTIMESTAMP(213, t)
1400 +#define TS_SCHED_C_START CPU_DTIMESTAMP(214, TSK_UNKNOWN)
1401 +#define TS_SCHED_C_END(t) CPU_TTIMESTAMP(215, t)
1402
1403 #endif /* !_SYS_TRACE_H_ */
1404 diff --git include/litmus/unistd_32.h include/litmus/unistd_32.h
1405 index 94264c2..664f73f 100644
1406 --- include/litmus/unistd_32.h
1407 +++ include/litmus/unistd_32.h
1408 @@ -17,5 +17,14 @@
1409 #define __NR_wait_for_ts_release __LSC(9)
1410 #define __NR_release_ts __LSC(10)
1411 #define __NR_null_call __LSC(11)
1412 +#define __NR_get_current_budget __LSC(12)
1413 +#define __NR_reservation_create __LSC(13)
1414 +#define __NR_reservation_destroy __LSC(14)
1415 +#define __NR_set_mc2_task_param __LSC(15)
1416 +#define __NR_set_page_color __LSC(16)
1417 +#define __NR_test_call __LSC(17)
1418 +#define __NR_run_test __LSC(18)
1419 +#define __NR_lock_buffer __LSC(19)
1420 +#define __NR_recolor_mem __LSC(20)
1421
1422 -#define NR_litmus_syscalls 12
1423 +#define NR_litmus_syscalls 21
1424 diff --git include/litmus/unistd_64.h include/litmus/unistd_64.h
1425 index d5ced0d..994e8a5 100644
1426 --- include/litmus/unistd_64.h
1427 +++ include/litmus/unistd_64.h
1428 @@ -29,5 +29,23 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release)
1429 __SYSCALL(__NR_release_ts, sys_release_ts)
1430 #define __NR_null_call __LSC(11)
1431 __SYSCALL(__NR_null_call, sys_null_call)
1432 +#define __NR_get_current_budget __LSC(12)
1433 +__SYSCALL(__NR_get_current_budget, sys_get_current_budget)
1434 +#define __NR_reservation_create __LSC(13)
1435 +__SYSCALL(__NR_reservation_create, sys_reservation_create)
1436 +#define __NR_reservation_destroy __LSC(14)
1437 +__SYSCALL(__NR_reservation_destroy, sys_reservation_destroy)
1438 +#define __NR_set_mc2_task_param __LSC(15)
1439 +__SYSCALL(__NR_set_mc2_task_param, sys_set_mc2_task_param)
1440 +#define __NR_set_page_color __LSC(16)
1441 +__SYSCALL(__NR_set_page_color, sys_set_page_color)
1442 +#define __NR_test_call __LSC(17)
1443 +__SYSCALL(__NR_test_call, sys_test_call)
1444 +#define __NR_run_test __LSC(18)
1445 +__SYSCALL(__NR_run_test, sys_run_test)
1446 +#define __NR_lock_buffer __LSC(19)
1447 +__SYSCALL(__NR_lock_buffer, sys_lock_buffer)
1448 +#define __NR_recolor_mem __LSC(20)
1449 +__SYSCALL(__NR_recolor_mem, sys_recolor_mem)
1450
1451 -#define NR_litmus_syscalls 12
1452 +#define NR_litmus_syscalls 21
1453 diff --git ipc/mqueue.c ipc/mqueue.c
1454 index 3aaea7f..4cb1b7f 100644
1455 --- ipc/mqueue.c
1456 +++ ipc/mqueue.c
1457 @@ -1004,7 +1004,7 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr,
1458
1459 /* First try to allocate memory, before doing anything with
1460 * existing queues. */
1461 - msg_ptr = load_msg(u_msg_ptr, msg_len);
1462 + msg_ptr = load_msg(u_msg_ptr, msg_len, 0);
1463 if (IS_ERR(msg_ptr)) {
1464 ret = PTR_ERR(msg_ptr);
1465 goto out_fput;
1466 diff --git ipc/msg.c ipc/msg.c
1467 index 2b6fdbb..da2b3e9 100644
1468 --- ipc/msg.c
1469 +++ ipc/msg.c
1470 @@ -38,6 +38,10 @@
1471 #include <linux/nsproxy.h>
1472 #include <linux/ipc_namespace.h>
1473
1474 +#include <litmus/trace.h>
1475 +#include <litmus/litmus.h>
1476 +#include <asm/cacheflush.h>
1477 +
1478 #include <asm/current.h>
1479 #include <linux/uaccess.h>
1480 #include "util.h"
1481 @@ -619,7 +623,7 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext,
1482 if (mtype < 1)
1483 return -EINVAL;
1484
1485 - msg = load_msg(mtext, msgsz);
1486 + msg = load_msg(mtext, msgsz, mtype);
1487 if (IS_ERR(msg))
1488 return PTR_ERR(msg);
1489
1490 @@ -752,13 +756,13 @@ static long do_msg_fill(void __user *dest, struct msg_msg *msg, size_t bufsz)
1491 {
1492 struct msgbuf __user *msgp = dest;
1493 size_t msgsz;
1494 -
1495 if (put_user(msg->m_type, &msgp->mtype))
1496 return -EFAULT;
1497 -
1498 msgsz = (bufsz > msg->m_ts) ? msg->m_ts : bufsz;
1499 +
1500 if (store_msg(msgp->mtext, msg, msgsz))
1501 return -EFAULT;
1502 +
1503 return msgsz;
1504 }
1505
1506 @@ -976,7 +980,6 @@ out_unlock1:
1507 free_copy(copy);
1508 return PTR_ERR(msg);
1509 }
1510 -
1511 bufsz = msg_handler(buf, msg, bufsz);
1512 free_msg(msg);
1513
1514 diff --git ipc/msgutil.c ipc/msgutil.c
1515 index 2b49159..1ffc1b2 100644
1516 --- ipc/msgutil.c
1517 +++ ipc/msgutil.c
1518 @@ -18,6 +18,12 @@
1519 #include <linux/utsname.h>
1520 #include <linux/proc_ns.h>
1521 #include <linux/uaccess.h>
1522 +#include <linux/dma-mapping.h>
1523 +#include <linux/mempool.h>
1524 +#include <litmus/cache_proc.h>
1525 +#include <litmus/trace.h>
1526 +#include <litmus/litmus.h>
1527 +#include <asm/cacheflush.h>
1528
1529 #include "util.h"
1530
1531 @@ -42,20 +48,35 @@ atomic_t nr_ipc_ns = ATOMIC_INIT(1);
1532 struct msg_msgseg {
1533 struct msg_msgseg *next;
1534 /* the next part of the message follows immediately */
1535 + dma_addr_t seg_handle;
1536 + size_t seg_len;
1537 };
1538
1539 #define DATALEN_MSG ((size_t)PAGE_SIZE-sizeof(struct msg_msg))
1540 #define DATALEN_SEG ((size_t)PAGE_SIZE-sizeof(struct msg_msgseg))
1541
1542 +static dma_addr_t handle;
1543
1544 -static struct msg_msg *alloc_msg(size_t len)
1545 +mempool_t *msgpool;
1546 +extern void *msgvaddr;
1547 +
1548 +static struct msg_msg *alloc_msg(size_t len, long mtype)
1549 {
1550 struct msg_msg *msg;
1551 struct msg_msgseg **pseg;
1552 size_t alen;
1553 + int n_seg = 0;
1554
1555 alen = min(len, DATALEN_MSG);
1556 - msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL);
1557 + if (mtype == 1) {
1558 + msg = dma_alloc_coherent(NULL, sizeof(*msg) + alen, &handle, GFP_KERNEL|GFP_COLOR);
1559 + msg->handle = handle;
1560 + msg->alloc_len = sizeof(*msg) + alen;
1561 + } else if (mtype == 2) {
1562 + msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL|GFP_COLOR);
1563 + } else {
1564 + msg = msgvaddr;
1565 + }
1566 if (msg == NULL)
1567 return NULL;
1568
1569 @@ -67,7 +88,16 @@ static struct msg_msg *alloc_msg(size_t len)
1570 while (len > 0) {
1571 struct msg_msgseg *seg;
1572 alen = min(len, DATALEN_SEG);
1573 - seg = kmalloc(sizeof(*seg) + alen, GFP_KERNEL);
1574 + if (mtype == 1) {
1575 + seg = dma_alloc_coherent(NULL, sizeof(*seg) + alen, &handle, GFP_KERNEL|GFP_COLOR);
1576 + seg->seg_handle = handle;
1577 + seg->seg_len = alen;
1578 + } else if (mtype == 2) {
1579 + seg = kmalloc(sizeof(*seg) + alen, GFP_KERNEL|GFP_COLOR);
1580 + } else {
1581 + n_seg++;
1582 + seg = msgvaddr + PAGE_SIZE*n_seg;
1583 + }
1584 if (seg == NULL)
1585 goto out_err;
1586 *pseg = seg;
1587 @@ -83,18 +113,19 @@ out_err:
1588 return NULL;
1589 }
1590
1591 -struct msg_msg *load_msg(const void __user *src, size_t len)
1592 +struct msg_msg *load_msg(const void __user *src, size_t len, long mtype)
1593 {
1594 struct msg_msg *msg;
1595 struct msg_msgseg *seg;
1596 int err = -EFAULT;
1597 size_t alen;
1598
1599 - msg = alloc_msg(len);
1600 + msg = alloc_msg(len, mtype);
1601 if (msg == NULL)
1602 return ERR_PTR(-ENOMEM);
1603
1604 alen = min(len, DATALEN_MSG);
1605 +
1606 if (copy_from_user(msg + 1, src, alen))
1607 goto out_err;
1608
1609 @@ -105,6 +136,9 @@ struct msg_msg *load_msg(const void __user *src, size_t len)
1610 if (copy_from_user(seg + 1, src, alen))
1611 goto out_err;
1612 }
1613 + if (mtype == 3) {
1614 + cache_lockdown(0xFFFF8000, smp_processor_id());
1615 + }
1616
1617 err = security_msg_msg_alloc(msg);
1618 if (err)
1619 @@ -172,14 +206,22 @@ int store_msg(void __user *dest, struct msg_msg *msg, size_t len)
1620 void free_msg(struct msg_msg *msg)
1621 {
1622 struct msg_msgseg *seg;
1623 -
1624 + long mtype = msg->m_type;
1625 +
1626 security_msg_msg_free(msg);
1627
1628 seg = msg->next;
1629 - kfree(msg);
1630 + if (mtype == 1) {
1631 + dma_free_coherent(NULL, msg->alloc_len, msg, msg->handle);
1632 + } else if (mtype != 3) {
1633 + kfree(msg);
1634 + }
1635 while (seg != NULL) {
1636 struct msg_msgseg *tmp = seg->next;
1637 - kfree(seg);
1638 + if (mtype == 1) {
1639 + dma_free_coherent(NULL, sizeof(*seg)+(seg->seg_len), seg, seg->seg_handle);
1640 + } else if (mtype != 3)
1641 + kfree(seg);
1642 seg = tmp;
1643 }
1644 }
1645 diff --git ipc/util.h ipc/util.h
1646 index 1a5a0fc..9a29983 100644
1647 --- ipc/util.h
1648 +++ ipc/util.h
1649 @@ -149,7 +149,7 @@ int ipc_parse_version(int *cmd);
1650 #endif
1651
1652 extern void free_msg(struct msg_msg *msg);
1653 -extern struct msg_msg *load_msg(const void __user *src, size_t len);
1654 +extern struct msg_msg *load_msg(const void __user *src, size_t len, long mtype);
1655 extern struct msg_msg *copy_msg(struct msg_msg *src, struct msg_msg *dst);
1656 extern int store_msg(void __user *dest, struct msg_msg *msg, size_t len);
1657
1658 diff --git kernel/sched/litmus.c kernel/sched/litmus.c
1659 index 9d58690..cd36358 100644
1660 --- kernel/sched/litmus.c
1661 +++ kernel/sched/litmus.c
1662 @@ -20,8 +20,9 @@ static void update_time_litmus(struct rq *rq, struct task_struct *p)
1663 /* task counter */
1664 p->se.sum_exec_runtime += delta;
1665 if (delta) {
1666 - TRACE_TASK(p, "charged %llu exec time (total:%llu, rem:%llu)\n",
1667 - delta, p->rt_param.job_params.exec_time, budget_remaining(p));
1668 + //TRACE_TASK(p, "charged %llu exec time (total:%llu, rem:%llu)\n",
1669 + //delta, p->rt_param.job_params.exec_time, budget_remaining(p));
1670 + ;
1671 }
1672 /* sched_clock() */
1673 p->se.exec_start = rq->clock;
1674 diff --git litmus/Kconfig litmus/Kconfig
1675 index babb43d..8f38bc3 100644
1676 --- litmus/Kconfig
1677 +++ litmus/Kconfig
1678 @@ -372,4 +372,26 @@ config PREEMPT_STATE_TRACE
1679
1680 endmenu
1681
1682 +menu "Memory Management"
1683 +
1684 +choice
1685 + prompt "UVC buffer allocation"
1686 + default UVC_ALLOC_LEVEL_C
1687 + help
1688 + Determine the location of frame buffers used by UVC (USB Video Class) devices. Other data structures such as URB (USB Request Block) and V4L2 (Video for Linux Version 2) use Level-C/OS banks.
1689 +
1690 + config UVC_ALLOC_LEVEL_C
1691 + bool "Level-C/OS"
1692 + help
1693 + Allocate frame buffers in Level-C/OS banks.
1694 +
1695 + config UVC_ALLOC_LEVEL_AB
1696 + bool "Level-A/B"
1697 + help
1698 + Allocate frame buffers in the task's Level-A/B bank. If this option is selected, the task that uses a camera must be a Level-A or -B task.
1699 +
1700 +endchoice
1701 +
1702 +endmenu
1703 +
1704 endmenu
1705 diff --git litmus/Makefile litmus/Makefile
1706 index 7970cd5..29ae4b0 100644
1707 --- litmus/Makefile
1708 +++ litmus/Makefile
1709 @@ -11,6 +11,7 @@ obj-y = sched_plugin.o litmus.o \
1710 sync.o \
1711 rt_domain.o \
1712 edf_common.o \
1713 + mc2_common.o \
1714 fp_common.o \
1715 fdso.o \
1716 locking.o \
1717 @@ -19,13 +20,20 @@ obj-y = sched_plugin.o litmus.o \
1718 binheap.o \
1719 ctrldev.o \
1720 uncachedev.o \
1721 + reservation.o \
1722 + polling_reservations.o \
1723 sched_gsn_edf.o \
1724 sched_psn_edf.o \
1725 - sched_pfp.o
1726 + sched_pfp.o \
1727 + sched_mc2.o \
1728 + bank_proc.o \
1729 + color_shm.o \
1730 + replicate_lib.o \
1731 + cache_proc.o \
1732 + page_dev.o
1733
1734 obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o
1735 obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o
1736 -
1737 obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o
1738 obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o
1739 obj-$(CONFIG_SCHED_DEBUG_TRACE) += sched_trace.o
1740 diff --git litmus/bank_proc.c litmus/bank_proc.c
1741 new file mode 100644
1742 index 0000000..3c4f703
1743 --- /dev/null
1744 +++ litmus/bank_proc.c
1745 @@ -0,0 +1,775 @@
1746 +/*
1747 + * bank_proc.c -- Implementation of the page coloring for cache and bank partition.
1748 + * The file will keep a pool of colored pages. Users can require pages with
1749 + * specific color or bank number.
1750 + * Part of the code is modified from Jonathan Herman's code
1751 + */
1752 +#include <linux/init.h>
1753 +#include <linux/types.h>
1754 +#include <linux/kernel.h>
1755 +#include <linux/module.h>
1756 +#include <linux/sysctl.h>
1757 +#include <linux/slab.h>
1758 +#include <linux/io.h>
1759 +#include <linux/mutex.h>
1760 +#include <linux/mm.h>
1761 +#include <linux/random.h>
1762 +
1763 +#include <litmus/litmus_proc.h>
1764 +#include <litmus/sched_trace.h>
1765 +#include <litmus/litmus.h>
1766 +
1767 +#define LITMUS_LOCKDEP_NAME_MAX_LEN 50
1768 +
1769 +// This Address Decoding is used in imx6-sabredsd platform
1770 +#define BANK_MASK 0x38000000
1771 +#define BANK_SHIFT 27
1772 +#define CACHE_MASK 0x0000f000
1773 +#define CACHE_SHIFT 12
1774 +
1775 +#define PAGES_PER_COLOR 2000
1776 +#define NUM_BANKS 8
1777 +#define NUM_COLORS 16
1778 +
1779 +unsigned int NUM_PAGE_LIST; //8*16
1780 +
1781 +unsigned int number_banks;
1782 +unsigned int number_cachecolors;
1783 +
1784 +unsigned int set_partition_max = 0x0000ffff;
1785 +unsigned int set_partition_min = 0;
1786 +unsigned int bank_partition_max = 0x000000ff;
1787 +unsigned int bank_partition_min = 0;
1788 +
1789 +int show_page_pool = 0;
1790 +int refill_page_pool = 0;
1791 +spinlock_t reclaim_lock;
1792 +
1793 +unsigned int set_partition[9] = {
1794 + 0x00000003, /* Core 0, and Level A*/
1795 + 0x00000003, /* Core 0, and Level B*/
1796 + 0x0000000C, /* Core 1, and Level A*/
1797 + 0x0000000C, /* Core 1, and Level B*/
1798 + 0x00000030, /* Core 2, and Level A*/
1799 + 0x00000030, /* Core 2, and Level B*/
1800 + 0x000000C0, /* Core 3, and Level A*/
1801 + 0x000000C0, /* Core 3, and Level B*/
1802 + 0x0000ff00, /* Level C */
1803 +};
1804 +
1805 +unsigned int bank_partition[9] = {
1806 + 0x00000010, /* Core 0, and Level A*/
1807 + 0x00000010, /* Core 0, and Level B*/
1808 + 0x00000020, /* Core 1, and Level A*/
1809 + 0x00000020, /* Core 1, and Level B*/
1810 + 0x00000040, /* Core 2, and Level A*/
1811 + 0x00000040, /* Core 2, and Level B*/
1812 + 0x00000080, /* Core 3, and Level A*/
1813 + 0x00000080, /* Core 3, and Level B*/
1814 + 0x0000000c, /* Level C */
1815 +};
1816 +
1817 +unsigned int set_index[9] = {
1818 + 0, 0, 0, 0, 0, 0, 0, 0, 0
1819 +};
1820 +
1821 +unsigned int bank_index[9] = {
1822 + 0, 0, 0, 0, 0, 0, 0, 0, 0
1823 +};
1824 +
1825 +int node_index[9] = {
1826 + -1, -1, -1, -1, -1, -1, -1, -1, -1
1827 +};
1828 +
1829 +struct mutex void_lockdown_proc;
1830 +
1831 +/*
1832 + * Every page list should contain a lock, a list, and a number recording how many pages it store
1833 + */
1834 +struct color_group {
1835 + spinlock_t lock;
1836 + char _lock_name[LITMUS_LOCKDEP_NAME_MAX_LEN];
1837 + struct list_head list;
1838 + atomic_t nr_pages;
1839 +};
1840 +
1841 +
1842 +static struct color_group *color_groups;
1843 +
1844 +/*
1845 + * Naive function to count the number of 1's
1846 + */
1847 +unsigned int counting_one_set(unsigned int v)
1848 +{
1849 + unsigned int c; // c accumulates the total bits set in v
1850 +
1851 + for (c = 0; v; v >>= 1)
1852 + {
1853 + c += v & 1;
1854 + }
1855 + return c;
1856 +}
1857 +
1858 +unsigned int two_exp(unsigned int e)
1859 +{
1860 + unsigned int v = 1;
1861 + for (; e>0; e-- )
1862 + {
1863 + v=v*2;
1864 + }
1865 + return v;
1866 +}
1867 +
1868 +/* helper functions to find the next colored pool index */
1869 +static inline unsigned int first_index(unsigned long node)
1870 +{
1871 + unsigned int bank_no = 0, color_no = 0;
1872 +
1873 + while(bank_no < NUM_BANKS) {
1874 + if ((bank_partition[node]>>bank_no) & 0x1)
1875 + break;
1876 + bank_no++;
1877 + }
1878 + while(color_no < NUM_COLORS) {
1879 + if ((set_partition[node]>>color_no) & 0x1)
1880 + break;
1881 + color_no++;
1882 + }
1883 + return NUM_COLORS*bank_no + color_no;
1884 +}
1885 +
1886 +static inline unsigned int last_index(unsigned long node)
1887 +{
1888 + unsigned int bank_no = NUM_BANKS-1, color_no = NUM_COLORS-1;
1889 +
1890 + while(bank_no >= 0) {
1891 + if ((bank_partition[node]>>bank_no) & 0x1)
1892 + break;
1893 + bank_no--;
1894 + }
1895 + while(color_no >= 0) {
1896 + if ((set_partition[node]>>color_no) & 0x1)
1897 + break;
1898 + color_no--;
1899 + }
1900 + return NUM_COLORS*bank_no + color_no;
1901 +}
1902 +
1903 +static inline unsigned int next_color(unsigned long node, unsigned int current_color)
1904 +{
1905 + int try = 0, ret = 0;
1906 + current_color++;
1907 + if (current_color == NUM_COLORS) {
1908 + current_color = 0;
1909 + ret = 1;
1910 + }
1911 +
1912 + while (try < NUM_COLORS) {
1913 + if ((set_partition[node]>>current_color)&0x1)
1914 + break;
1915 + current_color++;
1916 + if (current_color == NUM_COLORS) {
1917 + current_color = 0;
1918 + ret = 1;
1919 + }
1920 + try++;
1921 + }
1922 + if (!ret)
1923 + return current_color;
1924 + else
1925 + return current_color + NUM_COLORS;
1926 +}
1927 +
1928 +static inline unsigned int next_bank(unsigned long node, unsigned int current_bank)
1929 +{
1930 + int try = 0;
1931 + current_bank++;
1932 + if (current_bank == NUM_BANKS) {
1933 + current_bank = 0;
1934 + }
1935 +
1936 + while (try < NUM_BANKS) {
1937 + if ((bank_partition[node]>>current_bank)&0x1)
1938 + break;
1939 + current_bank++;
1940 + if (current_bank == NUM_BANKS) {
1941 + current_bank = 0;
1942 + }
1943 + try++;
1944 + }
1945 + return current_bank;
1946 +}
1947 +
1948 +static inline unsigned int get_next_index(unsigned long node, unsigned int current_index)
1949 +{
1950 + unsigned int bank_no, color_no, color_ret, bank_ret;
1951 + bank_no = current_index>>4; // 2^4 = 16 colors
1952 + color_no = current_index - bank_no*NUM_COLORS;
1953 + bank_ret = bank_no;
1954 + color_ret = next_color(node, color_no);
1955 + if (color_ret >= NUM_COLORS) {
1956 + // next bank
1957 + color_ret -= NUM_COLORS;
1958 + bank_ret = next_bank(node, bank_no);
1959 + }
1960 +
1961 + return bank_ret * NUM_COLORS + color_ret;
1962 +}
1963 +
1964 +/* Decoding page color, 0~15 */
1965 +static inline unsigned int page_color(struct page *page)
1966 +{
1967 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
1968 +}
1969 +
1970 +/* Decoding page bank number, 0~7 */
1971 +static inline unsigned int page_bank(struct page *page)
1972 +{
1973 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
1974 +}
1975 +
1976 +static inline unsigned int page_list_index(struct page *page)
1977 +{
1978 + unsigned int idx;
1979 + idx = (page_color(page) + page_bank(page)*(number_cachecolors));
1980 +
1981 + return idx;
1982 +}
1983 +
1984 +
1985 +
1986 +/*
1987 + * It is used to determine the smallest number of page lists.
1988 + */
1989 +static unsigned long smallest_nr_pages(void)
1990 +{
1991 + unsigned long i, min_pages;
1992 + struct color_group *cgroup;
1993 + cgroup = &color_groups[16*2];
1994 + min_pages =atomic_read(&cgroup->nr_pages);
1995 + for (i = 16*2; i < NUM_PAGE_LIST; ++i) {
1996 + cgroup = &color_groups[i];
1997 + if (atomic_read(&cgroup->nr_pages) < min_pages)
1998 + min_pages = atomic_read(&cgroup->nr_pages);
1999 + }
2000 + return min_pages;
2001 +}
2002 +
2003 +static void show_nr_pages(void)
2004 +{
2005 + unsigned long i;
2006 + struct color_group *cgroup;
2007 + printk("show nr pages***************************************\n");
2008 + for (i = 0; i < NUM_PAGE_LIST; ++i) {
2009 + cgroup = &color_groups[i];
2010 + printk("(%03ld) = %03d, ", i, atomic_read(&cgroup->nr_pages));
2011 + if((i % 8) ==7) {
2012 + printk("\n");
2013 + }
2014 + }
2015 +}
2016 +
2017 +/*
2018 + * Add a page to current pool.
2019 + */
2020 +void add_page_to_color_list(struct page *page)
2021 +{
2022 + const unsigned long color = page_list_index(page);
2023 + struct color_group *cgroup = &color_groups[color];
2024 + BUG_ON(in_list(&page->lru) || PageLRU(page));
2025 + BUG_ON(page_count(page) > 1);
2026 + spin_lock(&cgroup->lock);
2027 + list_add_tail(&page->lru, &cgroup->list);
2028 + atomic_inc(&cgroup->nr_pages);
2029 + SetPageLRU(page);
2030 + spin_unlock(&cgroup->lock);
2031 +}
2032 +
2033 +/*
2034 + * Replenish the page pool.
2035 + * If the newly allocate page is what we want, it will be pushed to the correct page list
2036 + * otherwise, it will be freed.
2037 + * A user needs to invoke this function until the page pool has enough pages.
2038 + */
2039 +static int do_add_pages(void)
2040 +{
2041 + struct page *page, *page_tmp;
2042 + LIST_HEAD(free_later);
2043 + unsigned long color;
2044 + int ret = 0;
2045 + int i = 0;
2046 + int free_counter = 0;
2047 + unsigned long counter[128]= {0};
2048 +
2049 + // until all the page lists contain enough pages
2050 + for (i=0; i< 1024*20;i++) {
2051 + page = alloc_page(GFP_HIGHUSER_MOVABLE);
2052 +
2053 + if (unlikely(!page)) {
2054 + printk(KERN_WARNING "Could not allocate pages.\n");
2055 + ret = -ENOMEM;
2056 + goto out;
2057 + }
2058 + color = page_list_index(page);
2059 + counter[color]++;
2060 + if (atomic_read(&color_groups[color].nr_pages) < PAGES_PER_COLOR && color>=0) {
2061 + add_page_to_color_list(page);
2062 + } else {
2063 + // Pages here will be freed later
2064 + list_add_tail(&page->lru, &free_later);
2065 + free_counter++;
2066 + }
2067 + }
2068 +
2069 + // Free the unwanted pages
2070 + list_for_each_entry_safe(page, page_tmp, &free_later, lru) {
2071 + list_del(&page->lru);
2072 + __free_page(page);
2073 + }
2074 +out:
2075 + return ret;
2076 +}
2077 +
2078 +/*
2079 + * Provide pages for replacement according cache color
2080 + * This should be the only implementation here
2081 + * This function should not be accessed by others directly.
2082 + *
2083 + */
2084 +static struct page *new_alloc_page_color( unsigned long color)
2085 +{
2086 +// printk("allocate new page color = %d\n", color);
2087 + struct color_group *cgroup;
2088 + struct page *rPage = NULL;
2089 +
2090 + if( (color <0) || (color)>(number_cachecolors*number_banks -1)) {
2091 + TRACE_CUR("Wrong color %lu\n", color);
2092 + goto out;
2093 + }
2094 +
2095 +
2096 + cgroup = &color_groups[color];
2097 + spin_lock(&cgroup->lock);
2098 + if (unlikely(!atomic_read(&cgroup->nr_pages))) {
2099 + TRACE_CUR("No free %lu colored pages.\n", color);
2100 + goto out_unlock;
2101 + }
2102 + rPage = list_first_entry(&cgroup->list, struct page, lru);
2103 + BUG_ON(page_count(rPage) > 1);
2104 + //get_page(rPage);
2105 + list_del(&rPage->lru);
2106 + atomic_dec(&cgroup->nr_pages);
2107 + ClearPageLRU(rPage);
2108 +out_unlock:
2109 + spin_unlock(&cgroup->lock);
2110 +out:
2111 + return rPage;
2112 +}
2113 +
2114 +struct page* get_colored_page(unsigned long color)
2115 +{
2116 + return new_alloc_page_color(color);
2117 +}
2118 +
2119 +/*
2120 + * provide pages for replacement according to
2121 + * node = 0 for Level A tasks in Cpu 0
2122 + * node = 1 for Level B tasks in Cpu 0
2123 + * node = 2 for Level A tasks in Cpu 1
2124 + * node = 3 for Level B tasks in Cpu 1
2125 + * node = 4 for Level A tasks in Cpu 2
2126 + * node = 5 for Level B tasks in Cpu 2
2127 + * node = 6 for Level A tasks in Cpu 3
2128 + * node = 7 for Level B tasks in Cpu 3
2129 + * node = 8 for Level C tasks
2130 + */
2131 +struct page *new_alloc_page(struct page *page, unsigned long node, int **x)
2132 +{
2133 + struct page *rPage = NULL;
2134 + int try = 0;
2135 + unsigned int idx;
2136 +
2137 + if (node_index[node] == -1)
2138 + idx = first_index(node);
2139 + else
2140 + idx = node_index[node];
2141 +
2142 + BUG_ON(idx<0 || idx>127);
2143 + rPage = new_alloc_page_color(idx);
2144 + if (node_index[node] == last_index(node))
2145 + node_index[node] = first_index(node);
2146 + else
2147 + node_index[node]++;
2148 +
2149 + while (!rPage) {
2150 + try++;
2151 + if (try>=256)
2152 + break;
2153 + idx = get_next_index(node, idx);
2154 + printk(KERN_ALERT "try = %d out of page! requesting node = %ld, idx = %d\n", try, node, idx);
2155 + BUG_ON(idx<0 || idx>127);
2156 + rPage = new_alloc_page_color(idx);
2157 + }
2158 + node_index[node] = idx;
2159 + return rPage;
2160 +}
2161 +
2162 +
2163 +/*
2164 + * Reclaim pages.
2165 + */
2166 +void reclaim_page(struct page *page)
2167 +{
2168 + const unsigned long color = page_list_index(page);
2169 + spin_lock(&reclaim_lock);
2170 + put_page(page);
2171 + add_page_to_color_list(page);
2172 +
2173 + spin_unlock(&reclaim_lock);
2174 + printk("Reclaimed page(%ld) = color %x, bank %x, [color] =%d \n", color, page_color(page), page_bank(page), atomic_read(&color_groups[color].nr_pages));
2175 +}
2176 +
2177 +
2178 +/*
2179 + * Initialize the numbers of banks and cache colors
2180 + */
2181 +static void __init init_variables(void)
2182 +{
2183 + number_banks = counting_one_set(BANK_MASK);
2184 + number_banks = two_exp(number_banks);
2185 +
2186 + number_cachecolors = counting_one_set(CACHE_MASK);
2187 + number_cachecolors = two_exp(number_cachecolors);
2188 + NUM_PAGE_LIST = number_banks * number_cachecolors;
2189 + printk(KERN_WARNING "number of banks = %d, number of cachecolors=%d\n", number_banks, number_cachecolors);
2190 + mutex_init(&void_lockdown_proc);
2191 + spin_lock_init(&reclaim_lock);
2192 +
2193 +}
2194 +
2195 +
2196 +/*
2197 + * Initialize the page pool
2198 + */
2199 +static int __init init_color_groups(void)
2200 +{
2201 + struct color_group *cgroup;
2202 + unsigned long i;
2203 + int err = 0;
2204 +
2205 + printk("NUM_PAGE_LIST = %d\n", NUM_PAGE_LIST);
2206 + color_groups = kmalloc(NUM_PAGE_LIST *sizeof(struct color_group), GFP_KERNEL);
2207 +
2208 + if (!color_groups) {
2209 + printk(KERN_WARNING "Could not allocate color groups.\n");
2210 + err = -ENOMEM;
2211 + }else{
2212 +
2213 + for (i = 0; i < NUM_PAGE_LIST; ++i) {
2214 + cgroup = &color_groups[i];
2215 + atomic_set(&cgroup->nr_pages, 0);
2216 + INIT_LIST_HEAD(&cgroup->list);
2217 + spin_lock_init(&cgroup->lock);
2218 + }
2219 + }
2220 + return err;
2221 +}
2222 +
2223 +int set_partition_handler(struct ctl_table *table, int write, void __user *buffer,
2224 + size_t *lenp, loff_t *ppos)
2225 +{
2226 + int ret = 0, i = 0;
2227 + mutex_lock(&void_lockdown_proc);
2228 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2229 + if (ret)
2230 + goto out;
2231 + if (write) {
2232 + printk("New set Partition : \n");
2233 + for(i =0;i <9;i++)
2234 + {
2235 + set_index[i] = 0;
2236 + printk("set[%d] = %x \n", i, set_partition[i]);
2237 + }
2238 + }
2239 +out:
2240 + mutex_unlock(&void_lockdown_proc);
2241 + return ret;
2242 +}
2243 +
2244 +int bank_partition_handler(struct ctl_table *table, int write, void __user *buffer,
2245 + size_t *lenp, loff_t *ppos)
2246 +{
2247 + int ret = 0, i = 0;
2248 + mutex_lock(&void_lockdown_proc);
2249 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2250 + if (ret)
2251 + goto out;
2252 + if (write) {
2253 + for(i =0;i <9;i++)
2254 + {
2255 + bank_index[i] = 0;
2256 + }
2257 + }
2258 +out:
2259 + mutex_unlock(&void_lockdown_proc);
2260 + return ret;
2261 +}
2262 +
2263 +int show_page_pool_handler(struct ctl_table *table, int write, void __user *buffer,
2264 + size_t *lenp, loff_t *ppos)
2265 +{
2266 + int ret = 0;
2267 + mutex_lock(&void_lockdown_proc);
2268 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2269 + if (ret)
2270 + goto out;
2271 + if (write) {
2272 + show_nr_pages();
2273 + }
2274 +out:
2275 + mutex_unlock(&void_lockdown_proc);
2276 + return ret;
2277 +}
2278 +
2279 +int refill_page_pool_handler(struct ctl_table *table, int write, void __user *buffer,
2280 + size_t *lenp, loff_t *ppos)
2281 +{
2282 + int ret = 0;
2283 + mutex_lock(&void_lockdown_proc);
2284 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2285 + if (ret)
2286 + goto out;
2287 + if (write) {
2288 + do_add_pages();
2289 + show_nr_pages();
2290 + }
2291 +out:
2292 + mutex_unlock(&void_lockdown_proc);
2293 + return ret;
2294 +}
2295 +
2296 +/*
2297 +static struct ctl_table cache_table[] =
2298 +{
2299 +
2300 + {
2301 + .procname = "C0_LA_set",
2302 + .mode = 0666,
2303 + .proc_handler = set_partition_handler,
2304 + .data = &set_partition[0],
2305 + .maxlen = sizeof(set_partition[0]),
2306 + .extra1 = &set_partition_min,
2307 + .extra2 = &set_partition_max,
2308 + },
2309 + {
2310 + .procname = "C0_LB_set",
2311 + .mode = 0666,
2312 + .proc_handler = set_partition_handler,
2313 + .data = &set_partition[1],
2314 + .maxlen = sizeof(set_partition[1]),
2315 + .extra1 = &set_partition_min,
2316 + .extra2 = &set_partition_max,
2317 + },
2318 + {
2319 + .procname = "C1_LA_set",
2320 + .mode = 0666,
2321 + .proc_handler = set_partition_handler,
2322 + .data = &set_partition[2],
2323 + .maxlen = sizeof(set_partition[2]),
2324 + .extra1 = &set_partition_min,
2325 + .extra2 = &set_partition_max,
2326 + },
2327 + {
2328 + .procname = "C1_LB_set",
2329 + .mode = 0666,
2330 + .proc_handler = set_partition_handler,
2331 + .data = &set_partition[3],
2332 + .maxlen = sizeof(set_partition[3]),
2333 + .extra1 = &set_partition_min,
2334 + .extra2 = &set_partition_max,
2335 + },
2336 + {
2337 + .procname = "C2_LA_set",
2338 + .mode = 0666,
2339 + .proc_handler = set_partition_handler,
2340 + .data = &set_partition[4],
2341 + .maxlen = sizeof(set_partition[4]),
2342 + .extra1 = &set_partition_min,
2343 + .extra2 = &set_partition_max,
2344 + },
2345 + {
2346 + .procname = "C2_LB_set",
2347 + .mode = 0666,
2348 + .proc_handler = set_partition_handler,
2349 + .data = &set_partition[5],
2350 + .maxlen = sizeof(set_partition[5]),
2351 + .extra1 = &set_partition_min,
2352 + .extra2 = &set_partition_max,
2353 + },
2354 + {
2355 + .procname = "C3_LA_set",
2356 + .mode = 0666,
2357 + .proc_handler = set_partition_handler,
2358 + .data = &set_partition[6],
2359 + .maxlen = sizeof(set_partition[6]),
2360 + .extra1 = &set_partition_min,
2361 + .extra2 = &set_partition_max,
2362 + },
2363 + {
2364 + .procname = "C3_LB_set",
2365 + .mode = 0666,
2366 + .proc_handler = set_partition_handler,
2367 + .data = &set_partition[7],
2368 + .maxlen = sizeof(set_partition[7]),
2369 + .extra1 = &set_partition_min,
2370 + .extra2 = &set_partition_max,
2371 + },
2372 + {
2373 + .procname = "Call_LC_set",
2374 + .mode = 0666,
2375 + .proc_handler = set_partition_handler,
2376 + .data = &set_partition[8],
2377 + .maxlen = sizeof(set_partition[8]),
2378 + .extra1 = &set_partition_min,
2379 + .extra2 = &set_partition_max,
2380 + },
2381 + {
2382 + .procname = "C0_LA_bank",
2383 + .mode = 0666,
2384 + .proc_handler = bank_partition_handler,
2385 + .data = &bank_partition[0],
2386 + .maxlen = sizeof(set_partition[0]),
2387 + .extra1 = &bank_partition_min,
2388 + .extra2 = &bank_partition_max,
2389 + },
2390 + {
2391 + .procname = "C0_LB_bank",
2392 + .mode = 0666,
2393 + .proc_handler = bank_partition_handler,
2394 + .data = &bank_partition[1],
2395 + .maxlen = sizeof(set_partition[1]),
2396 + .extra1 = &bank_partition_min,
2397 + .extra2 = &bank_partition_max,
2398 + },
2399 + {
2400 + .procname = "C1_LA_bank",
2401 + .mode = 0666,
2402 + .proc_handler = bank_partition_handler,
2403 + .data = &bank_partition[2],
2404 + .maxlen = sizeof(set_partition[2]),
2405 + .extra1 = &bank_partition_min,
2406 + .extra2 = &bank_partition_max,
2407 + },
2408 + {
2409 + .procname = "C1_LB_bank",
2410 + .mode = 0666,
2411 + .proc_handler = bank_partition_handler,
2412 + .data = &bank_partition[3],
2413 + .maxlen = sizeof(set_partition[3]),
2414 + .extra1 = &bank_partition_min,
2415 + .extra2 = &bank_partition_max,
2416 + },
2417 + {
2418 + .procname = "C2_LA_bank",
2419 + .mode = 0666,
2420 + .proc_handler = bank_partition_handler,
2421 + .data = &bank_partition[4],
2422 + .maxlen = sizeof(set_partition[4]),
2423 + .extra1 = &bank_partition_min,
2424 + .extra2 = &bank_partition_max,
2425 + },
2426 + {
2427 + .procname = "C2_LB_bank",
2428 + .mode = 0666,
2429 + .proc_handler = bank_partition_handler,
2430 + .data = &bank_partition[5],
2431 + .maxlen = sizeof(set_partition[5]),
2432 + .extra1 = &bank_partition_min,
2433 + .extra2 = &bank_partition_max,
2434 + },
2435 + {
2436 + .procname = "C3_LA_bank",
2437 + .mode = 0666,
2438 + .proc_handler = bank_partition_handler,
2439 + .data = &bank_partition[6],
2440 + .maxlen = sizeof(set_partition[6]),
2441 + .extra1 = &bank_partition_min,
2442 + .extra2 = &bank_partition_max,
2443 + },
2444 + {
2445 + .procname = "C3_LB_bank",
2446 + .mode = 0666,
2447 + .proc_handler = bank_partition_handler,
2448 + .data = &bank_partition[7],
2449 + .maxlen = sizeof(set_partition[7]),
2450 + .extra1 = &bank_partition_min,
2451 + .extra2 = &bank_partition_max,
2452 + },
2453 + {
2454 + .procname = "Call_LC_bank",
2455 + .mode = 0666,
2456 + .proc_handler = bank_partition_handler,
2457 + .data = &bank_partition[8],
2458 + .maxlen = sizeof(set_partition[8]),
2459 + .extra1 = &bank_partition_min,
2460 + .extra2 = &bank_partition_max,
2461 + },
2462 + {
2463 + .procname = "show_page_pool",
2464 + .mode = 0666,
2465 + .proc_handler = show_page_pool_handler,
2466 + .data = &show_page_pool,
2467 + .maxlen = sizeof(show_page_pool),
2468 + }, {
2469 + .procname = "refill_page_pool",
2470 + .mode = 0666,
2471 + .proc_handler = refill_page_pool_handler,
2472 + .data = &refill_page_pool,
2473 + .maxlen = sizeof(refill_page_pool),
2474 + },
2475 + { }
2476 +};
2477 +*/
2478 +/*
2479 +static struct ctl_table litmus_dir_table[] = {
2480 + {
2481 + .procname = "litmus",
2482 + .mode = 0555,
2483 + .child = cache_table,
2484 + },
2485 + { }
2486 +};
2487 +*/
2488 +
2489 +//static struct ctl_table_header *litmus_sysctls;
2490 +
2491 +
2492 +/*
2493 + * Initialzie this proc
2494 + */
2495 +static int __init litmus_color_init(void)
2496 +{
2497 + int err=0;
2498 + printk("Init bankproc.c\n");
2499 +/*
2500 + init_variables();
2501 +
2502 + printk(KERN_INFO "Registering LITMUS^RT proc color sysctl.\n");
2503 +
2504 + litmus_sysctls = register_sysctl_table(litmus_dir_table);
2505 + if (!litmus_sysctls) {
2506 + printk(KERN_WARNING "Could not register LITMUS^RT color sysctl.\n");
2507 + err = -EFAULT;
2508 + goto out;
2509 + }
2510 +
2511 + init_color_groups();
2512 + do_add_pages();
2513 +*/
2514 + printk(KERN_INFO "Registering LITMUS^RT color and bank proc.\n");
2515 +out:
2516 + return err;
2517 +}
2518 +
2519 +module_init(litmus_color_init);
2520 +
2521 diff --git litmus/budget.c litmus/budget.c
2522 index 47bf78a..d67f4b3 100644
2523 --- litmus/budget.c
2524 +++ litmus/budget.c
2525 @@ -1,9 +1,11 @@
2526 #include <linux/sched.h>
2527 #include <linux/percpu.h>
2528 #include <linux/hrtimer.h>
2529 +#include <linux/uaccess.h>
2530
2531 #include <litmus/litmus.h>
2532 #include <litmus/preempt.h>
2533 +#include <litmus/sched_plugin.h>
2534
2535 #include <litmus/budget.h>
2536
2537 @@ -113,4 +115,54 @@ static int __init init_budget_enforcement(void)
2538 return 0;
2539 }
2540
2541 +void litmus_current_budget(lt_t *used_so_far, lt_t *remaining)
2542 +{
2543 + struct task_struct *t = current;
2544 + unsigned long flags;
2545 + s64 delta;
2546 +
2547 + local_irq_save(flags);
2548 +
2549 + delta = sched_clock_cpu(smp_processor_id()) - t->se.exec_start;
2550 + if (delta < 0)
2551 + delta = 0;
2552 +
2553 + TRACE_CUR("current_budget: sc:%llu start:%llu lt_t:%llu delta:%lld exec-time:%llu rem:%llu\n",
2554 + sched_clock_cpu(smp_processor_id()), t->se.exec_start,
2555 + litmus_clock(), delta,
2556 + tsk_rt(t)->job_params.exec_time,
2557 + budget_remaining(t));
2558 +
2559 + if (used_so_far)
2560 + *used_so_far = tsk_rt(t)->job_params.exec_time + delta;
2561 +
2562 + if (remaining) {
2563 + *remaining = budget_remaining(t);
2564 + if (*remaining > delta)
2565 + *remaining -= delta;
2566 + else
2567 + *remaining = 0;
2568 + }
2569 +
2570 + local_irq_restore(flags);
2571 +}
2572 +
2573 +asmlinkage long sys_get_current_budget(
2574 + lt_t __user * _expended,
2575 + lt_t __user *_remaining)
2576 +{
2577 + lt_t expended = 0, remaining = 0;
2578 +
2579 + if (is_realtime(current))
2580 + litmus->current_budget(&expended, &remaining);
2581 +
2582 + if (_expended && put_user(expended, _expended))
2583 + return -EFAULT;
2584 +
2585 + if (_remaining && put_user(remaining, _remaining))
2586 + return -EFAULT;
2587 +
2588 + return 0;
2589 +}
2590 +
2591 module_init(init_budget_enforcement);
2592 diff --git litmus/cache_proc.c litmus/cache_proc.c
2593 new file mode 100644
2594 index 0000000..51d0c3d
2595 --- /dev/null
2596 +++ litmus/cache_proc.c
2597 @@ -0,0 +1,1426 @@
2598 +#include <asm/uaccess.h>
2599 +#include <linux/uaccess.h>
2600 +#include <linux/init.h>
2601 +#include <linux/types.h>
2602 +#include <linux/kernel.h>
2603 +#include <linux/module.h>
2604 +#include <linux/sysctl.h>
2605 +#include <linux/slab.h>
2606 +#include <linux/io.h>
2607 +#include <linux/mutex.h>
2608 +#include <linux/time.h>
2609 +#include <linux/random.h>
2610 +#include <linux/sched.h>
2611 +
2612 +#include <litmus/rt_param.h>
2613 +#include <litmus/litmus.h>
2614 +#include <litmus/litmus_proc.h>
2615 +#include <litmus/sched_trace.h>
2616 +#include <litmus/cache_proc.h>
2617 +#include <litmus/mc2_common.h>
2618 +
2619 +#include <asm/hardware/cache-l2x0.h>
2620 +#include <asm/cacheflush.h>
2621 +
2622 +#define UNLOCK_ALL 0x00000000 /* allocation in any way */
2623 +#define LOCK_ALL (~UNLOCK_ALL)
2624 +#define MAX_NR_WAYS 16
2625 +#define MAX_NR_COLORS 16
2626 +#define CACHELINE_SIZE 32
2627 +#define INTS_IN_CACHELINE (CACHELINE_SIZE/sizeof(int))
2628 +#define CACHELINES_IN_1KB (1024 / sizeof(cacheline_t))
2629 +
2630 +typedef struct cacheline
2631 +{
2632 + int line[INTS_IN_CACHELINE];
2633 +} __attribute__((aligned(CACHELINE_SIZE))) cacheline_t;
2634 +
2635 +void mem_lock(u32 lock_val, int cpu);
2636 +
2637 +/*
2638 + * unlocked_way[i] : allocation can occur in way i
2639 + *
2640 + * 0 = allocation can occur in the corresponding way
2641 + * 1 = allocation cannot occur in the corresponding way
2642 + */
2643 +u32 unlocked_way[MAX_NR_WAYS] = {
2644 + 0xFFFFFFFE, /* way 0 unlocked */
2645 + 0xFFFFFFFD,
2646 + 0xFFFFFFFB,
2647 + 0xFFFFFFF7,
2648 + 0xFFFFFFEF, /* way 4 unlocked */
2649 + 0xFFFFFFDF,
2650 + 0xFFFFFFBF,
2651 + 0xFFFFFF7F,
2652 + 0xFFFFFEFF, /* way 8 unlocked */
2653 + 0xFFFFFDFF,
2654 + 0xFFFFFBFF,
2655 + 0xFFFFF7FF,
2656 + 0xFFFFEFFF, /* way 12 unlocked */
2657 + 0xFFFFDFFF,
2658 + 0xFFFFBFFF,
2659 + 0xFFFF7FFF,
2660 +};
2661 +
2662 +u32 nr_unlocked_way[MAX_NR_WAYS+1] = {
2663 + 0x0000FFFF, /* all ways are locked. usable = 0*/
2664 + 0x0000FFFE, /* way ~0 unlocked. usable = 1 */
2665 + 0x0000FFFC,
2666 + 0x0000FFF8,
2667 + 0x0000FFF0,
2668 + 0x0000FFE0,
2669 + 0x0000FFC0,
2670 + 0x0000FF80,
2671 + 0x0000FF00,
2672 + 0x0000FE00,
2673 + 0x0000FC00,
2674 + 0x0000F800,
2675 + 0x0000F000,
2676 + 0x0000E000,
2677 + 0x0000C000,
2678 + 0x00008000,
2679 + 0x00000000, /* way ~15 unlocked. usable = 16 */
2680 +};
2681 +
2682 +u32 way_partition[4] = {
2683 + 0xfffffff0, /* cpu0 */
2684 + 0xffffff0f, /* cpu1 */
2685 + 0xfffff0ff, /* cpu2 */
2686 + 0xffff0fff, /* cpu3 */
2687 +};
2688 +
2689 +u32 way_partitions[9] = {
2690 + 0xffff00ff, /* cpu0 A */
2691 + 0xffff00ff, /* cpu0 B */
2692 + 0xffff00ff, /* cpu1 A */
2693 + 0xffff00ff, /* cpu1 B */
2694 + 0xffff00ff, /* cpu2 A */
2695 + 0xffff00ff, /* cpu2 B */
2696 + 0xffff00ff, /* cpu3 A */
2697 + 0xffff00ff, /* cpu3 B */
2698 + 0xffffff00, /* lv C */
2699 +};
2700 +
2701 +u32 prev_lockdown_d_reg[5] = {
2702 + 0x0000FF00,
2703 + 0x0000FF00,
2704 + 0x0000FF00,
2705 + 0x0000FF00,
2706 + 0x000000FF, /* share with level-C */
2707 +};
2708 +
2709 +u32 prev_lockdown_i_reg[5] = {
2710 + 0x0000FF00,
2711 + 0x0000FF00,
2712 + 0x0000FF00,
2713 + 0x0000FF00,
2714 + 0x000000FF, /* share with level-C */
2715 +};
2716 +
2717 +u32 prev_lbm_i_reg[8] = {
2718 + 0x00000000,
2719 + 0x00000000,
2720 + 0x00000000,
2721 + 0x00000000,
2722 + 0x00000000,
2723 + 0x00000000,
2724 + 0x00000000,
2725 + 0x00000000,
2726 +};
2727 +
2728 +u32 prev_lbm_d_reg[8] = {
2729 + 0x00000000,
2730 + 0x00000000,
2731 + 0x00000000,
2732 + 0x00000000,
2733 + 0x00000000,
2734 + 0x00000000,
2735 + 0x00000000,
2736 + 0x00000000,
2737 +};
2738 +
2739 +static void __iomem *cache_base;
2740 +static void __iomem *lockreg_d;
2741 +static void __iomem *lockreg_i;
2742 +
2743 +static u32 cache_id;
2744 +
2745 +struct mutex actlr_mutex;
2746 +struct mutex l2x0_prefetch_mutex;
2747 +struct mutex lockdown_proc;
2748 +static u32 way_partition_min;
2749 +static u32 way_partition_max;
2750 +
2751 +static int zero = 0;
2752 +static int one = 1;
2753 +
2754 +static int l1_prefetch_proc;
2755 +static int l2_prefetch_hint_proc;
2756 +static int l2_double_linefill_proc;
2757 +static int l2_data_prefetch_proc;
2758 +static int os_isolation;
2759 +static int use_part;
2760 +
2761 +static u32 debug_test_val;
2762 +struct mutex debug_mutex;
2763 +
2764 +u32 lockdown_reg[9] = {
2765 + 0x00000000,
2766 + 0x00000000,
2767 + 0x00000000,
2768 + 0x00000000,
2769 + 0x00000000,
2770 + 0x00000000,
2771 + 0x00000000,
2772 + 0x00000000,
2773 +};
2774 +
2775 +
2776 +#define ld_d_reg(cpu) ({ int __cpu = cpu; \
2777 + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_D_BASE + \
2778 + __cpu * L2X0_LOCKDOWN_STRIDE; __v; })
2779 +#define ld_i_reg(cpu) ({ int __cpu = cpu; \
2780 + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_I_BASE + \
2781 + __cpu * L2X0_LOCKDOWN_STRIDE; __v; })
2782 +
2783 +int lock_all;
2784 +int nr_lockregs;
2785 +static raw_spinlock_t cache_lock;
2786 +static raw_spinlock_t prefetch_lock;
2787 +static void ***flusher_pages = NULL;
2788 +
2789 +extern void l2c310_flush_all(void);
2790 +
2791 +static inline void cache_wait_way(void __iomem *reg, unsigned long mask)
2792 +{
2793 + /* wait for cache operation by line or way to complete */
2794 + while (readl_relaxed(reg) & mask)
2795 + cpu_relax();
2796 +}
2797 +
2798 +#ifdef CONFIG_CACHE_L2X0
2799 +static inline void cache_wait(void __iomem *reg, unsigned long mask)
2800 +{
2801 + /* cache operations by line are atomic on PL310 */
2802 +}
2803 +#else
2804 +#define cache_wait cache_wait_way
2805 +#endif
2806 +
2807 +static inline void cache_sync(void)
2808 +{
2809 + void __iomem *base = cache_base;
2810 +
2811 + writel_relaxed(0, base + L2X0_CACHE_SYNC);
2812 + cache_wait(base + L2X0_CACHE_SYNC, 1);
2813 +}
2814 +
2815 +static void print_lockdown_registers(int cpu)
2816 +{
2817 + int i;
2818 + //for (i = 0; i < nr_lockregs; i++) {
2819 + for (i = 0; i < 4; i++) {
2820 + printk("P%d Lockdown Data CPU %2d: 0x%04x\n", cpu,
2821 + i, readl_relaxed(ld_d_reg(i)));
2822 + printk("P%d Lockdown Inst CPU %2d: 0x%04x\n", cpu,
2823 + i, readl_relaxed(ld_i_reg(i)));
2824 + }
2825 +}
2826 +
2827 +static void test_lockdown(void *ignore)
2828 +{
2829 + int i, cpu;
2830 +
2831 + cpu = smp_processor_id();
2832 + printk("Start lockdown test on CPU %d.\n", cpu);
2833 +
2834 + for (i = 0; i < nr_lockregs; i++) {
2835 + printk("CPU %2d data reg: 0x%8p\n", i, ld_d_reg(i));
2836 + printk("CPU %2d inst reg: 0x%8p\n", i, ld_i_reg(i));
2837 + }
2838 +
2839 + printk("Lockdown initial state:\n");
2840 + print_lockdown_registers(cpu);
2841 + printk("---\n");
2842 +
2843 + for (i = 0; i < nr_lockregs; i++) {
2844 + writel_relaxed(1, ld_d_reg(i));
2845 + writel_relaxed(2, ld_i_reg(i));
2846 + }
2847 + printk("Lockdown all data=1 instr=2:\n");
2848 + print_lockdown_registers(cpu);
2849 + printk("---\n");
2850 +
2851 + for (i = 0; i < nr_lockregs; i++) {
2852 + writel_relaxed((1 << i), ld_d_reg(i));
2853 + writel_relaxed(((1 << 8) >> i), ld_i_reg(i));
2854 + }
2855 + printk("Lockdown varies:\n");
2856 + print_lockdown_registers(cpu);
2857 + printk("---\n");
2858 +
2859 + for (i = 0; i < nr_lockregs; i++) {
2860 + writel_relaxed(UNLOCK_ALL, ld_d_reg(i));
2861 + writel_relaxed(UNLOCK_ALL, ld_i_reg(i));
2862 + }
2863 + printk("Lockdown all zero:\n");
2864 + print_lockdown_registers(cpu);
2865 +
2866 + printk("End lockdown test.\n");
2867 +}
2868 +
2869 +void litmus_setup_lockdown(void __iomem *base, u32 id)
2870 +{
2871 + cache_base = base;
2872 + cache_id = id;
2873 + lockreg_d = cache_base + L2X0_LOCKDOWN_WAY_D_BASE;
2874 + lockreg_i = cache_base + L2X0_LOCKDOWN_WAY_I_BASE;
2875 +
2876 + if (L2X0_CACHE_ID_PART_L310 == (cache_id & L2X0_CACHE_ID_PART_MASK)) {
2877 + nr_lockregs = 8;
2878 + } else {
2879 + printk("Unknown cache ID!\n");
2880 + nr_lockregs = 1;
2881 + }
2882 +
2883 + mutex_init(&actlr_mutex);
2884 + mutex_init(&l2x0_prefetch_mutex);
2885 + mutex_init(&lockdown_proc);
2886 + mutex_init(&debug_mutex);
2887 + raw_spin_lock_init(&cache_lock);
2888 + raw_spin_lock_init(&prefetch_lock);
2889 +
2890 + test_lockdown(NULL);
2891 +}
2892 +
2893 +int way_partition_handler(struct ctl_table *table, int write, void __user *buffer,
2894 + size_t *lenp, loff_t *ppos)
2895 +{
2896 + int ret = 0, i;
2897 + unsigned long flags;
2898 +
2899 + mutex_lock(&lockdown_proc);
2900 +
2901 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2902 + if (ret)
2903 + goto out;
2904 +
2905 + if (write) {
2906 + printk("Way-partition settings:\n");
2907 + for (i = 0; i < 9; i++) {
2908 + printk("0x%08X\n", way_partitions[i]);
2909 + }
2910 + for (i = 0; i < 4; i++) {
2911 + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
2912 + i * L2X0_LOCKDOWN_STRIDE);
2913 + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
2914 + i * L2X0_LOCKDOWN_STRIDE);
2915 + }
2916 + }
2917 +
2918 + local_irq_save(flags);
2919 + print_lockdown_registers(smp_processor_id());
2920 + l2c310_flush_all();
2921 + local_irq_restore(flags);
2922 +out:
2923 + mutex_unlock(&lockdown_proc);
2924 + return ret;
2925 +}
2926 +
2927 +int lock_all_handler(struct ctl_table *table, int write, void __user *buffer,
2928 + size_t *lenp, loff_t *ppos)
2929 +{
2930 + int ret = 0, i;
2931 + unsigned long flags;
2932 +
2933 + mutex_lock(&lockdown_proc);
2934 +
2935 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2936 + if (ret)
2937 + goto out;
2938 +
2939 + if (write && lock_all == 1) {
2940 + for (i = 0; i < nr_lockregs; i++) {
2941 + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
2942 + i * L2X0_LOCKDOWN_STRIDE);
2943 + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
2944 + i * L2X0_LOCKDOWN_STRIDE);
2945 + }
2946 +/*
2947 + for (i = 0; i < nr_lockregs; i++) {
2948 + barrier();
2949 + mem_lock(LOCK_ALL, i);
2950 + barrier();
2951 + //writel_relaxed(nr_unlocked_way[0], ld_d_reg(i));
2952 + //writel_relaxed(nr_unlocked_way[0], ld_i_reg(i));
2953 + }
2954 +*/
2955 + }
2956 + if (write && lock_all == 0) {
2957 + for (i = 0; i < nr_lockregs; i++) {
2958 + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
2959 + i * L2X0_LOCKDOWN_STRIDE);
2960 + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
2961 + i * L2X0_LOCKDOWN_STRIDE);
2962 + }
2963 +
2964 + }
2965 +
2966 + local_irq_save(flags);
2967 + print_lockdown_registers(smp_processor_id());
2968 + l2c310_flush_all();
2969 + local_irq_restore(flags);
2970 +out:
2971 + mutex_unlock(&lockdown_proc);
2972 + return ret;
2973 +}
2974 +
2975 +void cache_lockdown(u32 lock_val, int cpu)
2976 +{
2977 + __asm__ __volatile__ (
2978 +" str %[lockval], [%[dcachereg]]\n"
2979 +" str %[lockval], [%[icachereg]]\n"
2980 + :
2981 + : [dcachereg] "r" (ld_d_reg(cpu)),
2982 + [icachereg] "r" (ld_i_reg(cpu)),
2983 + [lockval] "r" (lock_val)
2984 + : "cc");
2985 +}
2986 +
2987 +void do_partition(enum crit_level lv, int cpu)
2988 +{
2989 + u32 regs;
2990 + unsigned long flags;
2991 +
2992 + if (lock_all || !use_part)
2993 + return;
2994 + raw_spin_lock_irqsave(&cache_lock, flags);
2995 + switch(lv) {
2996 + case CRIT_LEVEL_A:
2997 + regs = ~way_partitions[cpu*2];
2998 + regs &= 0x0000ffff;
2999 + break;
3000 + case CRIT_LEVEL_B:
3001 + regs = ~way_partitions[cpu*2+1];
3002 + regs &= 0x0000ffff;
3003 + break;
3004 + case CRIT_LEVEL_C:
3005 + case NUM_CRIT_LEVELS:
3006 + regs = ~way_partitions[8];
3007 + regs &= 0x0000ffff;
3008 + break;
3009 + default:
3010 + BUG();
3011 +
3012 + }
3013 + barrier();
3014 +
3015 + writel_relaxed(regs, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + cpu * L2X0_LOCKDOWN_STRIDE);
3016 + writel_relaxed(regs, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + cpu * L2X0_LOCKDOWN_STRIDE);
3017 + barrier();
3018 +
3019 + raw_spin_unlock_irqrestore(&cache_lock, flags);
3020 +}
3021 +
3022 +void lock_cache(int cpu, u32 val)
3023 +{
3024 + unsigned long flags;
3025 +
3026 + local_irq_save(flags);
3027 + if (val != 0xffffffff) {
3028 + writel_relaxed(val, cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
3029 + cpu * L2X0_LOCKDOWN_STRIDE);
3030 + writel_relaxed(val, cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
3031 + cpu * L2X0_LOCKDOWN_STRIDE);
3032 + }
3033 + else {
3034 + int i;
3035 + for (i = 0; i < 4; i++)
3036 + do_partition(CRIT_LEVEL_A, i);
3037 + }
3038 + local_irq_restore(flags);
3039 +}
3040 +
3041 +int use_part_proc_handler(struct ctl_table *table, int write, void __user *buffer,
3042 + size_t *lenp, loff_t *ppos)
3043 +{
3044 + int ret = 0;
3045 +
3046 + mutex_lock(&lockdown_proc);
3047 +
3048 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3049 + if (ret)
3050 + goto out;
3051 +
3052 +
3053 + printk("USE_PART HANDLER = %d\n", use_part);
3054 +
3055 +out:
3056 + mutex_unlock(&lockdown_proc);
3057 + return ret;
3058 +}
3059 +
3060 +int os_isolation_proc_handler(struct ctl_table *table, int write, void __user *buffer,
3061 + size_t *lenp, loff_t *ppos)
3062 +{
3063 + int ret = 0;
3064 +
3065 + mutex_lock(&lockdown_proc);
3066 +
3067 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3068 + if (ret)
3069 + goto out;
3070 +
3071 +
3072 + printk("OS_ISOLATION HANDLER = %d\n", os_isolation);
3073 +
3074 +out:
3075 + mutex_unlock(&lockdown_proc);
3076 + return ret;
3077 +}
3078 +
3079 +int lockdown_reg_handler(struct ctl_table *table, int write, void __user *buffer,
3080 + size_t *lenp, loff_t *ppos)
3081 +{
3082 + int ret = 0, i;
3083 +
3084 + mutex_lock(&lockdown_proc);
3085 +
3086 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3087 + if (ret)
3088 + goto out;
3089 +
3090 + if (write) {
3091 + for (i = 0; i < nr_lockregs; i++) {
3092 + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
3093 + i * L2X0_LOCKDOWN_STRIDE);
3094 + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
3095 + i * L2X0_LOCKDOWN_STRIDE);
3096 + }
3097 + }
3098 +
3099 +out:
3100 + mutex_unlock(&lockdown_proc);
3101 + return ret;
3102 +}
3103 +
3104 +int lockdown_global_handler(struct ctl_table *table, int write, void __user *buffer,
3105 + size_t *lenp, loff_t *ppos)
3106 +{
3107 + int ret = 0, i;
3108 +
3109 + mutex_lock(&lockdown_proc);
3110 +
3111 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3112 + if (ret)
3113 + goto out;
3114 +
3115 + if (write) {
3116 + for (i = 0; i < nr_lockregs; i++) {
3117 + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
3118 + i * L2X0_LOCKDOWN_STRIDE);
3119 + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
3120 + i * L2X0_LOCKDOWN_STRIDE);
3121 + }
3122 + }
3123 +
3124 +out:
3125 + mutex_unlock(&lockdown_proc);
3126 + return ret;
3127 +}
3128 +
3129 +void inline enter_irq_mode(void)
3130 +{
3131 + int cpu = smp_processor_id();
3132 +
3133 + if (os_isolation == 0)
3134 + return;
3135 + prev_lockdown_i_reg[cpu] = readl_relaxed(ld_i_reg(cpu));
3136 + prev_lockdown_d_reg[cpu] = readl_relaxed(ld_d_reg(cpu));
3137 + writel_relaxed(way_partitions[8], ld_i_reg(cpu));
3138 + writel_relaxed(way_partitions[8], ld_d_reg(cpu));
3139 +}
3140 +
3141 +void inline exit_irq_mode(void)
3142 +{
3143 + int cpu = smp_processor_id();
3144 +
3145 + if (os_isolation == 0)
3146 + return;
3147 +
3148 + writel_relaxed(prev_lockdown_i_reg[cpu], ld_i_reg(cpu));
3149 + writel_relaxed(prev_lockdown_d_reg[cpu], ld_d_reg(cpu));
3150 +}
3151 +
3152 +/* Operate on the Cortex-A9's ACTLR register */
3153 +#define ACTLR_L2_PREFETCH_HINT (1 << 1)
3154 +#define ACTLR_L1_PREFETCH (1 << 2)
3155 +
3156 +/*
3157 + * Change the ACTLR.
3158 + * @mode - If 1 (0), set (clear) the bit given in @mask in the ACTLR.
3159 + * @mask - A mask in which one bit is set to operate on the ACTLR.
3160 + */
3161 +static void actlr_change(int mode, int mask)
3162 +{
3163 + u32 orig_value, new_value, reread_value;
3164 +
3165 + if (0 != mode && 1 != mode) {
3166 + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n",
3167 + __FUNCTION__);
3168 + return;
3169 + }
3170 +
3171 + /* get the original value */
3172 + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (orig_value));
3173 +
3174 + if (0 == mode)
3175 + new_value = orig_value & ~(mask);
3176 + else
3177 + new_value = orig_value | mask;
3178 +
3179 + asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (new_value));
3180 + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (reread_value));
3181 +
3182 + printk("ACTLR: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n",
3183 + orig_value, new_value, reread_value);
3184 +}
3185 +
3186 +int litmus_l1_prefetch_proc_handler(struct ctl_table *table, int write,
3187 + void __user *buffer, size_t *lenp, loff_t *ppos)
3188 +{
3189 + int ret, mode;
3190 +
3191 + mutex_lock(&actlr_mutex);
3192 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
3193 +
3194 + if (!ret && write) {
3195 + mode = *((int*)table->data);
3196 + actlr_change(mode, ACTLR_L1_PREFETCH);
3197 + }
3198 + mutex_unlock(&actlr_mutex);
3199 +
3200 + return ret;
3201 +}
3202 +
3203 +int litmus_l2_prefetch_hint_proc_handler(struct ctl_table *table, int write,
3204 + void __user *buffer, size_t *lenp, loff_t *ppos)
3205 +{
3206 + int ret, mode;
3207 +
3208 + mutex_lock(&actlr_mutex);
3209 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
3210 + if (!ret && write) {
3211 + mode = *((int*)table->data);
3212 + actlr_change(mode, ACTLR_L2_PREFETCH_HINT);
3213 + }
3214 + mutex_unlock(&actlr_mutex);
3215 +
3216 + return ret;
3217 +}
3218 +
3219 +
3220 +/* Operate on the PL-310's Prefetch Control Register, L310_PREFETCH_CTRL */
3221 +#define L2X0_PREFETCH_DOUBLE_LINEFILL (1 << 30)
3222 +#define L2X0_PREFETCH_INST_PREFETCH (1 << 29)
3223 +#define L2X0_PREFETCH_DATA_PREFETCH (1 << 28)
3224 +static void l2x0_prefetch_change(int mode, int mask)
3225 +{
3226 + u32 orig_value, new_value, reread_value;
3227 +
3228 + if (0 != mode && 1 != mode) {
3229 + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n",
3230 + __FUNCTION__);
3231 + return;
3232 + }
3233 +
3234 + orig_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL);
3235 +
3236 + if (0 == mode)
3237 + new_value = orig_value & ~(mask);
3238 + else
3239 + new_value = orig_value | mask;
3240 +
3241 + writel_relaxed(new_value, cache_base + L310_PREFETCH_CTRL);
3242 + reread_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL);
3243 +
3244 + printk("l2x0 prefetch: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n",
3245 + orig_value, new_value, reread_value);
3246 +}
3247 +
3248 +int litmus_l2_double_linefill_proc_handler(struct ctl_table *table, int write,
3249 + void __user *buffer, size_t *lenp, loff_t *ppos)
3250 +{
3251 + int ret, mode;
3252 +
3253 + mutex_lock(&l2x0_prefetch_mutex);
3254 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
3255 + if (!ret && write) {
3256 + mode = *((int*)table->data);
3257 + l2x0_prefetch_change(mode, L2X0_PREFETCH_DOUBLE_LINEFILL);
3258 + }
3259 + mutex_unlock(&l2x0_prefetch_mutex);
3260 +
3261 + return ret;
3262 +}
3263 +
3264 +int litmus_l2_data_prefetch_proc_handler(struct ctl_table *table, int write,
3265 + void __user *buffer, size_t *lenp, loff_t *ppos)
3266 +{
3267 + int ret, mode;
3268 +
3269 + mutex_lock(&l2x0_prefetch_mutex);
3270 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
3271 + if (!ret && write) {
3272 + mode = *((int*)table->data);
3273 + l2x0_prefetch_change(mode, L2X0_PREFETCH_DATA_PREFETCH|L2X0_PREFETCH_INST_PREFETCH);
3274 + }
3275 + mutex_unlock(&l2x0_prefetch_mutex);
3276 +
3277 + return ret;
3278 +}
3279 +
3280 +extern void *msgvaddr;
3281 +
3282 +int do_measure(void) {
3283 + lt_t t1, t2;
3284 + int i;
3285 +
3286 + barrier();
3287 + t1 = litmus_clock();
3288 + color_read_in_mem_lock(0xFFFF7FFF, 0xFFFF8000, msgvaddr, msgvaddr + 65536);
3289 + t2 = litmus_clock() - t1;
3290 + barrier();
3291 +
3292 + for (i = 0; i < 8; i++) {
3293 + cache_lockdown(0xFFFF8000, i);
3294 + }
3295 + printk("mem read time %lld\n", t2);
3296 +
3297 + return 0;
3298 +}
3299 +
3300 +int debug_test_handler(struct ctl_table *table, int write,
3301 + void __user *buffer, size_t *lenp, loff_t *ppos)
3302 +{
3303 + int ret;
3304 +
3305 + //mutex_lock(&debug_mutex);
3306 + //ret = proc_dointvec(table, write, buffer, lenp, ppos);
3307 + if (write) {
3308 + ret = do_measure();
3309 + }
3310 +
3311 + return ret;
3312 +}
3313 +
3314 +
3315 +int do_perf_test_proc_handler(struct ctl_table *table, int write,
3316 + void __user *buffer, size_t *lenp, loff_t *ppos);
3317 +
3318 +int setup_flusher_proc_handler(struct ctl_table *table, int write,
3319 + void __user *buffer, size_t *lenp, loff_t *ppos);
3320 +
3321 +static struct ctl_table cache_table[] =
3322 +{
3323 + {
3324 + .procname = "C0_LA_way",
3325 + .mode = 0666,
3326 + .proc_handler = way_partition_handler,
3327 + .data = &way_partitions[0],
3328 + .maxlen = sizeof(way_partitions[0]),
3329 + .extra1 = &way_partition_min,
3330 + .extra2 = &way_partition_max,
3331 + },
3332 + {
3333 + .procname = "C0_LB_way",
3334 + .mode = 0666,
3335 + .proc_handler = way_partition_handler,
3336 + .data = &way_partitions[1],
3337 + .maxlen = sizeof(way_partitions[1]),
3338 + .extra1 = &way_partition_min,
3339 + .extra2 = &way_partition_max,
3340 + },
3341 + {
3342 + .procname = "C1_LA_way",
3343 + .mode = 0666,
3344 + .proc_handler = way_partition_handler,
3345 + .data = &way_partitions[2],
3346 + .maxlen = sizeof(way_partitions[2]),
3347 + .extra1 = &way_partition_min,
3348 + .extra2 = &way_partition_max,
3349 + },
3350 + {
3351 + .procname = "C1_LB_way",
3352 + .mode = 0666,
3353 + .proc_handler = way_partition_handler,
3354 + .data = &way_partitions[3],
3355 + .maxlen = sizeof(way_partitions[3]),
3356 + .extra1 = &way_partition_min,
3357 + .extra2 = &way_partition_max,
3358 + },
3359 + {
3360 + .procname = "C2_LA_way",
3361 + .mode = 0666,
3362 + .proc_handler = way_partition_handler,
3363 + .data = &way_partitions[4],
3364 + .maxlen = sizeof(way_partitions[4]),
3365 + .extra1 = &way_partition_min,
3366 + .extra2 = &way_partition_max,
3367 + },
3368 + {
3369 + .procname = "C2_LB_way",
3370 + .mode = 0666,
3371 + .proc_handler = way_partition_handler,
3372 + .data = &way_partitions[5],
3373 + .maxlen = sizeof(way_partitions[5]),
3374 + .extra1 = &way_partition_min,
3375 + .extra2 = &way_partition_max,
3376 + },
3377 + {
3378 + .procname = "C3_LA_way",
3379 + .mode = 0666,
3380 + .proc_handler = way_partition_handler,
3381 + .data = &way_partitions[6],
3382 + .maxlen = sizeof(way_partitions[6]),
3383 + .extra1 = &way_partition_min,
3384 + .extra2 = &way_partition_max,
3385 + },
3386 + {
3387 + .procname = "C3_LB_way",
3388 + .mode = 0666,
3389 + .proc_handler = way_partition_handler,
3390 + .data = &way_partitions[7],
3391 + .maxlen = sizeof(way_partitions[7]),
3392 + .extra1 = &way_partition_min,
3393 + .extra2 = &way_partition_max,
3394 + },
3395 + {
3396 + .procname = "Call_LC_way",
3397 + .mode = 0666,
3398 + .proc_handler = way_partition_handler,
3399 + .data = &way_partitions[8],
3400 + .maxlen = sizeof(way_partitions[8]),
3401 + .extra1 = &way_partition_min,
3402 + .extra2 = &way_partition_max,
3403 + },
3404 + {
3405 + .procname = "lock_all",
3406 + .mode = 0666,
3407 + .proc_handler = lock_all_handler,
3408 + .data = &lock_all,
3409 + .maxlen = sizeof(lock_all),
3410 + .extra1 = &zero,
3411 + .extra2 = &one,
3412 + },
3413 + {
3414 + .procname = "l1_prefetch",
3415 + .mode = 0644,
3416 + .proc_handler = litmus_l1_prefetch_proc_handler,
3417 + .data = &l1_prefetch_proc,
3418 + .maxlen = sizeof(l1_prefetch_proc),
3419 + },
3420 + {
3421 + .procname = "l2_prefetch_hint",
3422 + .mode = 0644,
3423 + .proc_handler = litmus_l2_prefetch_hint_proc_handler,
3424 + .data = &l2_prefetch_hint_proc,
3425 + .maxlen = sizeof(l2_prefetch_hint_proc),
3426 + },
3427 + {
3428 + .procname = "l2_double_linefill",
3429 + .mode = 0644,
3430 + .proc_handler = litmus_l2_double_linefill_proc_handler,
3431 + .data = &l2_double_linefill_proc,
3432 + .maxlen = sizeof(l2_double_linefill_proc),
3433 + },
3434 + {
3435 + .procname = "l2_data_prefetch",
3436 + .mode = 0644,
3437 + .proc_handler = litmus_l2_data_prefetch_proc_handler,
3438 + .data = &l2_data_prefetch_proc,
3439 + .maxlen = sizeof(l2_data_prefetch_proc),
3440 + },
3441 + {
3442 + .procname = "os_isolation",
3443 + .mode = 0644,
3444 + .proc_handler = os_isolation_proc_handler,
3445 + .data = &os_isolation,
3446 + .maxlen = sizeof(os_isolation),
3447 + },
3448 + {
3449 + .procname = "use_part",
3450 + .mode = 0644,
3451 + .proc_handler = use_part_proc_handler,
3452 + .data = &use_part,
3453 + .maxlen = sizeof(use_part),
3454 + },
3455 + {
3456 + .procname = "do_perf_test",
3457 + .mode = 0644,
3458 + .proc_handler = do_perf_test_proc_handler,
3459 + },
3460 + {
3461 + .procname = "setup_flusher",
3462 + .mode = 0644,
3463 + .proc_handler = setup_flusher_proc_handler,
3464 + },
3465 + {
3466 + .procname = "lockdown_reg_0",
3467 + .mode = 0644,
3468 + .proc_handler = lockdown_reg_handler,
3469 + .data = &lockdown_reg[0],
3470 + .maxlen = sizeof(lockdown_reg[0]),
3471 + .extra1 = &way_partition_min,
3472 + .extra2 = &way_partition_max,
3473 + },
3474 + {
3475 + .procname = "lockdown_reg_1",
3476 + .mode = 0644,
3477 + .proc_handler = lockdown_reg_handler,
3478 + .data = &lockdown_reg[1],
3479 + .maxlen = sizeof(lockdown_reg[1]),
3480 + .extra1 = &way_partition_min,
3481 + .extra2 = &way_partition_max,
3482 + },
3483 + {
3484 + .procname = "lockdown_reg_2",
3485 + .mode = 0644,
3486 + .proc_handler = lockdown_reg_handler,
3487 + .data = &lockdown_reg[2],
3488 + .maxlen = sizeof(lockdown_reg[2]),
3489 + .extra1 = &way_partition_min,
3490 + .extra2 = &way_partition_max,
3491 + },
3492 + {
3493 + .procname = "lockdown_reg_3",
3494 + .mode = 0644,
3495 + .proc_handler = lockdown_reg_handler,
3496 + .data = &lockdown_reg[3],
3497 + .maxlen = sizeof(lockdown_reg[3]),
3498 + .extra1 = &way_partition_min,
3499 + .extra2 = &way_partition_max,
3500 + },
3501 + {
3502 + .procname = "lockdown_regs",
3503 + .mode = 0644,
3504 + .proc_handler = lockdown_global_handler,
3505 + .data = &lockdown_reg[8],
3506 + .maxlen = sizeof(lockdown_reg[8]),
3507 + .extra1 = &way_partition_min,
3508 + .extra2 = &way_partition_max,
3509 + },
3510 + {
3511 + .procname = "debug_test",
3512 + .mode = 0644,
3513 + .proc_handler = debug_test_handler,
3514 + },
3515 + { }
3516 +};
3517 +
3518 +static struct ctl_table litmus_dir_table[] = {
3519 + {
3520 + .procname = "litmus",
3521 + .mode = 0555,
3522 + .child = cache_table,
3523 + },
3524 + { }
3525 +};
3526 +
3527 +u32 color_read_in_mem(u32 lock_val, u32 unlock_val, void *start, void *end)
3528 +{
3529 + u32 v = 0;
3530 +
3531 + __asm__ __volatile__ (
3532 +" .align 5\n"
3533 +" str %[lockval], [%[cachereg]]\n"
3534 +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n"
3535 +" cmp %[end], %[addr] @ subtracts addr from end\n"
3536 +" bgt 1b\n @ read more, if necessary\n"
3537 + : [addr] "+r" (start),
3538 + [val] "+r" (v)
3539 + : [end] "r" (end),
3540 +#ifdef CONFIG_CACHE_L2X0
3541 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())),
3542 +#else
3543 + [cachereg] "r" (lockreg_d),
3544 +#endif
3545 + [lockval] "r" (lock_val)
3546 + : "cc");
3547 +
3548 + return v;
3549 +}
3550 +
3551 +
3552 +/*
3553 + * Prefetch by reading the first word of each cache line in a page.
3554 + *
3555 + * @lockdown_reg: address of the lockdown register to write
3556 + * @lock_val: value to be written to @lockdown_reg
3557 + * @unlock_val: will unlock the cache to this value
3558 + * @addr: start address to be prefetched
3559 + * @end_addr: end address to prefetch (exclusive)
3560 + *
3561 + * Assumes: addr < end_addr AND addr != end_addr
3562 + */
3563 +u32 color_read_in_mem_lock(u32 lock_val, u32 unlock_val, void *start, void *end)
3564 +{
3565 +#ifndef CONFIG_CACHE_L2X0
3566 + unsigned long flags;
3567 +#endif
3568 + u32 v = 0;
3569 +
3570 +#ifndef CONFIG_CACHE_L2X0
3571 + raw_spin_lock_irqsave(&prefetch_lock, flags);
3572 +#endif
3573 +
3574 + __asm__ __volatile__ (
3575 +" .align 5\n"
3576 +" str %[lockval], [%[cachereg]]\n"
3577 +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n"
3578 +" cmp %[end], %[addr] @ subtracts addr from end\n"
3579 +" bgt 1b\n @ read more, if necessary\n"
3580 +" str %[unlockval], [%[cachereg]]\n"
3581 + : [addr] "+r" (start),
3582 + [val] "+r" (v)
3583 + : [end] "r" (end),
3584 +#ifdef CONFIG_CACHE_L2X0
3585 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())),
3586 +#else
3587 + [cachereg] "r" (lockreg_d),
3588 +#endif
3589 + [lockval] "r" (lock_val),
3590 + [unlockval] "r" (unlock_val)
3591 + : "cc");
3592 +
3593 +#ifndef CONFIG_CACHE_L2X0
3594 + raw_spin_unlock_irqrestore(&prefetch_lock, flags);
3595 +#endif
3596 +
3597 + return v;
3598 +}
3599 +
3600 +static long update_timeval(struct timespec lhs, struct timespec rhs)
3601 +{
3602 + long val;
3603 + struct timespec ts;
3604 +
3605 + ts = timespec_sub(rhs, lhs);
3606 + val = ts.tv_sec*NSEC_PER_SEC + ts.tv_nsec;
3607 +
3608 + return val;
3609 +}
3610 +
3611 +extern void v7_flush_kern_dcache_area(void *, size_t);
3612 +extern void v7_flush_kern_cache_all(void);
3613 +/*
3614 + * Ensure that this page is not in the L1 or L2 cache.
3615 + * Since the L1 cache is VIPT and the L2 cache is PIPT, we can use either the
3616 + * kernel or user vaddr.
3617 + */
3618 +void color_flush_page(void *vaddr, size_t size)
3619 +{
3620 + v7_flush_kern_dcache_area(vaddr, size);
3621 + //v7_flush_kern_cache_all();
3622 +}
3623 +
3624 +extern struct page* get_colored_page(unsigned long color);
3625 +
3626 +int setup_flusher_array(void)
3627 +{
3628 + int color, way, ret = 0;
3629 + struct page *page;
3630 +
3631 + if (flusher_pages != NULL)
3632 + goto out;
3633 +
3634 + flusher_pages = (void***) kmalloc(MAX_NR_WAYS
3635 + * sizeof(*flusher_pages), GFP_KERNEL);
3636 + if (!flusher_pages) {
3637 + printk(KERN_WARNING "No memory for flusher array!\n");
3638 + ret = -EINVAL;
3639 + goto out;
3640 + }
3641 + for (way = 0; way < MAX_NR_WAYS; way++) {
3642 + void **flusher_color_arr;
3643 + flusher_color_arr = (void**) kmalloc(sizeof(**flusher_pages)
3644 + * MAX_NR_COLORS, GFP_KERNEL);
3645 + if (!flusher_color_arr) {
3646 + printk(KERN_WARNING "No memory for flusher array!\n");
3647 + ret = -ENOMEM;
3648 + goto out_free;
3649 + }
3650 +
3651 + flusher_pages[way] = flusher_color_arr;
3652 + for (color = 0; color < MAX_NR_COLORS; color++) {
3653 + int node;
3654 + node = color + 112; // populate from bank 7
3655 + page = get_colored_page(node);
3656 + if (!page) {
3657 + printk(KERN_WARNING "no more colored pages\n");
3658 + ret = -EINVAL;
3659 + goto out_free;
3660 + }
3661 + flusher_pages[way][color] = page_address(page);
3662 + if (!flusher_pages[way][color]) {
3663 + printk(KERN_WARNING "bad page address\n");
3664 + ret = -EINVAL;
3665 + goto out_free;
3666 + }
3667 + }
3668 + }
3669 +
3670 +out:
3671 + return ret;
3672 +out_free:
3673 + for (way = 0; way < MAX_NR_WAYS; way++) {
3674 + for (color = 0; color < MAX_NR_COLORS; color++) {
3675 + /* not bothering to try and give back colored pages */
3676 + }
3677 + kfree(flusher_pages[way]);
3678 + }
3679 + kfree(flusher_pages);
3680 + flusher_pages = NULL;
3681 + return ret;
3682 +}
3683 +
3684 +void flush_cache(int all)
3685 +{
3686 + int way, color, cpu;
3687 + unsigned long flags;
3688 +
3689 + raw_spin_lock_irqsave(&cache_lock, flags);
3690 + cpu = raw_smp_processor_id();
3691 +
3692 + prev_lbm_i_reg[cpu] = readl_relaxed(ld_i_reg(cpu));
3693 + prev_lbm_d_reg[cpu] = readl_relaxed(ld_d_reg(cpu));
3694 + for (way=0;way<MAX_NR_WAYS;way++) {
3695 + if (( (0x00000001 << way) & (prev_lbm_d_reg[cpu]) ) &&
3696 + !all)
3697 + continue;
3698 + for (color=0;color<MAX_NR_COLORS;color++) {
3699 + void *vaddr = flusher_pages[way][color];
3700 + u32 lvalue = unlocked_way[way];
3701 + color_read_in_mem_lock(lvalue, LOCK_ALL,
3702 + vaddr, vaddr + PAGE_SIZE);
3703 + }
3704 +
3705 + }
3706 +
3707 + writel_relaxed(prev_lbm_i_reg[cpu], ld_i_reg(cpu));
3708 + writel_relaxed(prev_lbm_d_reg[cpu], ld_d_reg(cpu));
3709 + raw_spin_unlock_irqrestore(&cache_lock, flags);
3710 +}
3711 +
3712 +/* src = shared, dst = local */
3713 +#if 1 // random
3714 +asmlinkage long sys_run_test(int type, int size, cacheline_t *src, cacheline_t *dst, lt_t __user *ts)
3715 +{
3716 + /* size is in KB */
3717 + long ret = 0;
3718 + lt_t t1, t2;
3719 + int numlines = size * CACHELINES_IN_1KB;
3720 + int next, sum = 0, ran;
3721 + unsigned long flags;
3722 +
3723 + get_random_bytes(&ran, sizeof(int));
3724 + next = ran % ((size*1024)/sizeof(cacheline_t));
3725 +
3726 + //preempt_disable();
3727 + if (type == 1) {
3728 + int i, j;
3729 + color_read_in_mem_lock(0x0000FFF0, 0x0000000f, (void*)src, (void*)src + size*1024);
3730 + color_read_in_mem_lock(0x0000FF0F, 0x0000000f, (void*)dst, (void*)dst + size*1024);
3731 +
3732 + local_irq_save(flags);
3733 + t1 = litmus_clock();
3734 + for (i = 0; i < numlines; i++) {
3735 + next = src[next].line[0];
3736 + for (j = 1; j < INTS_IN_CACHELINE; j++) {
3737 + //dst[next].line[j] = src[next].line[j]; // read
3738 + src[next].line[j] = dst[next].line[j]; // write
3739 + }
3740 + }
3741 + t2 = litmus_clock();
3742 + local_irq_restore(flags);
3743 + sum = next + (int)t2;
3744 + t2 -= t1;
3745 + ret = put_user(t2, ts);
3746 + }
3747 + else {
3748 + int i, j;
3749 + color_read_in_mem_lock(0x0000FF0F, 0x0000000f, (void*)dst, (void*)dst + size*1024);
3750 + local_irq_save(flags);
3751 + t1 = litmus_clock();
3752 + for (i = 0; i < numlines; i++) {
3753 + next = src[next].line[0];
3754 + for (j = 1; j < INTS_IN_CACHELINE; j++) {
3755 + //dst[next].line[j] = src[next].line[j]; //read
3756 + src[next].line[j] = dst[next].line[j]; //write
3757 + }
3758 + }
3759 + t2 = litmus_clock();
3760 + local_irq_restore(flags);
3761 + sum = next + (int)t2;
3762 + t2 -= t1;
3763 + ret = put_user(t2, ts);
3764 + v7_flush_kern_dcache_area(src, size*1024);
3765 + }
3766 + //preempt_enable();
3767 + flush_cache(1);
3768 +
3769 + return ret;
3770 +}
3771 +#else
3772 +// sequential
3773 +asmlinkage long sys_run_test(int type, int size, cacheline_t *src, cacheline_t *dst, lt_t __user *ts)
3774 +{
3775 + /* size is in KB */
3776 + long ret = 0;
3777 + lt_t t1, t2;
3778 + int numlines = size * CACHELINES_IN_1KB;
3779 + int sum = 0;
3780 + unsigned long flags;
3781 +
3782 + //preempt_disable();
3783 + if (type == 1) {
3784 + int i, j;
3785 + color_read_in_mem_lock(0x0000FFF0, 0x0000000f, (void*)src, (void*)src + size*1024);
3786 + color_read_in_mem_lock(0x0000FF0F, 0x0000000f, (void*)dst, (void*)dst + size*1024);
3787 +
3788 + local_irq_save(flags);
3789 + t1 = litmus_clock();
3790 + for (i = 0; i < numlines; i++) {
3791 + for (j = 0; j < INTS_IN_CACHELINE; j++) {
3792 + //dst[i].line[j] = src[i].line[j]; // read
3793 + src[i].line[j] = dst[i].line[j]; // write
3794 + }
3795 + }
3796 + t2 = litmus_clock();
3797 + local_irq_restore(flags);
3798 + sum = (int)(t1 + t2);
3799 + t2 -= t1;
3800 + ret = put_user(t2, ts);
3801 + }
3802 + else {
3803 + int i, j;
3804 + color_read_in_mem_lock(0x0000FF0F, 0x0000000f, (void*)dst, (void*)dst + size*1024);
3805 + local_irq_save(flags);
3806 + t1 = litmus_clock();
3807 + for (i = 0; i < numlines; i++) {
3808 + for (j = 0; j < INTS_IN_CACHELINE; j++) {
3809 + //dst[i].line[j] = src[i].line[j]; //read
3810 + src[i].line[j] = dst[i].line[j]; //write
3811 + }
3812 + }
3813 + t2 = litmus_clock();
3814 + local_irq_restore(flags);
3815 + sum = (int)(t1 + t2);
3816 + t2 -= t1;
3817 + ret = put_user(t2, ts);
3818 + v7_flush_kern_dcache_area(src, size*1024);
3819 + }
3820 + //preempt_enable();
3821 + flush_cache(1);
3822 +
3823 + return ret;
3824 +}
3825 +#endif
3826 +
3827 +asmlinkage long sys_lock_buffer(void* vaddr, size_t size, u32 lock_way, u32 unlock_way)
3828 +{
3829 + /* size is in bytes */
3830 + long ret = 0;
3831 + int i;
3832 + u32 lock_val, unlock_val;
3833 +
3834 + lock_val = ~lock_way & 0x0000ffff;
3835 + unlock_val = ~unlock_way & 0x0000ffff;
3836 + color_read_in_mem_lock(lock_val, unlock_val, (void*)vaddr, (void*)vaddr + size);
3837 +
3838 + return ret;
3839 +}
3840 +
3841 +#define TRIALS 1000
3842 +
3843 +static int perf_test(void) {
3844 + struct timespec before, after;
3845 + struct page *page, *page2;
3846 + void *vaddr, *vaddr2;
3847 + u32 *data, *data2;
3848 + long time, flush_time;
3849 + int i, n, num_pages = 1;
3850 + unsigned int order = 2;
3851 + lt_t t1 = 0, t2 = 0, t3, t4;
3852 + struct timeval t;
3853 + struct tm broken;
3854 +
3855 + do_gettimeofday(&t);
3856 + t3 = get_cycles();
3857 + t4 = litmus_clock();
3858 + time_to_tm(t.tv_sec, 0, &broken);
3859 + printk(KERN_INFO "gettimeofday() - %d:%d:%d:%ld\n", broken.tm_hour, broken.tm_min, broken.tm_sec, t.tv_usec);
3860 + printk(KERN_INFO "get_cycles() %llu\n", t3);
3861 + printk(KERN_INFO "litmus_clock() %llu\n", t4);
3862 +
3863 + for (i = 0; i < order; i++) {
3864 + num_pages = num_pages*2;
3865 + }
3866 +
3867 + printk("Number of pages: %d\n", num_pages);
3868 + //page = alloc_page(__GFP_MOVABLE);
3869 + page = alloc_pages(__GFP_MOVABLE, order);
3870 + if (!page) {
3871 + printk(KERN_WARNING "No memory\n");
3872 + return -ENOMEM;
3873 + }
3874 +
3875 + page2 = alloc_pages(__GFP_MOVABLE, order);
3876 + if (!page2) {
3877 + printk(KERN_WARNING "No memory\n");
3878 + return -ENOMEM;
3879 + }
3880 +
3881 + vaddr = page_address(page);
3882 + if (!vaddr)
3883 + printk(KERN_WARNING "%s: vaddr is null\n", __FUNCTION__);
3884 + data = (u32*) vaddr;
3885 +
3886 + vaddr2 = page_address(page2);
3887 + if (!vaddr2)
3888 + printk(KERN_WARNING "%s: vaddr2 is null\n", __FUNCTION__);
3889 + data2 = (u32*) vaddr2;
3890 +
3891 + for (i = 32; i < 4096; i *= 2) {
3892 + for (n = 0; n < TRIALS; n++) {
3893 + invalidate_kernel_vmap_range(vaddr, 8192);
3894 + invalidate_kernel_vmap_range(vaddr2, 8192);
3895 + barrier();
3896 + t1 = litmus_clock();
3897 + memcpy(vaddr2, vaddr, i);
3898 + barrier();
3899 + t2 += litmus_clock() - t1;
3900 + }
3901 + printk("Size %d, average for memcpy %lld\n", i, t2>>9);
3902 + }
3903 +/*
3904 + getnstimeofday(&before);
3905 + barrier();
3906 + for (i = 0; i < TRIALS; i++) {
3907 + color_flush_page(vaddr, PAGE_SIZE*num_pages);
3908 + }
3909 + barrier();
3910 + getnstimeofday(&after);
3911 + time = update_timeval(before, after);
3912 + printk("Average for flushes without re-reading: %ld\n", time / TRIALS);
3913 + flush_time = time / TRIALS;
3914 +
3915 + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3916 +
3917 + barrier();
3918 + getnstimeofday(&before);
3919 + barrier();
3920 + for (i = 0; i < TRIALS; i++) {
3921 + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3922 + }
3923 + barrier();
3924 + getnstimeofday(&after);
3925 + time = update_timeval(before, after);
3926 + printk("Average for read from cache: %ld\n", time / TRIALS);
3927 +
3928 + getnstimeofday(&before);
3929 + barrier();
3930 + for (i = 0; i < TRIALS; i++) {
3931 + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3932 + color_flush_page(vaddr, PAGE_SIZE*num_pages);
3933 + }
3934 + barrier();
3935 + getnstimeofday(&after);
3936 + time = update_timeval(before, after);
3937 + printk("Average for read from mem: %ld (%ld)\n", time / TRIALS - flush_time, time / TRIALS);
3938 +
3939 + // write in locked way
3940 + color_read_in_mem_lock(nr_unlocked_way[2], LOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3941 + for (i = 0; i < PAGE_SIZE*num_pages/sizeof(u32); i++) {
3942 + data[i] = i%63353;
3943 + }
3944 + // read
3945 + barrier();
3946 + getnstimeofday(&before);
3947 + barrier();
3948 + for (i = 0; i < TRIALS; i++) {
3949 + color_read_in_mem(unlocked_way[0], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3950 + }
3951 + barrier();
3952 + getnstimeofday(&after);
3953 + time = update_timeval(before, after);
3954 + printk("Average for read in after write: %ld\n", time / TRIALS);
3955 +
3956 +*/
3957 + //free_page((unsigned long)vaddr);
3958 + free_pages((unsigned long)vaddr, order);
3959 + free_pages((unsigned long)vaddr2, order);
3960 +
3961 + do_gettimeofday(&t);
3962 + t3 = get_cycles();
3963 + t4 = litmus_clock();
3964 + time_to_tm(t.tv_sec, 0, &broken);
3965 + printk(KERN_INFO "gettimeofday() - %d:%d:%d:%ld\n", broken.tm_hour, broken.tm_min, broken.tm_sec, t.tv_usec);
3966 + printk(KERN_INFO "get_cycles() %llu\n", t3);
3967 + printk(KERN_INFO "litmus_clock() %llu\n", t4);
3968 +
3969 + return 0;
3970 +}
3971 +
3972 +int do_perf_test_proc_handler(struct ctl_table *table, int write,
3973 + void __user *buffer, size_t *lenp, loff_t *ppos)
3974 +{
3975 + int ret = 0;
3976 +
3977 + if (write) {
3978 + ret = perf_test();
3979 + }
3980 +
3981 + return ret;
3982 +}
3983 +
3984 +int setup_flusher_proc_handler(struct ctl_table *table, int write,
3985 + void __user *buffer, size_t *lenp, loff_t *ppos)
3986 +{
3987 + int ret = -EINVAL;
3988 +
3989 + if (write && flusher_pages == NULL) {
3990 + ret = setup_flusher_array();
3991 + printk(KERN_INFO "setup flusher return: %d\n", ret);
3992 +
3993 + }
3994 + else if (flusher_pages) {
3995 + printk(KERN_INFO "flusher_pages is already set!\n");
3996 + ret = 0;
3997 + }
3998 +
3999 + return ret;
4000 +}
4001 +
4002 +static struct ctl_table_header *litmus_sysctls;
4003 +
4004 +static int __init litmus_sysctl_init(void)
4005 +{
4006 + int ret = 0;
4007 +
4008 + printk(KERN_INFO "Registering LITMUS^RT proc sysctl.\n");
4009 + litmus_sysctls = register_sysctl_table(litmus_dir_table);
4010 + if (!litmus_sysctls) {
4011 + printk(KERN_WARNING "Could not register LITMUS^RT sysctl.\n");
4012 + ret = -EFAULT;
4013 + goto out;
4014 + }
4015 +
4016 + way_partition_min = 0x00000000;
4017 + way_partition_max = 0x0000FFFF;
4018 +
4019 +out:
4020 + return ret;
4021 +}
4022 +
4023 +module_init(litmus_sysctl_init);
4024 diff --git litmus/color_shm.c litmus/color_shm.c
4025 new file mode 100644
4026 index 0000000..0842623
4027 --- /dev/null
4028 +++ litmus/color_shm.c
4029 @@ -0,0 +1,402 @@
4030 +#include <linux/sched.h>
4031 +#include <linux/mm.h>
4032 +#include <linux/fs.h>
4033 +#include <linux/miscdevice.h>
4034 +#include <linux/spinlock.h>
4035 +#include <linux/module.h>
4036 +#include <linux/highmem.h>
4037 +#include <linux/slab.h>
4038 +#include <linux/mutex.h>
4039 +#include <asm/uaccess.h>
4040 +
4041 +#include <litmus/litmus.h>
4042 +
4043 +#define DEV_NAME "litmus/color_shm"
4044 +
4045 +/* Major number assigned to our device.
4046 + * Refer Documentation/devices.txt */
4047 +#define SHM_MAJOR 240
4048 +#define MAX_COLORED_PAGE 256
4049 +#define NUM_BANKS 8
4050 +#define NUM_COLORS 16
4051 +
4052 +static struct mutex dev_lock;
4053 +static int bypass_cache;
4054 +
4055 +struct color_ioctl_cmd {
4056 + unsigned int color;
4057 + unsigned int bank;
4058 +};
4059 +
4060 +struct color_ioctl_offset {
4061 + unsigned long offset;
4062 + int lock;
4063 +};
4064 +
4065 +#define SET_COLOR_SHM_CMD _IOW(SHM_MAJOR, 0x1, struct color_ioctl_cmd)
4066 +#define SET_COLOR_SHM_OFFSET _IOW(SHM_MAJOR, 0x2, struct color_ioctl_offset)
4067 +
4068 +struct color_ioctl_cmd color_param;
4069 +struct color_ioctl_offset color_offset;
4070 +
4071 +static int mmap_common_checks(struct vm_area_struct *vma)
4072 +{
4073 + /* you can only map the "first" page */
4074 + if (vma->vm_pgoff != 0)
4075 + return -EINVAL;
4076 +
4077 + return 0;
4078 +}
4079 +
4080 +static void mmap_common_vma_flags(struct vm_area_struct *vma)
4081 +{
4082 + /* This mapping should not be kept across forks,
4083 + * cannot be expanded, and is not a "normal" page. */
4084 + //vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_IO | VM_SHARED | VM_MAYSHARE;
4085 + vma->vm_flags |= VM_SHARED | VM_MAYSHARE | VM_LOCKED;
4086 +
4087 + /* We don't want the first write access to trigger a "minor" page fault
4088 + * to mark the page as dirty. This is transient, private memory, we
4089 + * don't care if it was touched or not. __S011 means RW access, but not
4090 + * execute, and avoids copy-on-write behavior.
4091 + * See protection_map in mmap.c. */
4092 + vma->vm_page_prot = PAGE_SHARED;
4093 +}
4094 +
4095 +#define vma_nr_pages(vma) \
4096 + ({unsigned long v = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); v;})
4097 +
4098 +extern struct page* get_colored_page(unsigned long color);
4099 +
4100 +static int do_map_colored_page(struct vm_area_struct *vma,
4101 + const unsigned long addr,
4102 + const unsigned long color_no)
4103 +{
4104 + int err = 0;
4105 + unsigned long offset = 2048;
4106 +
4107 + struct page *page = get_colored_page(color_no);
4108 +
4109 + if (!page) {
4110 + printk(KERN_INFO "Could not get page with color %lu.\n",
4111 + color_no);
4112 + err = -ENOMEM;
4113 + goto out;
4114 + }
4115 +
4116 + printk(KERN_INFO "vma: %p addr: 0x%lx color_no: %lu\n",
4117 + vma, addr, color_no);
4118 +
4119 + printk(KERN_INFO "vm_start: %lu vm_end: %lu\n",
4120 + vma->vm_start, vma->vm_end);
4121 +
4122 + printk(KERN_INFO "inserting page (pa: 0x%lx) at vaddr: 0x%lx "
4123 + "flags: 0x%lx prot: 0x%lx\n",
4124 + page_to_phys(page), addr,
4125 + vma->vm_flags, pgprot_val(vma->vm_page_prot));
4126 +
4127 +
4128 + err = vm_insert_page(vma, addr, page);
4129 + if (err) {
4130 + printk(KERN_INFO "vm_insert_page() failed (%d)\n", err);
4131 + err = -EINVAL;
4132 + goto out;
4133 + }
4134 +out:
4135 + return err;
4136 +}
4137 +
4138 +static int do_map_colored_pages(struct vm_area_struct *vma)
4139 +{
4140 + const unsigned long nr_pages = vma_nr_pages(vma);
4141 + unsigned long nr_mapped;
4142 + int i, start_bank = -1, start_color = -1;
4143 + int cur_bank = -1, cur_color = -1, err = 0;
4144 + int colors[16] = {0}, banks[8] = {0};
4145 +
4146 + if (bypass_cache == 1)
4147 + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
4148 +
4149 + for (i = 0; i < NUM_BANKS; i++) {
4150 + if (((color_param.bank >> i)&0x1) == 1)
4151 + banks[i] = 1;
4152 + }
4153 +
4154 + for (i = 0; i < NUM_COLORS; i++) {
4155 + if (((color_param.color >> i)&0x1) == 1)
4156 + colors[i] = 1;
4157 + }
4158 +
4159 + for (i = 0; i < NUM_BANKS; i++) {
4160 + if (banks[i] == 1) {
4161 + start_bank = i;
4162 + break;
4163 + }
4164 + }
4165 + for (i = 0; i < NUM_COLORS; i++) {
4166 + if (colors[i] == 1) {
4167 + start_color = i;
4168 + break;
4169 + }
4170 + }
4171 +
4172 + cur_bank = start_bank;
4173 + cur_color = start_color;
4174 +
4175 + for (i = 0; i < NUM_BANKS; i++) {
4176 + printk(KERN_INFO "BANK[%d] = %d\n", i, banks[i]);
4177 + }
4178 + printk(KERN_INFO "cur_bank = %d\n", cur_bank);
4179 + for (i = 0; i < NUM_COLORS; i++) {
4180 + printk(KERN_INFO "COLOR[%d] = %d\n", i, colors[i]);
4181 + }
4182 + printk(KERN_INFO "cur_color = %d\n", cur_color);
4183 +
4184 +
4185 + TRACE_CUR("allocating %lu pages (flags:%lx prot:%lx)\n",
4186 + nr_pages, vma->vm_flags, pgprot_val(vma->vm_page_prot));
4187 +
4188 + for (nr_mapped = 0; nr_mapped < nr_pages; nr_mapped++) {
4189 + const unsigned long addr = vma->vm_start + (nr_mapped << PAGE_SHIFT);
4190 + const unsigned long color_no = cur_bank*NUM_COLORS + cur_color;
4191 +
4192 + err = do_map_colored_page(vma, addr, color_no);
4193 + printk(KERN_INFO "mapped bank[%d], color[%d], color_no = %lu at 0x%lx\n",
4194 + cur_bank, cur_color, color_no, addr);
4195 + if (err) {
4196 + TRACE_CUR("Could not map colored page set.\n");
4197 + err = -EINVAL;
4198 + goto out;
4199 + }
4200 + do {
4201 + cur_color++;
4202 + } while(colors[cur_color] == 0);
4203 +
4204 + if (cur_color >= NUM_COLORS) {
4205 + do {
4206 + cur_bank++;
4207 + } while(banks[cur_bank] == 0);
4208 + cur_color = start_color;
4209 + }
4210 +
4211 + if (cur_bank >= NUM_BANKS) {
4212 + cur_bank = start_bank;
4213 + }
4214 + }
4215 + TRACE_CUR("Successfully mapped %lu pages.\n", nr_mapped);
4216 + out:
4217 + return err;
4218 +}
4219 +
4220 +static int map_colored_pages(struct vm_area_struct *vma)
4221 +{
4222 + int err = 0;
4223 +
4224 + printk(KERN_INFO "User requests %lu pages.\n", vma_nr_pages(vma));
4225 + if (MAX_COLORED_PAGE < vma_nr_pages(vma)) {
4226 + TRACE_CUR("Max page request %lu but want %lu.\n",
4227 + MAX_COLORED_PAGE, vma_nr_pages(vma));
4228 + err = -EINVAL;
4229 + goto out;
4230 + }
4231 + err = do_map_colored_pages(vma);
4232 +out:
4233 + return err;
4234 +}
4235 +
4236 +static void litmus_color_shm_vm_close(struct vm_area_struct *vma)
4237 +{
4238 +
4239 + TRACE_CUR("flags=0x%lx prot=0x%lx\n",
4240 + vma->vm_flags, pgprot_val(vma->vm_page_prot));
4241 +
4242 + TRACE_CUR("%p:%p vma:%p vma->vm_private_data:%p closed.\n",
4243 + (void*) vma->vm_start, (void*) vma->vm_end, vma,
4244 + vma->vm_private_data);
4245 +
4246 +}
4247 +
4248 +static int litmus_color_shm_vm_fault(struct vm_area_struct *vma,
4249 + struct vm_fault *vmf)
4250 +{
4251 + /* This function should never be called, since
4252 + * all pages should have been mapped by mmap()
4253 + * already. */
4254 + TRACE_CUR("flags=0x%lx (off:%ld)\n", vma->vm_flags, vmf->pgoff);
4255 + printk(KERN_INFO "flags=0x%lx (off:%ld)\n", vma->vm_flags, vmf->pgoff);
4256 +
4257 + printk(KERN_INFO "Page fault in color ctrl page! prot=0x%lx\n", pgprot_val(vma->vm_page_prot));
4258 +
4259 + return VM_FAULT_SIGBUS;
4260 +}
4261 +
4262 +static struct vm_operations_struct litmus_color_shm_vm_ops = {
4263 + .close = litmus_color_shm_vm_close,
4264 + .fault = litmus_color_shm_vm_fault,
4265 +};
4266 +
4267 +static int litmus_color_shm_mmap(struct file *filp, struct vm_area_struct *vma)
4268 +{
4269 + int err = 0;
4270 +
4271 + printk(KERN_INFO "mmap called\n");
4272 +
4273 + if (color_param.color == 0x00000000 || color_param.bank == 0x00000000) {
4274 + printk(KERN_INFO "color_info not set.\n");
4275 + return -EINVAL;
4276 + }
4277 + if (color_offset.offset == 0xffffffff || color_offset.lock == -1) {
4278 + printk(KERN_INFO "color_offset not set.\n");
4279 + return -EINVAL;
4280 + }
4281 +
4282 + err = mmap_common_checks(vma);
4283 + if (err) {
4284 + TRACE_CUR("failed mmap common checks\n");
4285 + goto out;
4286 + }
4287 +
4288 + vma->vm_ops = &litmus_color_shm_vm_ops;
4289 + mmap_common_vma_flags(vma);
4290 +
4291 + err = map_colored_pages(vma);
4292 +
4293 + TRACE_CUR("flags=0x%lx prot=0x%lx\n", vma->vm_flags,
4294 + pgprot_val(vma->vm_page_prot));
4295 +out:
4296 + color_param.color = 0x00000000;
4297 + color_param.bank = 0x00000000;
4298 + color_offset.offset = 0xffffffff;
4299 + color_offset.lock = -1;
4300 +
4301 + return err;
4302 +
4303 +}
4304 +
4305 +static long litmus_color_shm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
4306 +{
4307 + long err = -ENOIOCTLCMD;
4308 + struct color_ioctl_cmd color_info;
4309 + struct color_ioctl_offset color_off;
4310 +
4311 + printk(KERN_INFO "color_shm ioctl\n");
4312 +
4313 + if (_IOC_TYPE(cmd) != SHM_MAJOR)
4314 + return -ENOTTY;
4315 +
4316 +
4317 + switch (cmd) {
4318 + case SET_COLOR_SHM_CMD:
4319 +
4320 + err = copy_from_user(&color_info, (void*)arg, sizeof(struct color_ioctl_cmd));
4321 +
4322 + color_param.color = color_info.color;
4323 + color_param.bank = color_info.bank;
4324 + printk(KERN_INFO "COLOR = %x\n", color_param.color);
4325 + printk(KERN_INFO "BANK = %x\n", color_param.bank);
4326 + err = 0;
4327 + break;
4328 + case SET_COLOR_SHM_OFFSET:
4329 + err = copy_from_user(&color_off, (void*)arg, sizeof(struct color_ioctl_offset));
4330 +
4331 + color_offset.offset = color_off.offset;
4332 + color_offset.lock = color_off.lock;
4333 + printk(KERN_INFO "OFFSET = %x\n", color_offset.offset);
4334 + printk(KERN_INFO "LOCK = %d\n", color_offset.lock);
4335 + err = 0;
4336 + break;
4337 +
4338 + default:
4339 + printk(KERN_INFO "Invalid IOCTL CMD\n");
4340 + err = -EINVAL;
4341 + }
4342 +
4343 + return err;
4344 +}
4345 +
4346 +static struct file_operations litmus_color_shm_fops = {
4347 + .owner = THIS_MODULE,
4348 + .mmap = litmus_color_shm_mmap,
4349 + .unlocked_ioctl = litmus_color_shm_ioctl,
4350 +};
4351 +
4352 +static struct miscdevice litmus_color_shm_dev = {
4353 + .name = DEV_NAME,
4354 + .minor = MISC_DYNAMIC_MINOR,
4355 + .fops = &litmus_color_shm_fops,
4356 +};
4357 +
4358 +struct mutex bypass_mutex;
4359 +
4360 +int bypass_proc_handler(struct ctl_table *table, int write,
4361 + void __user *buffer, size_t *lenp, loff_t *ppos)
4362 +{
4363 + int ret, mode;
4364 +
4365 + mutex_lock(&bypass_mutex);
4366 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
4367 + printk(KERN_INFO "shm_bypass = %d\n", bypass_cache);
4368 + mutex_unlock(&bypass_mutex);
4369 +
4370 + return ret;
4371 +}
4372 +
4373 +static int zero = 0;
4374 +static int one = 1;
4375 +
4376 +static struct ctl_table cache_table[] =
4377 +{
4378 + {
4379 + .procname = "shm_bypass",
4380 + .mode = 0666,
4381 + .proc_handler = bypass_proc_handler,
4382 + .data = &bypass_cache,
4383 + .maxlen = sizeof(bypass_cache),
4384 + .extra1 = &zero,
4385 + .extra2 = &one,
4386 + },
4387 + { }
4388 +};
4389 +
4390 +static struct ctl_table litmus_dir_table[] = {
4391 + {
4392 + .procname = "litmus",
4393 + .mode = 0555,
4394 + .child = cache_table,
4395 + },
4396 + { }
4397 +};
4398 +
4399 +static struct ctl_table_header *litmus_sysctls;
4400 +
4401 +static int __init init_color_shm_devices(void)
4402 +{
4403 + int err;
4404 +
4405 + printk(KERN_INFO "Registering LITMUS^RT color_shm devices.\n");
4406 + litmus_sysctls = register_sysctl_table(litmus_dir_table);
4407 + if (!litmus_sysctls) {
4408 + printk(KERN_WARNING "Could not register LITMUS^RT color_shm sysctl.\n");
4409 + err = -EFAULT;
4410 + }
4411 +
4412 + mutex_init(&dev_lock);
4413 + mutex_init(&bypass_mutex);
4414 + color_param.color = 0x00000000;
4415 + color_param.bank = 0x00000000;
4416 + color_offset.offset = 0xffffffff;
4417 + color_offset.lock = -1;
4418 + bypass_cache = 0;
4419 + err = misc_register(&litmus_color_shm_dev);
4420 +
4421 + return err;
4422 +}
4423 +
4424 +static void __exit exit_color_shm_devices(void)
4425 +{
4426 + misc_deregister(&litmus_color_shm_dev);
4427 + printk(KERN_INFO "Unregistering %s device.\n", DEV_NAME);
4428 +}
4429 +
4430 +module_init(init_color_shm_devices);
4431 +module_exit(exit_color_shm_devices);
4432 \ No newline at end of file
4433 diff --git litmus/litmus.c litmus/litmus.c
4434 index db5ce0e9..e6d1ae9 100644
4435 --- litmus/litmus.c
4436 +++ litmus/litmus.c
4437 @@ -14,6 +14,11 @@
4438 #include <linux/sched/rt.h>
4439 #include <linux/rwsem.h>
4440 #include <linux/interrupt.h>
4441 +#include <linux/migrate.h>
4442 +#include <linux/mm.h>
4443 +#include <linux/memcontrol.h>
4444 +#include <linux/mm_inline.h>
4445 +#include <linux/mempool.h>
4446
4447 #include <litmus/litmus.h>
4448 #include <litmus/bheap.h>
4449 @@ -21,6 +26,10 @@
4450 #include <litmus/rt_domain.h>
4451 #include <litmus/litmus_proc.h>
4452 #include <litmus/sched_trace.h>
4453 +#include <litmus/cache_proc.h>
4454 +#include <litmus/mc2_common.h>
4455 +#include <litmus/replicate_lib.h>
4456 +#include <litmus/page_dev.h>
4457
4458 #ifdef CONFIG_SCHED_CPU_AFFINITY
4459 #include <litmus/affinity.h>
4460 @@ -31,6 +40,8 @@
4461 #include <trace/events/litmus.h>
4462 #endif
4463
4464 +extern void l2c310_flush_all(void);
4465 +
4466 /* Number of RT tasks that exist in the system */
4467 atomic_t rt_task_count = ATOMIC_INIT(0);
4468
4469 @@ -160,6 +171,14 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param)
4470 pid, tp.budget_policy);
4471 goto out_unlock;
4472 }
4473 +#ifdef CONFIG_PGMRT_SUPPORT
4474 + if (tp.pgm_type < PGM_NOT_A_NODE || tp.pgm_type > PGM_INTERNAL) {
4475 + printk(KERN_INFO "litmus: real-time task %d rejected "
4476 + "because of unknown PGM node type specified (%d)\n",
4477 + pid, tp.pgm_type);
4478 + goto out_unlock;
4479 + }
4480 +#endif
4481
4482 target->rt_param.task_params = tp;
4483
4484 @@ -314,6 +333,378 @@ asmlinkage long sys_null_call(cycles_t __user *ts)
4485 return ret;
4486 }
4487
4488 +asmlinkage long sys_reservation_create(int type, void __user *config)
4489 +{
4490 + return litmus->reservation_create(type, config);
4491 +}
4492 +
4493 +asmlinkage long sys_reservation_destroy(unsigned int reservation_id, int cpu)
4494 +{
4495 + return litmus->reservation_destroy(reservation_id, cpu);
4496 +}
4497 +
4498 +// This Address Decoding is used in imx6-sabredsd platform
4499 +#define NUM_BANKS 8
4500 +#define BANK_MASK 0x38000000
4501 +#define BANK_SHIFT 27
4502 +
4503 +#define NUM_COLORS 16
4504 +#define CACHE_MASK 0x0000f000
4505 +#define CACHE_SHIFT 12
4506 +/* Decoding page color, 0~15 */
4507 +static inline unsigned int page_color(struct page *page)
4508 +{
4509 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
4510 +}
4511 +
4512 +/* Decoding page bank number, 0~7 */
4513 +static inline unsigned int page_bank(struct page *page)
4514 +{
4515 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
4516 +}
4517 +
4518 +
4519 +static unsigned long color_mask;
4520 +
4521 +extern int isolate_lru_page(struct page *page);
4522 +extern void putback_movable_page(struct page *page);
4523 +extern struct page *new_alloc_page(struct page *page, unsigned long node, int **x);
4524 +
4525 +static struct page *alloc_colored_page(struct page *page, unsigned long node, int **result)
4526 +{
4527 + struct page *newpage;
4528 + gfp_t gfp_mask;
4529 +
4530 + gfp_mask = GFP_HIGHUSER_MOVABLE;
4531 + if (node != 8)
4532 + gfp_mask |= GFP_COLOR;
4533 +
4534 + newpage = alloc_pages(gfp_mask, 0);
4535 +
4536 + return newpage;
4537 +}
4538 +
4539 +#define INVALID_PFN (0xffffffff)
4540 +LIST_HEAD(shared_lib_pages);
4541 +
4542 +EXPORT_SYMBOL(shared_lib_pages);
4543 +
4544 +/* Reallocate pages of a task
4545 + * Private pages - Migrate to a new page.
4546 + * Shared pages - Use a replica. Make a replica if necessary.
4547 + * @cpu : CPU id of the calling task
4548 + * returns the number of pages that is not moved.
4549 + */
4550 +asmlinkage long sys_set_page_color(int cpu)
4551 +{
4552 + long ret = 0;
4553 + struct vm_area_struct *vma_itr = NULL;
4554 + int nr_pages = 0, nr_shared_pages = 0, nr_failed = 0, nr_not_migrated = 0;
4555 + unsigned long node;
4556 + enum crit_level lv;
4557 + struct mm_struct *mm;
4558 +
4559 + LIST_HEAD(pagelist);
4560 + LIST_HEAD(task_shared_pagelist);
4561 +
4562 + migrate_prep();
4563 +
4564 + /* Find the current mm_struct */
4565 + rcu_read_lock();
4566 + get_task_struct(current);
4567 + rcu_read_unlock();
4568 + mm = get_task_mm(current);
4569 + put_task_struct(current);
4570 +
4571 + down_read(&mm->mmap_sem);
4572 +
4573 + vma_itr = mm->mmap;
4574 + /* Iterate all vm_area_struct */
4575 + while (vma_itr != NULL) {
4576 + unsigned int num_pages = 0, i;
4577 + struct page *old_page = NULL;
4578 + int pages_in_vma = 0;
4579 +
4580 + num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE;
4581 + /* Traverse all pages in vm_area_struct */
4582 + for (i = 0; i < num_pages; i++) {
4583 + old_page = follow_page(vma_itr, vma_itr->vm_start + PAGE_SIZE*i, FOLL_GET|FOLL_SPLIT);
4584 +
4585 + if (IS_ERR(old_page))
4586 + continue;
4587 + if (!old_page)
4588 + continue;
4589 +
4590 + if (PageReserved(old_page)) {
4591 + TRACE("Reserved Page!\n");
4592 + put_page(old_page);
4593 + continue;
4594 + }
4595 + TRACE_TASK(current, "addr: %08x, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-");
4596 + pages_in_vma++;
4597 +
4598 + /* Conditions for replicable pages */
4599 + if (page_count(old_page) > 2 && vma_itr->vm_file != NULL && !(vma_itr->vm_flags&VM_WRITE)) {
4600 + struct shared_lib_page *lib_page;
4601 + int is_exist = 0;
4602 +
4603 + /* Update PSL (Per-core shared library (master)) list */
4604 + /* Check if this page is in the PSL list */
4605 + rcu_read_lock();
4606 + list_for_each_entry(lib_page, &shared_lib_pages, list)
4607 + {
4608 + if (page_to_pfn(old_page) == lib_page->master_pfn) {
4609 + is_exist = 1;
4610 + break;
4611 + }
4612 + }
4613 + rcu_read_unlock();
4614 +
4615 + if (is_exist == 0) {
4616 + int cpu_i;
4617 + lib_page = kmalloc(sizeof(struct shared_lib_page), GFP_KERNEL);
4618 + lib_page->master_page = old_page;
4619 + lib_page->master_pfn = page_to_pfn(old_page);
4620 + for (cpu_i = 0; cpu_i < NR_CPUS+1; cpu_i++) {
4621 + lib_page->r_page[cpu_i] = NULL;
4622 + lib_page->r_pfn[cpu_i] = INVALID_PFN;
4623 + }
4624 + list_add_tail(&lib_page->list, &shared_lib_pages);
4625 + }
4626 +
4627 + /* add to task_shared_pagelist */
4628 + ret = isolate_lru_page(old_page);
4629 + if (!ret) {
4630 + list_add_tail(&old_page->lru, &task_shared_pagelist);
4631 + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page));
4632 + nr_shared_pages++;
4633 + } else {
4634 + TRACE_TASK(current, "isolate_lru_page for a shared page failed\n");
4635 + nr_failed++;
4636 + }
4637 + put_page(old_page);
4638 + }
4639 + else {
4640 + ret = isolate_lru_page(old_page);
4641 + if (!ret) {
4642 + list_add_tail(&old_page->lru, &pagelist);
4643 + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page));
4644 + nr_pages++;
4645 + } else if (!is_in_correct_bank(old_page, cpu)) {
4646 + TRACE_TASK(current, "isolate_lru_page for a private page failed\n");
4647 + nr_failed++;
4648 + } else {
4649 + TRACE_TASK(current, "page is already in the correct bank\n");
4650 + }
4651 + put_page(old_page);
4652 + }
4653 + }
4654 + TRACE_TASK(current, "PAGES_IN_VMA = %d size = %d KB\n", pages_in_vma, pages_in_vma*4);
4655 + vma_itr = vma_itr->vm_next;
4656 + }
4657 +
4658 + ret = 0;
4659 + lv = tsk_rt(current)->mc2_data->crit;
4660 + if (cpu == -1)
4661 + node = 8;
4662 + else
4663 + node = cpu*2 + lv;
4664 +printk(KERN_INFO "READY TO MIGRATE\n");
4665 + /* Migrate private pages */
4666 + if (!list_empty(&pagelist)) {
4667 + ret = migrate_pages(&pagelist, alloc_colored_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL);
4668 + TRACE_TASK(current, "%ld pages not migrated.\n", ret);
4669 + nr_not_migrated = ret;
4670 + if (ret) {
4671 + putback_movable_pages(&pagelist);
4672 + }
4673 + }
4674 +printk(KERN_INFO "READY TO MIGRATE2\n");
4675 + /* Replicate shared pages */
4676 + if (!list_empty(&task_shared_pagelist)) {
4677 + ret = replicate_pages(&task_shared_pagelist, alloc_colored_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL);
4678 + TRACE_TASK(current, "%ld shared pages not migrated.\n", ret);
4679 + nr_not_migrated += ret;
4680 + if (ret) {
4681 + putback_movable_pages(&task_shared_pagelist);
4682 + }
4683 + }
4684 +
4685 + up_read(&mm->mmap_sem);
4686 +
4687 + TRACE_TASK(current, "nr_pages = %d nr_failed = %d nr_not_migrated = %d\n", nr_pages, nr_failed, nr_not_migrated);
4688 + printk(KERN_INFO "node = %ld, nr_private_pages = %d, nr_shared_pages = %d, nr_failed_to_isolate_lru = %d, nr_not_migrated = %d\n", node, nr_pages, nr_shared_pages, nr_failed, nr_not_migrated);
4689 +
4690 + return nr_not_migrated;
4691 +}
4692 +
4693 +/* sys_test_call() is a test system call for debugging */
4694 +asmlinkage long sys_test_call(unsigned int param)
4695 +{
4696 + long ret = 0;
4697 + struct vm_area_struct *vma_itr = NULL;
4698 +
4699 + TRACE_CUR("test_call param = %d\n", param);
4700 +
4701 + /* if param == 0,
4702 + * show vm regions and the page frame numbers
4703 + * associated with the vm region.
4704 + * if param == 1,
4705 + * print the master list.
4706 + */
4707 + if (param == 0) {
4708 + down_read(¤t->mm->mmap_sem);
4709 + vma_itr = current->mm->mmap;
4710 + while (vma_itr != NULL) {
4711 + int i, num_pages;
4712 + struct page* old_page;
4713 + TRACE_TASK(current, "------------------------------------------------------\n");
4714 + TRACE_TASK(current, "vm_start : %lx\n", vma_itr->vm_start);
4715 + TRACE_TASK(current, "vm_end : %lx\n", vma_itr->vm_end);
4716 + TRACE_TASK(current, "vm_flags : %lx\n", vma_itr->vm_flags);
4717 + TRACE_TASK(current, "vm_prot : %x\n", pgprot_val(vma_itr->vm_page_prot));
4718 + TRACE_TASK(current, "VM_SHARED? %ld\n", vma_itr->vm_flags & VM_SHARED);
4719 +
4720 + num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE;
4721 + for (i = 0; i < num_pages; i++) {
4722 + old_page = follow_page(vma_itr, vma_itr->vm_start + PAGE_SIZE*i, FOLL_GET|FOLL_SPLIT);
4723 +
4724 + if (IS_ERR(old_page))
4725 + continue;
4726 + if (!old_page)
4727 + continue;
4728 +
4729 + if (PageReserved(old_page)) {
4730 + TRACE("Reserved Page!\n");
4731 + put_page(old_page);
4732 + continue;
4733 + }
4734 +
4735 + TRACE_TASK(current, "addr: %08x, phy: %08x, color: %d, bank: %d, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s mapping: %p\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_phys(old_page), page_color(old_page), page_bank(old_page), page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-", &(old_page->mapping));
4736 + put_page(old_page);
4737 + }
4738 + vma_itr = vma_itr->vm_next;
4739 + }
4740 + TRACE_TASK(current, "------------------------------------------------------\n");
4741 + up_read(¤t->mm->mmap_sem);
4742 + } else if (param == 1) {
4743 + TRACE_TASK(current, "Shared pages and replicas.\n");
4744 + {
4745 + struct shared_lib_page *lpage;
4746 +
4747 + rcu_read_lock();
4748 + list_for_each_entry(lpage, &shared_lib_pages, list)
4749 + {
4750 + TRACE_TASK(current, "master_PFN = %05lx r_PFN = %05lx, %05lx, %05lx, %05lx, %05lx\n", lpage->master_pfn, lpage->r_pfn[0], lpage->r_pfn[1], lpage->r_pfn[2], lpage->r_pfn[3], lpage->r_pfn[4]);
4751 + }
4752 + rcu_read_unlock();
4753 + }
4754 + } else if (param == 2) {
4755 + flush_cache_all();
4756 + }
4757 +
4758 + return ret;
4759 +}
4760 +
4761 +asmlinkage long sys_recolor_mem(void* mem, int n_pages, int cpu)
4762 +{
4763 + long ret = 0;
4764 + struct vm_area_struct *vma_itr = NULL;
4765 + int nr_pages = 0, nr_failed = 0, nr_not_migrated = 0;
4766 + unsigned long node, user_vaddr;
4767 + enum crit_level lv;
4768 + struct mm_struct *mm;
4769 +
4770 + LIST_HEAD(pagelist);
4771 +
4772 + user_vaddr = (unsigned long)mem;
4773 +
4774 + migrate_prep();
4775 +
4776 + /* Find the current mm_struct */
4777 + rcu_read_lock();
4778 + get_task_struct(current);
4779 + rcu_read_unlock();
4780 + mm = get_task_mm(current);
4781 + put_task_struct(current);
4782 +
4783 + down_read(&mm->mmap_sem);
4784 +
4785 + vma_itr = mm->mmap;
4786 + /* Iterate all vm_area_struct */
4787 + while (vma_itr != NULL) {
4788 + unsigned int num_pages = 0, i;
4789 + struct page *old_page = NULL;
4790 + int pages_in_vma = 0;
4791 +
4792 + num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE;
4793 + if ((num_pages != n_pages) || (vma_itr->vm_start != user_vaddr)) {
4794 + vma_itr = vma_itr->vm_next;
4795 + continue;
4796 + }
4797 +
4798 + /* Traverse all pages in vm_area_struct */
4799 + for (i = 0; i < num_pages; i++) {
4800 + old_page = follow_page(vma_itr, vma_itr->vm_start + PAGE_SIZE*i, FOLL_GET|FOLL_SPLIT);
4801 +
4802 + if (IS_ERR(old_page))
4803 + continue;
4804 + if (!old_page)
4805 + continue;
4806 +
4807 + if (PageReserved(old_page)) {
4808 + TRACE("Reserved Page!\n");
4809 + put_page(old_page);
4810 + continue;
4811 + }
4812 + TRACE_TASK(current, "addr: %08x, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-");
4813 + pages_in_vma++;
4814 +
4815 + if (page_count(old_page) == 2) {
4816 + ret = isolate_lru_page(old_page);
4817 + if (!ret) {
4818 + list_add_tail(&old_page->lru, &pagelist);
4819 + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page));
4820 + nr_pages++;
4821 + } else if (!is_in_correct_bank(old_page, cpu)) {
4822 + TRACE_TASK(current, "isolate_lru_page for a private page failed\n");
4823 + nr_failed++;
4824 + } else {
4825 + TRACE_TASK(current, "page is already in the correct bank\n");
4826 + }
4827 + put_page(old_page);
4828 + }
4829 + }
4830 + TRACE_TASK(current, "PAGES_IN_VMA = %d size = %d KB\n", pages_in_vma, pages_in_vma*4);
4831 + vma_itr = vma_itr->vm_next;
4832 + }
4833 +
4834 + ret = 0;
4835 + lv = tsk_rt(current)->mc2_data->crit;
4836 + if (cpu == -1)
4837 + node = 8;
4838 + else
4839 + node = cpu*2 + lv;
4840 +
4841 + /* Migrate private pages */
4842 +
4843 + if (!list_empty(&pagelist)) {
4844 + ret = migrate_pages(&pagelist, alloc_colored_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL);
4845 + TRACE_TASK(current, "%ld pages not migrated.\n", ret);
4846 + nr_not_migrated = ret;
4847 + if (ret) {
4848 + putback_movable_pages(&pagelist);
4849 + }
4850 + }
4851 +
4852 + up_read(&mm->mmap_sem);
4853 +
4854 + TRACE_TASK(current, "nr_pages = %d nr_failed = %d nr_not_migrated = %d\n", nr_pages, nr_failed, nr_not_migrated);
4855 + printk(KERN_INFO "node = %ld, nr_private_pages = %d, nr_failed_to_isolate_lru = %d, nr_not_migrated = %d\n", node, nr_pages, nr_failed, nr_not_migrated);
4856 +
4857 + return nr_not_migrated;
4858 +}
4859 +
4860 /* p is a real-time task. Re-init its state as a best-effort task. */
4861 static void reinit_litmus_state(struct task_struct* p, int restore)
4862 {
4863 @@ -645,12 +1036,45 @@ static struct notifier_block shutdown_notifier = {
4864 .notifier_call = litmus_shutdown_nb,
4865 };
4866
4867 +extern mempool_t *msgpool;
4868 +struct page *msgpages;
4869 +void *msgvaddr;
4870 +
4871 +/* Pre-allocate message buffers and lock in the LLC for CL. Default is 4 */
4872 +static int litmus_msgpool_init(void)
4873 +{
4874 + int i, order = 4, size;
4875 +
4876 + msgpages = alloc_pages(GFP_KERNEL, order);
4877 + if (!msgpages) {
4878 + printk(KERN_WARNING "No memory\n");
4879 + return -ENOMEM;
4880 + }
4881 + msgvaddr = page_address(msgpages);
4882 +
4883 +
4884 + for (i = 0; i < 8; i++) {
4885 + cache_lockdown(0xFFFF8000, i);
4886 + }
4887 + size = order << (PAGE_SHIFT+2);
4888 + color_read_in_mem_lock(0xFFFF7FFF, 0xFFFF8000, msgvaddr, msgvaddr + size);
4889 + printk(KERN_INFO "SysV Msg Pool %d B initialized\n", size);
4890 +
4891 + return 0;
4892 +}
4893 +
4894 static int __init _init_litmus(void)
4895 {
4896 /* Common initializers,
4897 * mode change lock is used to enforce single mode change
4898 * operation.
4899 */
4900 +#if defined(CONFIG_CPU_V7)
4901 + unsigned int line_size_log = 5; // 2^5 = 32 byte
4902 + unsigned int cache_info_sets = 2048; // 64KB (way_size) / 32B (line_size) = 2048
4903 + printk("LITMIS^RT-ARM kernel\n");
4904 +#endif
4905 +
4906 printk("Starting LITMUS^RT kernel\n");
4907
4908 register_sched_plugin(&linux_sched_plugin);
4909 @@ -665,11 +1089,17 @@ static int __init _init_litmus(void)
4910 else
4911 printk("Could not register kill rt tasks magic sysrq.\n");
4912 #endif
4913 -
4914 init_litmus_proc();
4915
4916 register_reboot_notifier(&shutdown_notifier);
4917
4918 +#if defined(CONFIG_CPU_V7)
4919 + color_mask = ((cache_info_sets << line_size_log) - 1) ^ (PAGE_SIZE - 1);
4920 + printk("Page color mask %lx\n", color_mask);
4921 +#endif
4922 +
4923 + litmus_msgpool_init();
4924 +
4925 return 0;
4926 }
4927
4928 diff --git litmus/mc2_common.c litmus/mc2_common.c
4929 new file mode 100644
4930 index 0000000..a8ea5d9
4931 --- /dev/null
4932 +++ litmus/mc2_common.c
4933 @@ -0,0 +1,78 @@
4934 +/*
4935 + * litmus/mc2_common.c
4936 + *
4937 + * Common functions for MC2 plugin.
4938 + */
4939 +
4940 +#include <linux/percpu.h>
4941 +#include <linux/sched.h>
4942 +#include <linux/list.h>
4943 +#include <linux/slab.h>
4944 +#include <asm/uaccess.h>
4945 +
4946 +#include <litmus/litmus.h>
4947 +#include <litmus/sched_plugin.h>
4948 +#include <litmus/sched_trace.h>
4949 +
4950 +#include <litmus/mc2_common.h>
4951 +
4952 +long mc2_task_client_init(struct task_client *tc, struct mc2_task *mc2_param, struct task_struct *tsk, struct reservation *res)
4953 +{
4954 + task_client_init(tc, tsk, res);
4955 + if ((mc2_param->crit < CRIT_LEVEL_A) ||
4956 + (mc2_param->crit > CRIT_LEVEL_C))
4957 + return -EINVAL;
4958 +
4959 + TRACE_TASK(tsk, "mc2_task_client_init: crit_level = %d\n", mc2_param->crit);
4960 +
4961 + return 0;
4962 +}
4963 +
4964 +asmlinkage long sys_set_mc2_task_param(pid_t pid, struct mc2_task __user * param)
4965 +{
4966 + struct task_struct *target;
4967 + int retval = -EINVAL;
4968 + struct mc2_task *mp = kzalloc(sizeof(*mp), GFP_KERNEL);
4969 +
4970 + if (!mp)
4971 + return -ENOMEM;
4972 +
4973 + printk("Setting up mc^2 task parameters for process %d.\n", pid);
4974 +
4975 + if (pid < 0 || param == 0) {
4976 + goto out;
4977 + }
4978 + if (copy_from_user(mp, param, sizeof(*mp))) {
4979 + retval = -EFAULT;
4980 + goto out;
4981 + }
4982 +
4983 + /* Task search and manipulation must be protected */
4984 + read_lock_irq(&tasklist_lock);
4985 + if (!(target = find_task_by_vpid(pid))) {
4986 + retval = -ESRCH;
4987 + goto out_unlock;
4988 + }
4989 +
4990 + if (is_realtime(target)) {
4991 + /* The task is already a real-time task.
4992 + * We cannot not allow parameter changes at this point.
4993 + */
4994 + retval = -EBUSY;
4995 + goto out_unlock;
4996 + }
4997 + if (mp->crit < CRIT_LEVEL_A || mp->crit >= NUM_CRIT_LEVELS) {
4998 + printk(KERN_INFO "litmus: real-time task %d rejected "
4999 + "because of invalid criticality level\n", pid);
5000 + goto out_unlock;
5001 + }
5002 +
5003 + //target->rt_param.plugin_state = mp;
5004 + target->rt_param.mc2_data = mp;
5005 +
5006 + retval = 0;
5007 +out_unlock:
5008 + read_unlock_irq(&tasklist_lock);
5009 +out:
5010 + return retval;
5011 +}
5012 \ No newline at end of file
5013 diff --git litmus/page_dev.c litmus/page_dev.c
5014 new file mode 100644
5015 index 0000000..a26a712
5016 --- /dev/null
5017 +++ litmus/page_dev.c
5018 @@ -0,0 +1,414 @@
5019 +#include <litmus/page_dev.h>
5020 +#include <litmus/debug_trace.h>
5021 +
5022 +// This Address Decoding is used in imx6-sabredsd platform
5023 +#define NUM_BANKS 8
5024 +#define BANK_MASK 0x38000000
5025 +#define BANK_SHIFT 27
5026 +
5027 +#define NUM_COLORS 16
5028 +#define CACHE_MASK 0x0000f000
5029 +#define CACHE_SHIFT 12
5030 +
5031 +#define NR_LLC_PARTITIONS 9
5032 +#define NR_DRAM_PARTITIONS 5
5033 +
5034 +struct mutex dev_mutex;
5035 +
5036 +/* Initial partitions for LLC and DRAM bank */
5037 +/* 4 color for each core, all colors for Level C */
5038 +unsigned int llc_partition[NR_LLC_PARTITIONS] = {
5039 + 0x0000ffff, /* Core 0, and Level A*/
5040 + 0x0000ffff, /* Core 0, and Level B*/
5041 + 0x0000ffff, /* Core 1, and Level A*/
5042 + 0x0000ffff, /* Core 1, and Level B*/
5043 + 0x0000ffff, /* Core 2, and Level A*/
5044 + 0x0000ffff, /* Core 2, and Level B*/
5045 + 0x0000ffff, /* Core 3, and Level A*/
5046 + 0x0000ffff, /* Core 3, and Level B*/
5047 + 0x0000ffff, /* Level C */
5048 +};
5049 +
5050 +/* 1 bank for each core, 2 banks for Level C */
5051 +unsigned int dram_partition[NR_DRAM_PARTITIONS] = {
5052 + 0x00000010,
5053 + 0x00000020,
5054 + 0x00000040,
5055 + 0x00000080,
5056 + 0x0000000f,
5057 +};
5058 +
5059 +/* Decoding page color, 0~15 */
5060 +static inline unsigned int page_color(struct page *page)
5061 +{
5062 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
5063 +}
5064 +
5065 +/* Decoding page bank number, 0~7 */
5066 +static inline unsigned int page_bank(struct page *page)
5067 +{
5068 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
5069 +}
5070 +
5071 +int bank_to_partition(unsigned int bank)
5072 +{
5073 + int i;
5074 + unsigned int bank_bit = 0x1<<bank;
5075 +
5076 + for (i = 0; i<NR_DRAM_PARTITIONS; i++) {
5077 + if (dram_partition[i] & bank_bit)
5078 + return i;
5079 + }
5080 +
5081 + return -EINVAL;
5082 +}
5083 +
5084 +int get_area_index(int cpu)
5085 +{
5086 + int index = 0x10, area_index = 0;
5087 +
5088 + while (index < 0x100) {
5089 + if (dram_partition[cpu]&index)
5090 + break;
5091 + index = index << 1;
5092 + area_index++;
5093 + }
5094 +
5095 + return area_index;
5096 +}
5097 +
5098 +/* use this function ONLY for Lv.A/B pages */
5099 +int is_in_correct_bank(struct page* page, int cpu)
5100 +{
5101 + int bank;
5102 + unsigned int page_bank_bit;
5103 +
5104 + bank = page_bank(page);
5105 + page_bank_bit = 1 << bank;
5106 +
5107 + if (cpu == -1 || cpu == NR_CPUS)
5108 + return (page_bank_bit & dram_partition[NR_CPUS]);
5109 + else
5110 + return (page_bank_bit & dram_partition[cpu]);
5111 +}
5112 +
5113 +int is_in_llc_partition(struct page* page, int cpu)
5114 +{
5115 + int color;
5116 + unsigned int page_color_bit;
5117 +
5118 + color = page_color(page);
5119 + page_color_bit = 1 << color;
5120 +
5121 + if (cpu == -1 || cpu == NR_CPUS)
5122 + return (page_color_bit & llc_partition[8]);
5123 + else
5124 + return (page_color_bit & (llc_partition[cpu*2] | llc_partition[cpu*2+1]));
5125 +}
5126 +
5127 +/* Bounds for values */
5128 +unsigned int llc_partition_max = 0x0000ffff;
5129 +unsigned int llc_partition_min = 0;
5130 +unsigned int dram_partition_max = 0x000000ff;
5131 +unsigned int dram_partition_min = 0;
5132 +
5133 +/* slabtest module */
5134 +int buf_size = 0;
5135 +int buf_num = 1;
5136 +
5137 +int slabtest_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
5138 +{
5139 + int ret = 0, i;
5140 + int** testbuffer;
5141 + mutex_lock(&dev_mutex);
5142 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
5143 +
5144 + if (ret)
5145 + goto out;
5146 +
5147 + if (write) {
5148 + int idx;
5149 + int n_data = buf_size/sizeof(int);
5150 +
5151 + printk(KERN_INFO "-------SLABTEST on CPU%d with %d buffer size\n", raw_smp_processor_id(), buf_size);
5152 +
5153 + testbuffer = kmalloc(sizeof(int*)*buf_num, GFP_KERNEL|GFP_COLOR);
5154 +
5155 + for (idx=0; idx<buf_num; idx++)
5156 + {
5157 + printk(KERN_INFO "kmalloc size %d, n_data %d\n", buf_size, n_data);
5158 + testbuffer[idx] = kmalloc(buf_size, GFP_KERNEL|GFP_COLOR);
5159 +
5160 + if (!testbuffer[idx]) {
5161 + printk(KERN_ERR "kmalloc failed size = %d\n", buf_size);
5162 + goto out;
5163 + }
5164 + }
5165 +
5166 +
5167 + /* do test */
5168 + for (idx=0; idx<buf_num; idx++)
5169 + {
5170 + int t = 0;
5171 + printk(KERN_INFO "kmalloc size = %d n_data = %d\n", buf_size, n_data);
5172 + printk(KERN_INFO "write data to buffer\n");
5173 + for (i = 0; i < n_data; i++) {
5174 + testbuffer[idx][i] = i%27;
5175 + }
5176 + printk(KERN_INFO "read data from buffer\n");
5177 + for (i = 0; i < n_data; i++) {
5178 + t += testbuffer[idx][i];
5179 + //printk(KERN_INFO "[%d] = %d\n", i, testbuffer[idx][i]);
5180 + }
5181 + }
5182 +
5183 + for (idx=0; idx<buf_num; idx++)
5184 + kfree(testbuffer[idx]);
5185 +
5186 + kfree(testbuffer);
5187 + printk(KERN_INFO "-------SLABTEST FINISHED on CPU%d\n", raw_smp_processor_id());
5188 + }
5189 +out:
5190 + mutex_unlock(&dev_mutex);
5191 + return ret;
5192 +}
5193 +
5194 +int num_buffer_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
5195 +{
5196 + int ret = 0;
5197 + mutex_lock(&dev_mutex);
5198 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
5199 +
5200 + if (ret)
5201 + goto out;
5202 +
5203 + if (write) {
5204 + printk(KERN_INFO "buf_num = %d\n", buf_num);
5205 + }
5206 +out:
5207 + mutex_unlock(&dev_mutex);
5208 + return ret;
5209 +}
5210 +
5211 +static struct ctl_table partition_table[] =
5212 +{
5213 +
5214 + {
5215 + .procname = "C0_LA_color",
5216 + .mode = 0666,
5217 + .proc_handler = llc_partition_handler,
5218 + .data = &llc_partition[0],
5219 + .maxlen = sizeof(llc_partition[0]),
5220 + .extra1 = &llc_partition_min,
5221 + .extra2 = &llc_partition_max,
5222 + },
5223 + {
5224 + .procname = "C0_LB_color",
5225 + .mode = 0666,
5226 + .proc_handler = llc_partition_handler,
5227 + .data = &llc_partition[1],
5228 + .maxlen = sizeof(llc_partition[1]),
5229 + .extra1 = &llc_partition_min,
5230 + .extra2 = &llc_partition_max,
5231 + },
5232 + {
5233 + .procname = "C1_LA_color",
5234 + .mode = 0666,
5235 + .proc_handler = llc_partition_handler,
5236 + .data = &llc_partition[2],
5237 + .maxlen = sizeof(llc_partition[2]),
5238 + .extra1 = &llc_partition_min,
5239 + .extra2 = &llc_partition_max,
5240 + },
5241 + {
5242 + .procname = "C1_LB_color",
5243 + .mode = 0666,
5244 + .proc_handler = llc_partition_handler,
5245 + .data = &llc_partition[3],
5246 + .maxlen = sizeof(llc_partition[3]),
5247 + .extra1 = &llc_partition_min,
5248 + .extra2 = &llc_partition_max,
5249 + },
5250 + {
5251 + .procname = "C2_LA_color",
5252 + .mode = 0666,
5253 + .proc_handler = llc_partition_handler,
5254 + .data = &llc_partition[4],
5255 + .maxlen = sizeof(llc_partition[4]),
5256 + .extra1 = &llc_partition_min,
5257 + .extra2 = &llc_partition_max,
5258 + },
5259 + {
5260 + .procname = "C2_LB_color",
5261 + .mode = 0666,
5262 + .proc_handler = llc_partition_handler,
5263 + .data = &llc_partition[5],
5264 + .maxlen = sizeof(llc_partition[5]),
5265 + .extra1 = &llc_partition_min,
5266 + .extra2 = &llc_partition_max,
5267 + },
5268 + {
5269 + .procname = "C3_LA_color",
5270 + .mode = 0666,
5271 + .proc_handler = llc_partition_handler,
5272 + .data = &llc_partition[6],
5273 + .maxlen = sizeof(llc_partition[6]),
5274 + .extra1 = &llc_partition_min,
5275 + .extra2 = &llc_partition_max,
5276 + },
5277 + {
5278 + .procname = "C3_LB_color",
5279 + .mode = 0666,
5280 + .proc_handler = llc_partition_handler,
5281 + .data = &llc_partition[7],
5282 + .maxlen = sizeof(llc_partition[7]),
5283 + .extra1 = &llc_partition_min,
5284 + .extra2 = &llc_partition_max,
5285 + },
5286 + {
5287 + .procname = "Call_LC_color",
5288 + .mode = 0666,
5289 + .proc_handler = llc_partition_handler,
5290 + .data = &llc_partition[8],
5291 + .maxlen = sizeof(llc_partition[8]),
5292 + .extra1 = &llc_partition_min,
5293 + .extra2 = &llc_partition_max,
5294 + },
5295 + {
5296 + .procname = "C0_dram",
5297 + .mode = 0666,
5298 + .proc_handler = dram_partition_handler,
5299 + .data = &dram_partition[0],
5300 + .maxlen = sizeof(llc_partition[0]),
5301 + .extra1 = &dram_partition_min,
5302 + .extra2 = &dram_partition_max,
5303 + },
5304 + {
5305 + .procname = "C1_dram",
5306 + .mode = 0666,
5307 + .proc_handler = dram_partition_handler,
5308 + .data = &dram_partition[1],
5309 + .maxlen = sizeof(llc_partition[1]),
5310 + .extra1 = &dram_partition_min,
5311 + .extra2 = &dram_partition_max,
5312 + },
5313 + {
5314 + .procname = "C2_dram",
5315 + .mode = 0666,
5316 + .proc_handler = dram_partition_handler,
5317 + .data = &dram_partition[2],
5318 + .maxlen = sizeof(llc_partition[2]),
5319 + .extra1 = &dram_partition_min,
5320 + .extra2 = &dram_partition_max,
5321 + },
5322 + {
5323 + .procname = "C3_dram",
5324 + .mode = 0666,
5325 + .proc_handler = dram_partition_handler,
5326 + .data = &dram_partition[3],
5327 + .maxlen = sizeof(llc_partition[3]),
5328 + .extra1 = &dram_partition_min,
5329 + .extra2 = &dram_partition_max,
5330 + },
5331 + {
5332 + .procname = "CS_dram",
5333 + .mode = 0666,
5334 + .proc_handler = dram_partition_handler,
5335 + .data = &dram_partition[4],
5336 + .maxlen = sizeof(llc_partition[4]),
5337 + .extra1 = &dram_partition_min,
5338 + .extra2 = &dram_partition_max,
5339 + },
5340 + {
5341 + .procname = "slabtest",
5342 + .mode = 0666,
5343 + .proc_handler = slabtest_handler,
5344 + .data = &buf_size,
5345 + .maxlen = sizeof(buf_size),
5346 + },
5347 + {
5348 + .procname = "num_buffer",
5349 + .mode = 0666,
5350 + .proc_handler = num_buffer_handler,
5351 + .data = &buf_num,
5352 + .maxlen = sizeof(buf_num),
5353 + },
5354 + { }
5355 +};
5356 +
5357 +static struct ctl_table litmus_dir_table[] = {
5358 + {
5359 + .procname = "litmus",
5360 + .mode = 0555,
5361 + .child = partition_table,
5362 + },
5363 + { }
5364 +};
5365 +
5366 +static struct ctl_table_header *litmus_sysctls;
5367 +
5368 +int llc_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
5369 +{
5370 + int ret = 0, i;
5371 + mutex_lock(&dev_mutex);
5372 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
5373 + if (ret)
5374 + goto out;
5375 + if (write) {
5376 + printk("New LLC Partition : \n");
5377 + for(i = 0; i < NR_LLC_PARTITIONS; i++) {
5378 + printk("llc_partition[%d] = 0x%04x\n", i, llc_partition[i]);
5379 + }
5380 + }
5381 +out:
5382 + mutex_unlock(&dev_mutex);
5383 + return ret;
5384 +}
5385 +
5386 +int dram_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
5387 +{
5388 + int ret = 0, i;
5389 + mutex_lock(&dev_mutex);
5390 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
5391 + if (ret)
5392 + goto out;
5393 + if (write) {
5394 + for(i = 0; i < NR_DRAM_PARTITIONS; i++) {
5395 + printk("dram_partition[%d] = 0x%04x\n", i, dram_partition[i]);
5396 + }
5397 + }
5398 +out:
5399 + mutex_unlock(&dev_mutex);
5400 + return ret;
5401 +}
5402 +
5403 +/*
5404 + * Initialize this page_dev proc.
5405 + */
5406 +static int __init init_litmus_page_dev(void)
5407 +{
5408 + int err = 0;
5409 +
5410 + printk("Initialize page_dev.c\n");
5411 +
5412 + mutex_init(&dev_mutex);
5413 +
5414 + litmus_sysctls = register_sysctl_table(litmus_dir_table);
5415 + if (!litmus_sysctls) {
5416 + printk(KERN_WARNING "Could not register LITMUS^RT page_dev sysctl.\n");
5417 + err = -EFAULT;
5418 + goto out;
5419 + }
5420 +
5421 + printk(KERN_INFO "Registering LITMUS^RT page_dev proc.\n");
5422 +out:
5423 + return err;
5424 +}
5425 +
5426 +static void __exit exit_litmus_page_dev(void)
5427 +{
5428 + mutex_destroy(&dev_mutex);
5429 +}
5430 +
5431 +module_init(init_litmus_page_dev);
5432 +module_exit(exit_litmus_page_dev);
5433 \ No newline at end of file
5434 diff --git litmus/polling_reservations.c litmus/polling_reservations.c
5435 new file mode 100644
5436 index 0000000..d44a403
5437 --- /dev/null
5438 +++ litmus/polling_reservations.c
5439 @@ -0,0 +1,563 @@
5440 +#include <linux/sched.h>
5441 +
5442 +#include <litmus/litmus.h>
5443 +#include <litmus/reservation.h>
5444 +#include <litmus/polling_reservations.h>
5445 +
5446 +static void periodic_polling_client_arrives(
5447 + struct reservation* res,
5448 + struct reservation_client *client
5449 +)
5450 +{
5451 + struct polling_reservation *pres =
5452 + container_of(res, struct polling_reservation, res);
5453 + lt_t instances, tmp;
5454 +
5455 + list_add_tail(&client->list, &res->clients);
5456 +
5457 + switch (res->state) {
5458 + case RESERVATION_INACTIVE:
5459 + /* Figure out next replenishment time. */
5460 + if (res->env->time_zero == 0) {
5461 + tmp = res->env->current_time - res->env->time_zero;
5462 + instances = div64_u64(tmp, pres->period);
5463 + res->next_replenishment =
5464 + (instances + 1) * pres->period + pres->offset;
5465 + }
5466 + else {
5467 + tmp = res->env->current_time - res->env->time_zero;
5468 + instances = div64_u64(tmp, pres->period);
5469 + res->next_replenishment = res->env->time_zero + instances * pres->period;
5470 + }
5471 +
5472 + TRACE("ENV_TIME_ZERO %llu\n", res->env->time_zero);
5473 + TRACE("pol-res: R%d activate tmp=%llu instances=%llu period=%llu nextrp=%llu cur=%llu\n",
5474 + res->id, tmp, instances, pres->period, res->next_replenishment,
5475 + res->env->current_time);
5476 +
5477 + res->env->change_state(res->env, res,
5478 + RESERVATION_DEPLETED);
5479 + break;
5480 +
5481 + case RESERVATION_ACTIVE:
5482 + case RESERVATION_DEPLETED:
5483 + /* do nothing */
5484 + break;
5485 +
5486 + case RESERVATION_ACTIVE_IDLE:
5487 + res->blocked_by_ghost = 0;
5488 + res->env->change_state(res->env, res,
5489 + RESERVATION_ACTIVE);
5490 + break;
5491 + }
5492 +}
5493 +
5494 +
5495 +static void periodic_polling_client_departs(
5496 + struct reservation *res,
5497 + struct reservation_client *client,
5498 + int did_signal_job_completion
5499 +)
5500 +{
5501 + list_del(&client->list);
5502 +
5503 + switch (res->state) {
5504 + case RESERVATION_INACTIVE:
5505 + case RESERVATION_ACTIVE_IDLE:
5506 + BUG(); /* INACTIVE or IDLE <=> no client */
5507 + break;
5508 +
5509 + case RESERVATION_ACTIVE:
5510 + if (list_empty(&res->clients)) {
5511 + res->env->change_state(res->env, res,
5512 +// RESERVATION_ACTIVE_IDLE);
5513 + res->cur_budget ?
5514 + RESERVATION_ACTIVE_IDLE :
5515 + RESERVATION_DEPLETED);
5516 +// did_signal_job_completion ?
5517 +// RESERVATION_DEPLETED :
5518 +// RESERVATION_ACTIVE_IDLE);
5519 + } /* else: nothing to do, more clients ready */
5520 + break;
5521 +
5522 + case RESERVATION_DEPLETED:
5523 + /* do nothing */
5524 + break;
5525 + }
5526 +}
5527 +
5528 +static void periodic_polling_on_replenishment(
5529 + struct reservation *res
5530 +)
5531 +{
5532 + struct polling_reservation *pres =
5533 + container_of(res, struct polling_reservation, res);
5534 +
5535 + /* replenish budget */
5536 + res->cur_budget = pres->max_budget;
5537 + res->next_replenishment += pres->period;
5538 + res->budget_consumed = 0;
5539 +
5540 + TRACE("polling_replenish(%u): next_replenishment=%llu\n", res->id, res->next_replenishment);
5541 + switch (res->state) {
5542 + case RESERVATION_DEPLETED:
5543 + case RESERVATION_INACTIVE:
5544 + case RESERVATION_ACTIVE_IDLE:
5545 + if (list_empty(&res->clients))
5546 + /* no clients => poll again later */
5547 + res->env->change_state(res->env, res,
5548 + RESERVATION_INACTIVE);
5549 + else
5550 + /* we have clients & budget => ACTIVE */
5551 + res->env->change_state(res->env, res,
5552 + RESERVATION_ACTIVE);
5553 + break;
5554 +
5555 + case RESERVATION_ACTIVE:
5556 + /* Replenished while active => tardy? In any case,
5557 + * go ahead and stay active. */
5558 + break;
5559 + }
5560 +}
5561 +
5562 +static void periodic_polling_on_replenishment_edf(
5563 + struct reservation *res
5564 +)
5565 +{
5566 + struct polling_reservation *pres =
5567 + container_of(res, struct polling_reservation, res);
5568 +
5569 + /* update current priority */
5570 + res->priority = res->next_replenishment + pres->deadline;
5571 +
5572 + /* do common updates */
5573 + periodic_polling_on_replenishment(res);
5574 +}
5575 +
5576 +static void common_drain_budget(
5577 + struct reservation *res,
5578 + lt_t how_much)
5579 +{
5580 + if (how_much >= res->cur_budget)
5581 + res->cur_budget = 0;
5582 + else
5583 + res->cur_budget -= how_much;
5584 +
5585 + res->budget_consumed += how_much;
5586 + res->budget_consumed_total += how_much;
5587 +
5588 + switch (res->state) {
5589 + case RESERVATION_DEPLETED:
5590 + case RESERVATION_INACTIVE:
5591 + //BUG();
5592 + TRACE("!!!!!!!!!!!!!!!STATE ERROR R%d STATE(%d)\n", res->id, res->state);
5593 + break;
5594 +
5595 + case RESERVATION_ACTIVE_IDLE:
5596 + case RESERVATION_ACTIVE:
5597 + if (!res->cur_budget) {
5598 + res->env->change_state(res->env, res,
5599 + RESERVATION_DEPLETED);
5600 + } /* else: stay in current state */
5601 + break;
5602 + }
5603 +}
5604 +
5605 +static struct reservation_ops periodic_polling_ops_fp = {
5606 + .dispatch_client = default_dispatch_client,
5607 + .client_arrives = periodic_polling_client_arrives,
5608 + .client_departs = periodic_polling_client_departs,
5609 + .replenish = periodic_polling_on_replenishment,
5610 + .drain_budget = common_drain_budget,
5611 +};
5612 +
5613 +static struct reservation_ops periodic_polling_ops_edf = {
5614 + .dispatch_client = default_dispatch_client,
5615 + .client_arrives = periodic_polling_client_arrives,
5616 + .client_departs = periodic_polling_client_departs,
5617 + .replenish = periodic_polling_on_replenishment_edf,
5618 + .drain_budget = common_drain_budget,
5619 +};
5620 +
5621 +
5622 +
5623 +
5624 +static void sporadic_polling_client_arrives_fp(
5625 + struct reservation* res,
5626 + struct reservation_client *client
5627 +)
5628 +{
5629 + struct polling_reservation *pres =
5630 + container_of(res, struct polling_reservation, res);
5631 +
5632 + list_add_tail(&client->list, &res->clients);
5633 +
5634 + switch (res->state) {
5635 + case RESERVATION_INACTIVE:
5636 + /* Replenish now. */
5637 + res->cur_budget = pres->max_budget;
5638 + res->next_replenishment =
5639 + res->env->current_time + pres->period;
5640 +
5641 + res->env->change_state(res->env, res,
5642 + RESERVATION_ACTIVE);
5643 + break;
5644 +
5645 + case RESERVATION_ACTIVE:
5646 + case RESERVATION_DEPLETED:
5647 + /* do nothing */
5648 + break;
5649 +
5650 + case RESERVATION_ACTIVE_IDLE:
5651 + res->env->change_state(res->env, res,
5652 + RESERVATION_ACTIVE);
5653 + break;
5654 + }
5655 +}
5656 +
5657 +static void sporadic_polling_client_arrives_edf(
5658 + struct reservation* res,
5659 + struct reservation_client *client
5660 +)
5661 +{
5662 + struct polling_reservation *pres =
5663 + container_of(res, struct polling_reservation, res);
5664 +
5665 + list_add_tail(&client->list, &res->clients);
5666 +
5667 + switch (res->state) {
5668 + case RESERVATION_INACTIVE:
5669 + /* Replenish now. */
5670 + res->cur_budget = pres->max_budget;
5671 + res->next_replenishment =
5672 + res->env->current_time + pres->period;
5673 + res->priority =
5674 + res->env->current_time + pres->deadline;
5675 +
5676 + res->env->change_state(res->env, res,
5677 + RESERVATION_ACTIVE);
5678 + break;
5679 +
5680 + case RESERVATION_ACTIVE:
5681 + case RESERVATION_DEPLETED:
5682 + /* do nothing */
5683 + break;
5684 +
5685 + case RESERVATION_ACTIVE_IDLE:
5686 + res->env->change_state(res->env, res,
5687 + RESERVATION_ACTIVE);
5688 + break;
5689 + }
5690 +}
5691 +
5692 +static struct reservation_ops sporadic_polling_ops_fp = {
5693 + .dispatch_client = default_dispatch_client,
5694 + .client_arrives = sporadic_polling_client_arrives_fp,
5695 + .client_departs = periodic_polling_client_departs,
5696 + .replenish = periodic_polling_on_replenishment,
5697 + .drain_budget = common_drain_budget,
5698 +};
5699 +
5700 +static struct reservation_ops sporadic_polling_ops_edf = {
5701 + .dispatch_client = default_dispatch_client,
5702 + .client_arrives = sporadic_polling_client_arrives_edf,
5703 + .client_departs = periodic_polling_client_departs,
5704 + .replenish = periodic_polling_on_replenishment_edf,
5705 + .drain_budget = common_drain_budget,
5706 +};
5707 +
5708 +void polling_reservation_init(
5709 + struct polling_reservation *pres,
5710 + int use_edf_prio,
5711 + int use_periodic_polling,
5712 + lt_t budget, lt_t period, lt_t deadline, lt_t offset
5713 +)
5714 +{
5715 + if (!deadline)
5716 + deadline = period;
5717 + BUG_ON(budget > period);
5718 + BUG_ON(budget > deadline);
5719 + BUG_ON(offset >= period);
5720 +
5721 + reservation_init(&pres->res);
5722 + pres->max_budget = budget;
5723 + pres->period = period;
5724 + pres->deadline = deadline;
5725 + pres->offset = offset;
5726 + TRACE_TASK(current, "polling_reservation_init: periodic %d, use_edf %d\n", use_periodic_polling, use_edf_prio);
5727 + if (use_periodic_polling) {
5728 + if (use_edf_prio)
5729 + pres->res.ops = &periodic_polling_ops_edf;
5730 + else
5731 + pres->res.ops = &periodic_polling_ops_fp;
5732 + } else {
5733 + if (use_edf_prio)
5734 + pres->res.ops = &sporadic_polling_ops_edf;
5735 + else
5736 + pres->res.ops = &sporadic_polling_ops_fp;
5737 + }
5738 +}
5739 +
5740 +
5741 +static lt_t td_cur_major_cycle_start(struct table_driven_reservation *tdres)
5742 +{
5743 + lt_t x, tmp;
5744 +
5745 + tmp = tdres->res.env->current_time - tdres->res.env->time_zero;
5746 + x = div64_u64(tmp, tdres->major_cycle);
5747 + x *= tdres->major_cycle;
5748 + return x;
5749 +}
5750 +
5751 +
5752 +static lt_t td_next_major_cycle_start(struct table_driven_reservation *tdres)
5753 +{
5754 + lt_t x, tmp;
5755 +
5756 + tmp = tdres->res.env->current_time - tdres->res.env->time_zero;
5757 + x = div64_u64(tmp, tdres->major_cycle) + 1;
5758 + x *= tdres->major_cycle;
5759 + return x;
5760 +}
5761 +
5762 +static void td_client_arrives(
5763 + struct reservation* res,
5764 + struct reservation_client *client
5765 +)
5766 +{
5767 + struct table_driven_reservation *tdres =
5768 + container_of(res, struct table_driven_reservation, res);
5769 +
5770 + list_add_tail(&client->list, &res->clients);
5771 +
5772 + switch (res->state) {
5773 + case RESERVATION_INACTIVE:
5774 + /* Figure out first replenishment time. */
5775 + tdres->major_cycle_start = td_next_major_cycle_start(tdres);
5776 + res->next_replenishment = tdres->major_cycle_start;
5777 + res->next_replenishment += tdres->intervals[0].start;
5778 + tdres->next_interval = 0;
5779 +
5780 + res->env->change_state(res->env, res,
5781 + RESERVATION_DEPLETED);
5782 + break;
5783 +
5784 + case RESERVATION_ACTIVE:
5785 + case RESERVATION_DEPLETED:
5786 + /* do nothing */
5787 + break;
5788 +
5789 + case RESERVATION_ACTIVE_IDLE:
5790 + res->env->change_state(res->env, res,
5791 + RESERVATION_ACTIVE);
5792 + break;
5793 + }
5794 +}
5795 +
5796 +static void td_client_departs(
5797 + struct reservation *res,
5798 + struct reservation_client *client,
5799 + int did_signal_job_completion
5800 +)
5801 +{
5802 + list_del(&client->list);
5803 +
5804 + switch (res->state) {
5805 + case RESERVATION_INACTIVE:
5806 + case RESERVATION_ACTIVE_IDLE:
5807 + //BUG(); /* INACTIVE or IDLE <=> no client */
5808 + break;
5809 +
5810 + case RESERVATION_ACTIVE:
5811 + if (list_empty(&res->clients)) {
5812 + res->env->change_state(res->env, res,
5813 + RESERVATION_ACTIVE_IDLE);
5814 + } /* else: nothing to do, more clients ready */
5815 + break;
5816 +
5817 + case RESERVATION_DEPLETED:
5818 + /* do nothing */
5819 + break;
5820 + }
5821 +}
5822 +
5823 +static lt_t td_time_remaining_until_end(struct table_driven_reservation *tdres)
5824 +{
5825 + lt_t now = tdres->res.env->current_time;
5826 + lt_t end = tdres->cur_interval.end;
5827 + //TRACE("td_remaining(%u): start=%llu now=%llu end=%llu state=%d\n", tdres->res.id, tdres->cur_interval.start, now, end, tdres->res.state);
5828 + if (now >= end)
5829 + return 0;
5830 + else
5831 + return end - now;
5832 +}
5833 +
5834 +static void td_replenish(
5835 + struct reservation *res)
5836 +{
5837 + struct table_driven_reservation *tdres =
5838 + container_of(res, struct table_driven_reservation, res);
5839 +
5840 + //TRACE("td_replenish(%u): expected_replenishment=%llu\n", res->id, res->next_replenishment);
5841 +
5842 + /* figure out current interval */
5843 + tdres->cur_interval.start = tdres->major_cycle_start +
5844 + tdres->intervals[tdres->next_interval].start;
5845 + tdres->cur_interval.end = tdres->major_cycle_start +
5846 + tdres->intervals[tdres->next_interval].end;
5847 +/* TRACE("major_cycle_start=%llu => [%llu, %llu]\n",
5848 + tdres->major_cycle_start,
5849 + tdres->cur_interval.start,
5850 + tdres->cur_interval.end);
5851 +*/
5852 + /* reset budget */
5853 + res->cur_budget = td_time_remaining_until_end(tdres);
5854 + res->budget_consumed = 0;
5855 + //TRACE("td_replenish(%u): %s budget=%llu\n", res->id, res->cur_budget ? "" : "WARNING", res->cur_budget);
5856 +
5857 + /* prepare next slot */
5858 + tdres->next_interval = (tdres->next_interval + 1) % tdres->num_intervals;
5859 + if (!tdres->next_interval)
5860 + /* wrap to next major cycle */
5861 + tdres->major_cycle_start += tdres->major_cycle;
5862 +
5863 + /* determine next time this reservation becomes eligible to execute */
5864 + res->next_replenishment = tdres->major_cycle_start;
5865 + res->next_replenishment += tdres->intervals[tdres->next_interval].start;
5866 + //TRACE("td_replenish(%u): next_replenishment=%llu\n", res->id, res->next_replenishment);
5867 +
5868 +
5869 + switch (res->state) {
5870 + case RESERVATION_DEPLETED:
5871 + case RESERVATION_ACTIVE:
5872 + case RESERVATION_ACTIVE_IDLE:
5873 + if (list_empty(&res->clients))
5874 + res->env->change_state(res->env, res,
5875 + RESERVATION_ACTIVE_IDLE);
5876 + else
5877 + /* we have clients & budget => ACTIVE */
5878 + res->env->change_state(res->env, res,
5879 + RESERVATION_ACTIVE);
5880 + break;
5881 +
5882 + case RESERVATION_INACTIVE:
5883 + BUG();
5884 + break;
5885 + }
5886 +}
5887 +
5888 +static void td_drain_budget(
5889 + struct reservation *res,
5890 + lt_t how_much)
5891 +{
5892 + struct table_driven_reservation *tdres =
5893 + container_of(res, struct table_driven_reservation, res);
5894 +
5895 + res->budget_consumed += how_much;
5896 + res->budget_consumed_total += how_much;
5897 +
5898 + /* Table-driven scheduling: instead of tracking the budget, we compute
5899 + * how much time is left in this allocation interval. */
5900 +
5901 + /* sanity check: we should never try to drain from future slots */
5902 + //TRACE("TD_DRAIN STATE(%d) [%llu,%llu] %llu ?\n", res->state, tdres->cur_interval.start, tdres->cur_interval.end, res->env->current_time);
5903 + //BUG_ON(tdres->cur_interval.start > res->env->current_time);
5904 + if (tdres->cur_interval.start > res->env->current_time)
5905 + TRACE("TD_DRAIN BUG!!!!!!!!!!\n");
5906 +
5907 + switch (res->state) {
5908 + case RESERVATION_DEPLETED:
5909 + case RESERVATION_INACTIVE:
5910 + //BUG();
5911 + TRACE("TD_DRAIN!!!!!!!!! RES_STATE = %d\n", res->state);
5912 + break;
5913 +
5914 + case RESERVATION_ACTIVE_IDLE:
5915 + case RESERVATION_ACTIVE:
5916 + res->cur_budget = td_time_remaining_until_end(tdres);
5917 + //TRACE("td_drain_budget(%u): drained to budget=%llu\n", res->id, res->cur_budget);
5918 + if (!res->cur_budget) {
5919 + res->env->change_state(res->env, res,
5920 + RESERVATION_DEPLETED);
5921 + } else {
5922 + /* sanity check budget calculation */
5923 + //BUG_ON(res->env->current_time >= tdres->cur_interval.end);
5924 + //BUG_ON(res->env->current_time < tdres->cur_interval.start);
5925 + if (res->env->current_time >= tdres->cur_interval.end)
5926 + printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING1\n");
5927 + if (res->env->current_time < tdres->cur_interval.start)
5928 + printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING2\n");
5929 + }
5930 +
5931 + break;
5932 + }
5933 +}
5934 +
5935 +static struct task_struct* td_dispatch_client(
5936 + struct reservation *res,
5937 + lt_t *for_at_most)
5938 +{
5939 + struct task_struct *t;
5940 + struct table_driven_reservation *tdres =
5941 + container_of(res, struct table_driven_reservation, res);
5942 +
5943 + /* usual logic for selecting a client */
5944 + t = default_dispatch_client(res, for_at_most);
5945 +
5946 + TRACE_TASK(t, "td_dispatch_client(%u): selected, budget=%llu\n",
5947 + res->id, res->cur_budget);
5948 +
5949 + /* check how much budget we have left in this time slot */
5950 + res->cur_budget = td_time_remaining_until_end(tdres);
5951 +
5952 + TRACE_TASK(t, "td_dispatch_client(%u): updated to budget=%llu next=%d\n",
5953 + res->id, res->cur_budget, tdres->next_interval);
5954 +
5955 + if (unlikely(!res->cur_budget)) {
5956 + /* Unlikely case: if we ran out of budget, the user configured
5957 + * a broken scheduling table (overlapping table slots).
5958 + * Not much we can do about this, but we can't dispatch a job
5959 + * now without causing overload. So let's register this reservation
5960 + * as depleted and wait for the next allocation. */
5961 + TRACE("td_dispatch_client(%u): budget unexpectedly depleted "
5962 + "(check scheduling table for unintended overlap)\n",
5963 + res->id);
5964 + res->env->change_state(res->env, res,
5965 + RESERVATION_DEPLETED);
5966 + return NULL;
5967 + } else
5968 + return t;
5969 +}
5970 +
5971 +static struct reservation_ops td_ops = {
5972 + .dispatch_client = td_dispatch_client,
5973 + .client_arrives = td_client_arrives,
5974 + .client_departs = td_client_departs,
5975 + .replenish = td_replenish,
5976 + .drain_budget = td_drain_budget,
5977 +};
5978 +
5979 +void table_driven_reservation_init(
5980 + struct table_driven_reservation *tdres,
5981 + lt_t major_cycle,
5982 + struct lt_interval *intervals,
5983 + unsigned int num_intervals)
5984 +{
5985 + unsigned int i;
5986 +
5987 + /* sanity checking */
5988 + BUG_ON(!num_intervals);
5989 + for (i = 0; i < num_intervals; i++)
5990 + BUG_ON(intervals[i].end <= intervals[i].start);
5991 + for (i = 0; i + 1 < num_intervals; i++)
5992 + BUG_ON(intervals[i + 1].start <= intervals[i].end);
5993 + BUG_ON(intervals[num_intervals - 1].end > major_cycle);
5994 +
5995 + reservation_init(&tdres->res);
5996 + tdres->major_cycle = major_cycle;
5997 + tdres->intervals = intervals;
5998 + tdres->cur_interval.start = 0;
5999 + tdres->cur_interval.end = 0;
6000 + tdres->num_intervals = num_intervals;
6001 + tdres->res.ops = &td_ops;
6002 +}
6003 diff --git litmus/replicate_lib.c litmus/replicate_lib.c
6004 new file mode 100644
6005 index 0000000..cfc5258
6006 --- /dev/null
6007 +++ litmus/replicate_lib.c
6008 @@ -0,0 +1,50 @@
6009 +#include <asm/uaccess.h>
6010 +#include <linux/uaccess.h>
6011 +#include <linux/init.h>
6012 +#include <linux/types.h>
6013 +#include <linux/kernel.h>
6014 +#include <linux/module.h>
6015 +#include <linux/sysctl.h>
6016 +#include <linux/slab.h>
6017 +#include <linux/io.h>
6018 +#include <linux/mutex.h>
6019 +#include <linux/time.h>
6020 +#include <linux/migrate.h>
6021 +#include <linux/mm.h>
6022 +#include <linux/memcontrol.h>
6023 +#include <linux/mm_inline.h>
6024 +
6025 +#include <litmus/litmus_proc.h>
6026 +#include <litmus/sched_trace.h>
6027 +#include <litmus/cache_proc.h>
6028 +#include <litmus/mc2_common.h>
6029 +#include <litmus/replicate_lib.h>
6030 +
6031 +DEFINE_PER_CPU(struct list_head, shared_lib_page_list);
6032 +
6033 +#define shared_lib_pages_for(cpu_id) (&per_cpu(shared_lib_page_list, cpu_id))
6034 +#define local_shared_lib_pages() (this_cpu_ptr(&shared_lib_page_list))
6035 +
6036 +#define INVALID_PFN (0xffffffff)
6037 +
6038 +static int __init litmus_replicate_lib_init(void)
6039 +{
6040 + int cpu, ret = 0;
6041 +
6042 + printk(KERN_INFO "Registering LITMUS^RT Per-core Shared Library module.\n");
6043 +
6044 + for_each_online_cpu(cpu) {
6045 + INIT_LIST_HEAD(shared_lib_pages_for(cpu));
6046 + printk(KERN_INFO "CPU%d PSL-list initialized.\n", cpu);
6047 + }
6048 +
6049 + return ret;
6050 +}
6051 +
6052 +static void litmus_replicate_lib_exit(void)
6053 +{
6054 + return;
6055 +}
6056 +
6057 +module_init(litmus_replicate_lib_init);
6058 +module_exit(litmus_replicate_lib_exit);
6059 \ No newline at end of file
6060 diff --git litmus/reservation.c litmus/reservation.c
6061 new file mode 100644
6062 index 0000000..5eee01a
6063 --- /dev/null
6064 +++ litmus/reservation.c
6065 @@ -0,0 +1,706 @@
6066 +#include <linux/sched.h>
6067 +#include <linux/slab.h>
6068 +
6069 +#include <litmus/litmus.h>
6070 +#include <litmus/reservation.h>
6071 +
6072 +#define BUDGET_ENFORCEMENT_AT_C 0
6073 +
6074 +void reservation_init(struct reservation *res)
6075 +{
6076 + memset(res, sizeof(*res), 0);
6077 + res->state = RESERVATION_INACTIVE;
6078 + INIT_LIST_HEAD(&res->clients);
6079 +}
6080 +
6081 +struct task_struct* default_dispatch_client(
6082 + struct reservation *res,
6083 + lt_t *for_at_most)
6084 +{
6085 + struct reservation_client *client, *next;
6086 + struct task_struct* tsk;
6087 +
6088 + BUG_ON(res->state != RESERVATION_ACTIVE);
6089 + *for_at_most = 0;
6090 +
6091 + list_for_each_entry_safe(client, next, &res->clients, list) {
6092 + tsk = client->dispatch(client);
6093 + if (likely(tsk)) {
6094 + return tsk;
6095 + }
6096 + }
6097 + return NULL;
6098 +}
6099 +
6100 +static struct task_struct * task_client_dispatch(struct reservation_client *client)
6101 +{
6102 + struct task_client *tc = container_of(client, struct task_client, client);
6103 + return tc->task;
6104 +}
6105 +
6106 +void task_client_init(struct task_client *tc, struct task_struct *tsk,
6107 + struct reservation *res)
6108 +{
6109 + memset(&tc->client, sizeof(tc->client), 0);
6110 + tc->client.dispatch = task_client_dispatch;
6111 + tc->client.reservation = res;
6112 + tc->task = tsk;
6113 +}
6114 +
6115 +static void sup_scheduler_update_at(
6116 + struct sup_reservation_environment* sup_env,
6117 + lt_t when)
6118 +{
6119 + //TRACE("SCHEDULER_UPDATE_AT update: %llu > when %llu\n", sup_env->next_scheduler_update, when);
6120 + if (sup_env->next_scheduler_update > when)
6121 + sup_env->next_scheduler_update = when;
6122 +}
6123 +
6124 +void sup_scheduler_update_after(
6125 + struct sup_reservation_environment* sup_env,
6126 + lt_t timeout)
6127 +{
6128 + sup_scheduler_update_at(sup_env, sup_env->env.current_time + timeout);
6129 +}
6130 +
6131 +static int _sup_queue_depleted(
6132 + struct sup_reservation_environment* sup_env,
6133 + struct reservation *res)
6134 +{
6135 + struct list_head *pos;
6136 + struct reservation *queued;
6137 + int passed_earlier = 0;
6138 +
6139 + list_for_each(pos, &sup_env->depleted_reservations) {
6140 + queued = list_entry(pos, struct reservation, list);
6141 + if (queued->next_replenishment > res->next_replenishment) {
6142 + list_add(&res->list, pos->prev);
6143 + return passed_earlier;
6144 + } else
6145 + passed_earlier = 1;
6146 + }
6147 +
6148 + list_add_tail(&res->list, &sup_env->depleted_reservations);
6149 +
6150 + return passed_earlier;
6151 +}
6152 +
6153 +static void sup_queue_depleted(
6154 + struct sup_reservation_environment* sup_env,
6155 + struct reservation *res)
6156 +{
6157 + int passed_earlier = _sup_queue_depleted(sup_env, res);
6158 +
6159 + /* check for updated replenishment time */
6160 + if (!passed_earlier)
6161 + sup_scheduler_update_at(sup_env, res->next_replenishment);
6162 +}
6163 +
6164 +static int _sup_queue_active(
6165 + struct sup_reservation_environment* sup_env,
6166 + struct reservation *res)
6167 +{
6168 + struct list_head *pos;
6169 + struct reservation *queued;
6170 + int passed_active = 0;
6171 +
6172 + list_for_each(pos, &sup_env->active_reservations) {
6173 + queued = list_entry(pos, struct reservation, list);
6174 + if (queued->priority > res->priority) {
6175 + list_add(&res->list, pos->prev);
6176 + return passed_active;
6177 + } else if (queued->state == RESERVATION_ACTIVE)
6178 + passed_active = 1;
6179 + }
6180 +
6181 + list_add_tail(&res->list, &sup_env->active_reservations);
6182 + return passed_active;
6183 +}
6184 +
6185 +static void sup_queue_active(
6186 + struct sup_reservation_environment* sup_env,
6187 + struct reservation *res)
6188 +{
6189 + int passed_active = _sup_queue_active(sup_env, res);
6190 +
6191 + /* check for possible preemption */
6192 + if (res->state == RESERVATION_ACTIVE && !passed_active)
6193 + sup_env->next_scheduler_update = SUP_RESCHEDULE_NOW;
6194 + else {
6195 + /* Active means this reservation is draining budget => make sure
6196 + * the scheduler is called to notice when the reservation budget has been
6197 + * drained completely. */
6198 + sup_scheduler_update_after(sup_env, res->cur_budget);
6199 + }
6200 +}
6201 +
6202 +static void sup_queue_reservation(
6203 + struct sup_reservation_environment* sup_env,
6204 + struct reservation *res)
6205 +{
6206 + switch (res->state) {
6207 + case RESERVATION_INACTIVE:
6208 + list_add(&res->list, &sup_env->inactive_reservations);
6209 + break;
6210 +
6211 + case RESERVATION_DEPLETED:
6212 + sup_queue_depleted(sup_env, res);
6213 + break;
6214 +
6215 + case RESERVATION_ACTIVE_IDLE:
6216 + case RESERVATION_ACTIVE:
6217 + sup_queue_active(sup_env, res);
6218 + break;
6219 + }
6220 +}
6221 +
6222 +void sup_add_new_reservation(
6223 + struct sup_reservation_environment* sup_env,
6224 + struct reservation* new_res)
6225 +{
6226 + new_res->env = &sup_env->env;
6227 + sup_queue_reservation(sup_env, new_res);
6228 +}
6229 +
6230 +struct reservation* sup_find_by_id(struct sup_reservation_environment* sup_env,
6231 + unsigned int id)
6232 +{
6233 + struct reservation *res;
6234 +
6235 + list_for_each_entry(res, &sup_env->active_reservations, list) {
6236 + if (res->id == id)
6237 + return res;
6238 + }
6239 + list_for_each_entry(res, &sup_env->inactive_reservations, list) {
6240 + if (res->id == id)
6241 + return res;
6242 + }
6243 + list_for_each_entry(res, &sup_env->depleted_reservations, list) {
6244 + if (res->id == id)
6245 + return res;
6246 + }
6247 +
6248 + return NULL;
6249 +}
6250 +
6251 +static void sup_charge_budget(
6252 + struct sup_reservation_environment* sup_env,
6253 + lt_t delta)
6254 +{
6255 + struct list_head *pos, *next;
6256 + struct reservation *res;
6257 +
6258 + int encountered_active = 0;
6259 +
6260 + list_for_each_safe(pos, next, &sup_env->active_reservations) {
6261 + /* charge all ACTIVE_IDLE up to the first ACTIVE reservation */
6262 + res = list_entry(pos, struct reservation, list);
6263 + if (res->state == RESERVATION_ACTIVE) {
6264 + TRACE("sup_charge_budget ACTIVE R%u drain %llu\n", res->id, delta);
6265 + if (encountered_active == 0 && res->blocked_by_ghost == 0) {
6266 + TRACE("DRAIN !!\n");
6267 + res->ops->drain_budget(res, delta);
6268 + encountered_active = 1;
6269 + }
6270 + } else {
6271 + //BUG_ON(res->state != RESERVATION_ACTIVE_IDLE);
6272 + TRACE("sup_charge_budget INACTIVE R%u drain %llu\n", res->id, delta);
6273 + res->ops->drain_budget(res, delta);
6274 + }
6275 + if (res->state == RESERVATION_ACTIVE ||
6276 + res->state == RESERVATION_ACTIVE_IDLE)
6277 + {
6278 + /* make sure scheduler is invoked when this reservation expires
6279 + * its remaining budget */
6280 + TRACE("requesting scheduler update for reservation %u in %llu nanoseconds\n",
6281 + res->id, res->cur_budget);
6282 + sup_scheduler_update_after(sup_env, res->cur_budget);
6283 + }
6284 + //if (encountered_active == 2)
6285 + /* stop at the first ACTIVE reservation */
6286 + // break;
6287 + }
6288 + //TRACE("finished charging budgets\n");
6289 +}
6290 +
6291 +static void sup_replenish_budgets(struct sup_reservation_environment* sup_env)
6292 +{
6293 + struct list_head *pos, *next;
6294 + struct reservation *res;
6295 +
6296 + list_for_each_safe(pos, next, &sup_env->depleted_reservations) {
6297 + res = list_entry(pos, struct reservation, list);
6298 + if (res->next_replenishment <= sup_env->env.current_time) {
6299 + res->ops->replenish(res);
6300 + } else {
6301 + /* list is ordered by increasing depletion times */
6302 + break;
6303 + }
6304 + }
6305 + //TRACE("finished replenishing budgets\n");
6306 +
6307 + /* request a scheduler update at the next replenishment instant */
6308 + res = list_first_entry_or_null(&sup_env->depleted_reservations,
6309 + struct reservation, list);
6310 + if (res)
6311 + sup_scheduler_update_at(sup_env, res->next_replenishment);
6312 +}
6313 +
6314 +void sup_update_time(
6315 + struct sup_reservation_environment* sup_env,
6316 + lt_t now)
6317 +{
6318 + lt_t delta;
6319 +
6320 + /* If the time didn't advance, there is nothing to do.
6321 + * This check makes it safe to call sup_advance_time() potentially
6322 + * multiple times (e.g., via different code paths. */
6323 + //TRACE("(sup_update_time) now: %llu, current_time: %llu\n", now, sup_env->env.current_time);
6324 + if (unlikely(now <= sup_env->env.current_time))
6325 + return;
6326 +
6327 + delta = now - sup_env->env.current_time;
6328 + sup_env->env.current_time = now;
6329 +
6330 + /* check if future updates are required */
6331 + if (sup_env->next_scheduler_update <= sup_env->env.current_time)
6332 + sup_env->next_scheduler_update = SUP_NO_SCHEDULER_UPDATE;
6333 +
6334 + /* deplete budgets by passage of time */
6335 + //TRACE("CHARGE###\n");
6336 + sup_charge_budget(sup_env, delta);
6337 +
6338 + /* check if any budgets where replenished */
6339 + //TRACE("REPLENISH###\n");
6340 + sup_replenish_budgets(sup_env);
6341 +}
6342 +
6343 +struct task_struct* sup_dispatch(struct sup_reservation_environment* sup_env)
6344 +{
6345 + struct reservation *res, *next;
6346 + struct task_struct *tsk = NULL;
6347 + lt_t time_slice;
6348 +
6349 + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) {
6350 + if (res->state == RESERVATION_ACTIVE) {
6351 + tsk = res->ops->dispatch_client(res, &time_slice);
6352 + if (likely(tsk)) {
6353 + if (time_slice)
6354 + sup_scheduler_update_after(sup_env, time_slice);
6355 + sup_scheduler_update_after(sup_env, res->cur_budget);
6356 + return tsk;
6357 + }
6358 + }
6359 + }
6360 +
6361 + return NULL;
6362 +}
6363 +
6364 +static void sup_res_change_state(
6365 + struct reservation_environment* env,
6366 + struct reservation *res,
6367 + reservation_state_t new_state)
6368 +{
6369 + struct sup_reservation_environment* sup_env;
6370 +
6371 + sup_env = container_of(env, struct sup_reservation_environment, env);
6372 +
6373 + TRACE("reservation R%d state %d->%d at %llu\n",
6374 + res->id, res->state, new_state, env->current_time);
6375 +
6376 + list_del(&res->list);
6377 + /* check if we need to reschedule because we lost an active reservation */
6378 + if (res->state == RESERVATION_ACTIVE && !sup_env->will_schedule)
6379 + sup_env->next_scheduler_update = SUP_RESCHEDULE_NOW;
6380 + res->state = new_state;
6381 + sup_queue_reservation(sup_env, res);
6382 +}
6383 +
6384 +void sup_init(struct sup_reservation_environment* sup_env)
6385 +{
6386 + memset(sup_env, sizeof(*sup_env), 0);
6387 +
6388 + INIT_LIST_HEAD(&sup_env->active_reservations);
6389 + INIT_LIST_HEAD(&sup_env->depleted_reservations);
6390 + INIT_LIST_HEAD(&sup_env->inactive_reservations);
6391 +
6392 + sup_env->env.change_state = sup_res_change_state;
6393 +
6394 + sup_env->next_scheduler_update = SUP_NO_SCHEDULER_UPDATE;
6395 +}
6396 +
6397 +struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env,
6398 + unsigned int id)
6399 +{
6400 + struct reservation *res;
6401 +
6402 + list_for_each_entry(res, &gmp_env->active_reservations, list) {
6403 + if (res->id == id)
6404 + return res;
6405 + }
6406 + list_for_each_entry(res, &gmp_env->inactive_reservations, list) {
6407 + if (res->id == id)
6408 + return res;
6409 + }
6410 + list_for_each_entry(res, &gmp_env->depleted_reservations, list) {
6411 + if (res->id == id)
6412 + return res;
6413 + }
6414 +
6415 + return NULL;
6416 +}
6417 +
6418 +
6419 +struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env,
6420 + unsigned int id)
6421 +{
6422 + struct next_timer_event *event;
6423 +
6424 + list_for_each_entry(event, &gmp_env->next_events, list) {
6425 + if (event->id == id)
6426 + return event;
6427 + }
6428 +
6429 + return NULL;
6430 +}
6431 +
6432 +
6433 +struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env,
6434 + lt_t when)
6435 +{
6436 + struct next_timer_event *event;
6437 +
6438 + list_for_each_entry(event, &gmp_env->next_events, list) {
6439 + if (event->next_update == when)
6440 + return event;
6441 + }
6442 +
6443 + return NULL;
6444 +}
6445 +
6446 +#define TIMER_RESOLUTION 100000L
6447 +
6448 +static void gmp_add_event(
6449 + struct gmp_reservation_environment* gmp_env,
6450 + lt_t when, unsigned int id, event_type_t type)
6451 +{
6452 + struct next_timer_event *nevent, *queued;
6453 + struct list_head *pos;
6454 + int found = 0, update = 0;
6455 +
6456 + //when = div64_u64(when, TIMER_RESOLUTION);
6457 + //when *= TIMER_RESOLUTION;
6458 +//printk(KERN_ALERT "GMP_ADD id=%d type=%d when=%llu\n", id, type, when);
6459 + nevent = gmp_find_event_by_id(gmp_env, id);
6460 +
6461 + if (nevent)
6462 + TRACE("EVENT R%d update prev = %llu, new = %llu\n", nevent->id, nevent->next_update, when);
6463 +
6464 + if (nevent && nevent->next_update > when) {
6465 + list_del(&nevent->list);
6466 + update = 1;
6467 +
6468 + }
6469 +
6470 + if (!nevent || nevent->type != type || update == 1) {
6471 + if (update == 0)
6472 + nevent = kzalloc(sizeof(*nevent), GFP_ATOMIC);
6473 + BUG_ON(!nevent);
6474 + nevent->next_update = when;
6475 + nevent->id = id;
6476 + nevent->type = type;
6477 + nevent->timer_armed_on = NO_CPU;
6478 +
6479 + list_for_each(pos, &gmp_env->next_events) {
6480 + queued = list_entry(pos, struct next_timer_event, list);
6481 + if (queued->next_update > nevent->next_update) {
6482 + list_add(&nevent->list, pos->prev);
6483 + found = 1;
6484 + TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at before %llu\n", nevent->id, nevent->type, nevent->next_update, queued->next_update);
6485 + break;
6486 + }
6487 + }
6488 +
6489 + if (!found) {
6490 + list_add_tail(&nevent->list, &gmp_env->next_events);
6491 + TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at TAIL\n", nevent->id, nevent->type, nevent->next_update);
6492 + }
6493 + } else {
6494 + //TRACE("EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when);
6495 +; //printk(KERN_ALERT "EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when);
6496 + }
6497 +
6498 + TRACE("======START PRINTING EVENT LIST======\n");
6499 + gmp_print_events(gmp_env, litmus_clock());
6500 + TRACE("======FINISH PRINTING EVENT LIST======\n");
6501 +}
6502 +
6503 +void gmp_add_event_after(
6504 + struct gmp_reservation_environment* gmp_env, lt_t timeout, unsigned int id, event_type_t type)
6505 +{
6506 + //printk(KERN_ALERT "ADD_EVENT_AFTER id = %d\n", id);
6507 + gmp_add_event(gmp_env, gmp_env->env.current_time + timeout, id, type);
6508 +}
6509 +
6510 +static void gmp_queue_depleted(
6511 + struct gmp_reservation_environment* gmp_env,
6512 + struct reservation *res)
6513 +{
6514 + struct list_head *pos;
6515 + struct reservation *queued;
6516 + int found = 0;
6517 +
6518 +//printk(KERN_ALERT "R%d request to enqueue depleted_list\n", res->id);
6519 +
6520 + list_for_each(pos, &gmp_env->depleted_reservations) {
6521 + queued = list_entry(pos, struct reservation, list);
6522 + if (queued && (queued->next_replenishment > res->next_replenishment)) {
6523 +//printk(KERN_ALERT "QUEUED R%d %llu\n", queued->id, queued->next_replenishment);
6524 + list_add(&res->list, pos->prev);
6525 + found = 1;
6526 + break;
6527 + }
6528 + }
6529 +
6530 + if (!found)
6531 + list_add_tail(&res->list, &gmp_env->depleted_reservations);
6532 +
6533 + TRACE("R%d queued to depleted_list\n", res->id);
6534 +//printk(KERN_ALERT "R%d queued to depleted_list\n", res->id);
6535 + gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
6536 +}
6537 +
6538 +static void gmp_queue_active(
6539 + struct gmp_reservation_environment* gmp_env,
6540 + struct reservation *res)
6541 +{
6542 + struct list_head *pos;
6543 + struct reservation *queued;
6544 + int check_preempt = 1, found = 0;
6545 +
6546 + list_for_each(pos, &gmp_env->active_reservations) {
6547 + queued = list_entry(pos, struct reservation, list);
6548 + if (queued->priority > res->priority) {
6549 + list_add(&res->list, pos->prev);
6550 + found = 1;
6551 + break;
6552 + } else if (queued->scheduled_on == NO_CPU)
6553 + check_preempt = 0;
6554 + }
6555 +
6556 + if (!found)
6557 + list_add_tail(&res->list, &gmp_env->active_reservations);
6558 +
6559 + /* check for possible preemption */
6560 + if (res->state == RESERVATION_ACTIVE && check_preempt)
6561 + gmp_env->schedule_now++;
6562 +
6563 +#if BUDGET_ENFORCEMENT_AT_C
6564 + gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN);
6565 +#endif
6566 + res->event_added = 1;
6567 +}
6568 +
6569 +static void gmp_queue_reservation(
6570 + struct gmp_reservation_environment* gmp_env,
6571 + struct reservation *res)
6572 +{
6573 +
6574 +//printk(KERN_ALERT "DEBUG: Passed %s %d %p R%d STATE %d\n",__FUNCTION__,__LINE__, gmp_env, res->id, res->state);
6575 + switch (res->state) {
6576 + case RESERVATION_INACTIVE:
6577 + list_add(&res->list, &gmp_env->inactive_reservations);
6578 + break;
6579 +
6580 + case RESERVATION_DEPLETED:
6581 + gmp_queue_depleted(gmp_env, res);
6582 + break;
6583 +
6584 + case RESERVATION_ACTIVE_IDLE:
6585 + case RESERVATION_ACTIVE:
6586 + gmp_queue_active(gmp_env, res);
6587 + break;
6588 + }
6589 +}
6590 +
6591 +void gmp_add_new_reservation(
6592 + struct gmp_reservation_environment* gmp_env,
6593 + struct reservation* new_res)
6594 +{
6595 + new_res->env = &gmp_env->env;
6596 + gmp_queue_reservation(gmp_env, new_res);
6597 +}
6598 +
6599 +#if BUDGET_ENFORCEMENT_AT_C
6600 +static void gmp_charge_budget(
6601 + struct gmp_reservation_environment* gmp_env,
6602 + lt_t delta)
6603 +{
6604 + struct list_head *pos, *next;
6605 + struct reservation *res;
6606 +
6607 + list_for_each_safe(pos, next, &gmp_env->active_reservations) {
6608 + int drained = 0;
6609 + /* charge all ACTIVE_IDLE up to the first ACTIVE reservation */
6610 + res = list_entry(pos, struct reservation, list);
6611 + if (res->state == RESERVATION_ACTIVE) {
6612 + TRACE("gmp_charge_budget ACTIVE R%u scheduled_on=%d drain %llu\n", res->id, res->scheduled_on, delta);
6613 + if (res->scheduled_on != NO_CPU && res->blocked_by_ghost == 0) {
6614 + TRACE("DRAIN !!\n");
6615 + drained = 1;
6616 + res->ops->drain_budget(res, delta);
6617 + } else {
6618 + TRACE("NO DRAIN (not scheduled)!!\n");
6619 + }
6620 + } else {
6621 + //BUG_ON(res->state != RESERVATION_ACTIVE_IDLE);
6622 + if (res->state != RESERVATION_ACTIVE_IDLE)
6623 + TRACE("BUG!!!!!!!!!!!! gmp_charge_budget()\n");
6624 + TRACE("gmp_charge_budget INACTIVE R%u drain %llu\n", res->id, delta);
6625 + //if (res->is_ghost != NO_CPU) {
6626 + TRACE("DRAIN !!\n");
6627 + drained = 1;
6628 + res->ops->drain_budget(res, delta);
6629 + //}
6630 + }
6631 + if ((res->state == RESERVATION_ACTIVE ||
6632 + res->state == RESERVATION_ACTIVE_IDLE) && (drained == 1))
6633 + {
6634 + /* make sure scheduler is invoked when this reservation expires
6635 + * its remaining budget */
6636 + TRACE("requesting gmp_scheduler update for reservation %u in %llu nanoseconds\n", res->id, res->cur_budget);
6637 + gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN);
6638 + res->event_added = 1;
6639 + }
6640 + //if (encountered_active == 2)
6641 + /* stop at the first ACTIVE reservation */
6642 + // break;
6643 + }
6644 + //TRACE("finished charging budgets\n");
6645 +}
6646 +#else
6647 +
6648 +static void gmp_charge_budget(
6649 + struct gmp_reservation_environment* gmp_env,
6650 + lt_t delta)
6651 +{
6652 + return;
6653 +}
6654 +
6655 +#endif
6656 +
6657 +static void gmp_replenish_budgets(struct gmp_reservation_environment* gmp_env)
6658 +{
6659 + struct list_head *pos, *next;
6660 + struct reservation *res;
6661 +
6662 + list_for_each_safe(pos, next, &gmp_env->depleted_reservations) {
6663 + res = list_entry(pos, struct reservation, list);
6664 + if (res->next_replenishment <= gmp_env->env.current_time) {
6665 + res->ops->replenish(res);
6666 + if (res->is_ghost != NO_CPU) {
6667 + TRACE("R%d replenished! scheduled_on=%d\n", res->id, res->scheduled_on);
6668 + }
6669 + } else {
6670 + /* list is ordered by increasing depletion times */
6671 + break;
6672 + }
6673 + }
6674 + //TRACE("finished replenishing budgets\n");
6675 +}
6676 +
6677 +#define EPSILON 50
6678 +
6679 +/* return schedule_now */
6680 +int gmp_update_time(
6681 + struct gmp_reservation_environment* gmp_env,
6682 + lt_t now)
6683 +{
6684 + struct next_timer_event *event, *next;
6685 + lt_t delta, ret;
6686 +
6687 + /* If the time didn't advance, there is nothing to do.
6688 + * This check makes it safe to call sup_advance_time() potentially
6689 + * multiple times (e.g., via different code paths. */
6690 + //TRACE("(gmp_update_time) now: %llu, current_time: %llu\n", now, gmp_env->env.current_time);
6691 + if (unlikely(now <= gmp_env->env.current_time + EPSILON))
6692 + return 0;
6693 +
6694 + delta = now - gmp_env->env.current_time;
6695 + gmp_env->env.current_time = now;
6696 +
6697 +
6698 + //gmp_print_events(gmp_env, now);
6699 + /* deplete budgets by passage of time */
6700 + //TRACE("CHARGE###\n");
6701 + gmp_charge_budget(gmp_env, delta);
6702 +
6703 + /* check if any budgets where replenished */
6704 + //TRACE("REPLENISH###\n");
6705 + gmp_replenish_budgets(gmp_env);
6706 +
6707 +
6708 + list_for_each_entry_safe(event, next, &gmp_env->next_events, list) {
6709 + if (event->next_update < now) {
6710 + list_del(&event->list);
6711 + //TRACE("EVENT at %llu IS DELETED\n", event->next_update);
6712 + kfree(event);
6713 + } else {
6714 + break;
6715 + }
6716 + }
6717 +
6718 + //gmp_print_events(gmp_env, litmus_clock());
6719 +
6720 + ret = min(gmp_env->schedule_now, NR_CPUS);
6721 + gmp_env->schedule_now = 0;
6722 +
6723 + return ret;
6724 +}
6725 +
6726 +void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now)
6727 +{
6728 + struct next_timer_event *event, *next;
6729 +
6730 + TRACE("GLOBAL EVENTS now=%llu\n", now);
6731 + list_for_each_entry_safe(event, next, &gmp_env->next_events, list) {
6732 + TRACE("at %llu type=%d id=%d armed_on=%d\n", event->next_update, event->type, event->id, event->timer_armed_on);
6733 + }
6734 +}
6735 +
6736 +static void gmp_res_change_state(
6737 + struct reservation_environment* env,
6738 + struct reservation *res,
6739 + reservation_state_t new_state)
6740 +{
6741 + struct gmp_reservation_environment* gmp_env;
6742 +
6743 + gmp_env = container_of(env, struct gmp_reservation_environment, env);
6744 +
6745 + TRACE("GMP reservation R%d state %d->%d at %llu\n",
6746 + res->id, res->state, new_state, env->current_time);
6747 +
6748 + list_del(&res->list);
6749 + /* check if we need to reschedule because we lost an active reservation */
6750 + if (res->state == RESERVATION_ACTIVE)
6751 + gmp_env->schedule_now++;
6752 + res->state = new_state;
6753 + gmp_queue_reservation(gmp_env, res);
6754 +}
6755 +
6756 +void gmp_init(struct gmp_reservation_environment* gmp_env)
6757 +{
6758 + memset(gmp_env, sizeof(*gmp_env), 0);
6759 +
6760 + INIT_LIST_HEAD(&gmp_env->active_reservations);
6761 + INIT_LIST_HEAD(&gmp_env->depleted_reservations);
6762 + INIT_LIST_HEAD(&gmp_env->inactive_reservations);
6763 + INIT_LIST_HEAD(&gmp_env->next_events);
6764 +
6765 + gmp_env->env.change_state = gmp_res_change_state;
6766 +
6767 + gmp_env->schedule_now = 0;
6768 + gmp_env->will_schedule = false;
6769 +
6770 + raw_spin_lock_init(&gmp_env->lock);
6771 +}
6772 diff --git litmus/sched_mc2.c litmus/sched_mc2.c
6773 new file mode 100644
6774 index 0000000..dba69df
6775 --- /dev/null
6776 +++ litmus/sched_mc2.c
6777 @@ -0,0 +1,1592 @@
6778 +/*
6779 + * litmus/sched_mc2.c
6780 + *
6781 + * Implementation of the Mixed-Criticality on MultiCore scheduler
6782 + *
6783 + * This plugin implements a scheduling algorithm proposed in
6784 + * "Mixed-Criticality Real-Time Scheduling for Multicore System" paper.
6785 + */
6786 +
6787 +#include <linux/percpu.h>
6788 +#include <linux/slab.h>
6789 +#include <asm/uaccess.h>
6790 +
6791 +#include <litmus/sched_plugin.h>
6792 +#include <litmus/preempt.h>
6793 +#include <litmus/debug_trace.h>
6794 +
6795 +#include <litmus/litmus.h>
6796 +#include <litmus/jobs.h>
6797 +#include <litmus/budget.h>
6798 +#include <litmus/litmus_proc.h>
6799 +#include <litmus/sched_trace.h>
6800 +#include <litmus/cache_proc.h>
6801 +#include <litmus/trace.h>
6802 +
6803 +#include <litmus/mc2_common.h>
6804 +#include <litmus/reservation.h>
6805 +#include <litmus/polling_reservations.h>
6806 +
6807 +#define BUDGET_ENFORCEMENT_AT_C 0
6808 +
6809 +extern void do_partition(enum crit_level lv, int cpu);
6810 +
6811 +/* _global_env - reservation container for level-C tasks*/
6812 +struct gmp_reservation_environment _global_env;
6813 +
6814 +/* cpu_entry - keep track of a running task on a cpu
6815 + * This state is used to decide the lowest priority cpu
6816 + */
6817 +struct cpu_entry {
6818 + struct task_struct *scheduled;
6819 + lt_t deadline;
6820 + int cpu;
6821 + enum crit_level lv;
6822 + /* if will_schedule is true, this cpu is already selected and
6823 + call mc2_schedule() soon. */
6824 + bool will_schedule;
6825 +};
6826 +
6827 +/* cpu_priority - a global state for choosing the lowest priority CPU */
6828 +struct cpu_priority {
6829 + raw_spinlock_t lock;
6830 + struct cpu_entry cpu_entries[NR_CPUS];
6831 +};
6832 +
6833 +struct cpu_priority _lowest_prio_cpu;
6834 +
6835 +/* mc2_task_state - a task state structure */
6836 +struct mc2_task_state {
6837 + struct task_client res_info;
6838 + /* if cpu == -1, this task is a global task (level C) */
6839 + int cpu;
6840 + bool has_departed;
6841 + struct mc2_task mc2_param;
6842 +};
6843 +
6844 +/* crit_entry - maintain the logically running job (ghost job) */
6845 +struct crit_entry {
6846 + enum crit_level level;
6847 + struct task_struct *running;
6848 +};
6849 +
6850 +/* mc2_cpu_state - maintain the scheduled state and ghost jobs
6851 + * timer : timer for partitioned tasks (level A and B)
6852 + * g_timer : timer for global tasks (level C)
6853 + */
6854 +struct mc2_cpu_state {
6855 + raw_spinlock_t lock;
6856 +
6857 + struct sup_reservation_environment sup_env;
6858 + struct hrtimer timer;
6859 +
6860 + int cpu;
6861 + struct task_struct* scheduled;
6862 + struct crit_entry crit_entries[NUM_CRIT_LEVELS];
6863 +};
6864 +
6865 +static int resched_cpu[NR_CPUS];
6866 +static DEFINE_PER_CPU(struct mc2_cpu_state, mc2_cpu_state);
6867 +static int level_a_priorities[NR_CPUS];
6868 +
6869 +#define cpu_state_for(cpu_id) (&per_cpu(mc2_cpu_state, cpu_id))
6870 +#define local_cpu_state() (this_cpu_ptr(&mc2_cpu_state))
6871 +
6872 +/* get_mc2_state - get the task's state */
6873 +static struct mc2_task_state* get_mc2_state(struct task_struct *tsk)
6874 +{
6875 + struct mc2_task_state* tinfo;
6876 +
6877 + tinfo = (struct mc2_task_state*)tsk_rt(tsk)->plugin_state;
6878 +
6879 + if (tinfo)
6880 + return tinfo;
6881 + else
6882 + return NULL;
6883 +}
6884 +
6885 +/* get_task_crit_level - return the criticaility level of a task */
6886 +static enum crit_level get_task_crit_level(struct task_struct *tsk)
6887 +{
6888 + struct mc2_task *mp;
6889 +
6890 + if (!tsk || !is_realtime(tsk))
6891 + return NUM_CRIT_LEVELS;
6892 +
6893 + mp = tsk_rt(tsk)->mc2_data;
6894 +
6895 + if (!mp)
6896 + return NUM_CRIT_LEVELS;
6897 + else
6898 + return mp->crit;
6899 +}
6900 +
6901 +/* task_depart - remove a task from its reservation
6902 + * If the job has remaining budget, convert it to a ghost job
6903 + * and update crit_entries[]
6904 + *
6905 + * @job_complete indicate whether job completes or not
6906 + */
6907 +static void task_departs(struct task_struct *tsk, int job_complete)
6908 +{
6909 + struct mc2_task_state* tinfo = get_mc2_state(tsk);
6910 +
6911 + struct reservation* res = NULL;
6912 + struct reservation_client *client = NULL;
6913 +
6914 + BUG_ON(!is_realtime(tsk));
6915 +
6916 + res = tinfo->res_info.client.reservation;
6917 + client = &tinfo->res_info.client;
6918 + BUG_ON(!res);
6919 + BUG_ON(!client);
6920 +
6921 + /* No ghost job handling, empty remaining budget */
6922 + if (job_complete) {
6923 + res->cur_budget = 0;
6924 + }
6925 +
6926 + res->ops->client_departs(res, client, job_complete);
6927 + tinfo->has_departed = true;
6928 + TRACE_TASK(tsk, "Client departs with budget %llu at %llu\n", res->cur_budget, litmus_clock());
6929 +}
6930 +
6931 +/* task_arrive - put a task into its reservation
6932 + * If the job was a ghost job, remove it from crit_entries[]
6933 + */
6934 +static void task_arrives(struct mc2_cpu_state *state, struct task_struct *tsk)
6935 +{
6936 + struct mc2_task_state* tinfo = get_mc2_state(tsk);
6937 + struct reservation* res;
6938 + struct reservation_client *client;
6939 + //enum crit_level lv = get_task_crit_level(tsk);
6940 +
6941 + res = tinfo->res_info.client.reservation;
6942 + client = &tinfo->res_info.client;
6943 +
6944 + tinfo->has_departed = false;
6945 +
6946 + res->ops->client_arrives(res, client);
6947 + TRACE_TASK(tsk, "Client arrives at %llu\n", litmus_clock());
6948 +/*
6949 + if (lv != NUM_CRIT_LEVELS) {
6950 + struct crit_entry *ce;
6951 + ce = &state->crit_entries[lv];
6952 + // if the currrent task is a ghost job, remove it
6953 + if (ce->running == tsk)
6954 + ce->running = NULL;
6955 + }
6956 +*/
6957 +}
6958 +
6959 +/* get_lowest_prio_cpu - return the lowest priority cpu
6960 + * This will be used for scheduling level-C tasks.
6961 + * If all CPUs are running tasks which has
6962 + * higher priority than level C, return NO_CPU.
6963 + */
6964 +static int get_lowest_prio_cpu(lt_t priority)
6965 +{
6966 + struct cpu_entry *ce;
6967 + int cpu, ret = NO_CPU;
6968 + lt_t latest_deadline = 0;
6969 +
6970 + if (priority == LITMUS_NO_PRIORITY)
6971 + return ret;
6972 +
6973 + ce = &_lowest_prio_cpu.cpu_entries[local_cpu_state()->cpu];
6974 + if (!ce->will_schedule && !ce->scheduled) {
6975 + TRACE("CPU %d (local) is the lowest!\n", ce->cpu);
6976 + return ce->cpu;
6977 + } else {
6978 + TRACE("Local CPU will_schedule=%d, scheduled=(%s/%d)\n", ce->will_schedule, ce->scheduled ? (ce->scheduled)->comm : "null", ce->scheduled ? (ce->scheduled)->pid : 0);
6979 + }
6980 +
6981 + for_each_online_cpu(cpu) {
6982 + ce = &_lowest_prio_cpu.cpu_entries[cpu];
6983 + /* If a CPU will call schedule() in the near future, we don't
6984 + return that CPU. */
6985 + /*
6986 + TRACE("CPU %d will_schedule=%d, scheduled=(%s/%d:%d)\n", cpu, ce->will_schedule,
6987 + ce->scheduled ? (ce->scheduled)->comm : "null",
6988 + ce->scheduled ? (ce->scheduled)->pid : 0,
6989 + ce->scheduled ? (ce->scheduled)->rt_param.job_params.job_no : 0);
6990 + */
6991 + if (!ce->will_schedule) {
6992 + if (!ce->scheduled) {
6993 + /* Idle cpu, return this. */
6994 + TRACE("CPU %d is the lowest!\n", ce->cpu);
6995 + return ce->cpu;
6996 + } else if (ce->lv == CRIT_LEVEL_C &&
6997 + ce->deadline > latest_deadline) {
6998 + latest_deadline = ce->deadline;
6999 + ret = ce->cpu;
7000 + }
7001 + }
7002 + }
7003 +
7004 + if (priority >= latest_deadline)
7005 + ret = NO_CPU;
7006 +
7007 + TRACE("CPU %d is the lowest!\n", ret);
7008 +
7009 + return ret;
7010 +}
7011 +
7012 +/* NOTE: drops state->lock */
7013 +/* mc2_update_timer_and_unlock - set a timer and g_timer and unlock
7014 + * Whenever res_env.current_time is updated,
7015 + * we check next_scheduler_update and set
7016 + * a timer.
7017 + * If there exist a global event which is
7018 + * not armed on any CPU and g_timer is not
7019 + * active, set a g_timer for that event.
7020 + */
7021 +static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
7022 +{
7023 + int local, cpus;
7024 + lt_t update, now;
7025 + struct next_timer_event *event, *next;
7026 + int reschedule[NR_CPUS];
7027 + unsigned long flags;
7028 +
7029 + local_irq_save(flags);
7030 +
7031 + for (cpus = 0; cpus<NR_CPUS; cpus++)
7032 + reschedule[cpus] = 0;
7033 +
7034 + update = state->sup_env.next_scheduler_update;
7035 + now = state->sup_env.env.current_time;
7036 +
7037 + /* Be sure we're actually running on the right core,
7038 + * as pres_update_timer() is also called from pres_task_resume(),
7039 + * which might be called on any CPU when a thread resumes.
7040 + */
7041 + local = local_cpu_state() == state;
7042 +
7043 + raw_spin_lock(&_global_env.lock);
7044 +
7045 + list_for_each_entry_safe(event, next, &_global_env.next_events, list) {
7046 + /* If the event time is already passed, we call schedule() on
7047 + the lowest priority cpu */
7048 + if (event->next_update >= update) {
7049 + break;
7050 + }
7051 +
7052 + if (event->next_update < litmus_clock()) {
7053 + if (event->timer_armed_on == NO_CPU) {
7054 + struct reservation *res = gmp_find_by_id(&_global_env, event->id);
7055 + int cpu = get_lowest_prio_cpu(res?res->priority:0);
7056 + //TRACE("GLOBAL EVENT PASSED!! poking CPU %d to reschedule\n", cpu);
7057 + list_del(&event->list);
7058 + kfree(event);
7059 + if (cpu != NO_CPU) {
7060 + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
7061 + reschedule[cpu] = 1;
7062 + }
7063 + }
7064 + } else if (event->next_update < update && (event->timer_armed_on == NO_CPU || event->timer_armed_on == state->cpu)) {
7065 + event->timer_armed_on = state->cpu;
7066 + update = event->next_update;
7067 + break;
7068 + }
7069 + }
7070 +
7071 + /* Must drop state lock before calling into hrtimer_start(), which
7072 + * may raise a softirq, which in turn may wake ksoftirqd. */
7073 + raw_spin_unlock(&_global_env.lock);
7074 + local_irq_restore(flags);
7075 + raw_spin_unlock(&state->lock);
7076 +
7077 + if (update <= now || reschedule[state->cpu]) {
7078 + reschedule[state->cpu] = 0;
7079 + litmus_reschedule(state->cpu);
7080 + } else if (likely(local && update != SUP_NO_SCHEDULER_UPDATE)) {
7081 + /* Reprogram only if not already set correctly. */
7082 + if (!hrtimer_active(&state->timer) ||
7083 + ktime_to_ns(hrtimer_get_expires(&state->timer)) != update) {
7084 + TRACE("canceling timer...at %llu\n",
7085 + ktime_to_ns(hrtimer_get_expires(&state->timer)));
7086 + hrtimer_cancel(&state->timer);
7087 + TRACE("setting scheduler timer for %llu\n", update);
7088 + /* We cannot use hrtimer_start() here because the
7089 + * wakeup flag must be set to zero. */
7090 + __hrtimer_start_range_ns(&state->timer,
7091 + ns_to_ktime(update),
7092 + 0 /* timer coalescing slack */,
7093 + HRTIMER_MODE_ABS_PINNED,
7094 + 0 /* wakeup */);
7095 + if (update < litmus_clock()) {
7096 + /* uh oh, timer expired while trying to set it */
7097 + TRACE("timer expired during setting "
7098 + "update:%llu now:%llu actual:%llu\n",
7099 + update, now, litmus_clock());
7100 + /* The timer HW may not have been reprogrammed
7101 + * correctly; force rescheduling now. */
7102 + litmus_reschedule(state->cpu);
7103 + }
7104 + }
7105 + } else if (unlikely(!local && update != SUP_NO_SCHEDULER_UPDATE)) {
7106 + /* Poke remote core only if timer needs to be set earlier than
7107 + * it is currently set.
7108 + */
7109 + TRACE("mc2_update_timer for remote CPU %d (update=%llu, "
7110 + "active:%d, set:%llu)\n",
7111 + state->cpu, update, hrtimer_active(&state->timer),
7112 + ktime_to_ns(hrtimer_get_expires(&state->timer)));
7113 + if (!hrtimer_active(&state->timer) ||
7114 + ktime_to_ns(hrtimer_get_expires(&state->timer)) > update) {
7115 + TRACE("poking CPU %d so that it can update its "
7116 + "scheduling timer (active:%d, set:%llu)\n",
7117 + state->cpu,
7118 + hrtimer_active(&state->timer),
7119 + ktime_to_ns(hrtimer_get_expires(&state->timer)));
7120 + //raw_spin_lock(&state->lock);
7121 + //preempt_if_preemptable(state->scheduled, state->cpu);
7122 + //raw_spin_unlock(&state->lock);
7123 + //reschedule[state->cpu] = 0;
7124 + }
7125 + }
7126 +
7127 + for (cpus = 0; cpus<NR_CPUS; cpus++) {
7128 + if (reschedule[cpus]) {
7129 + litmus_reschedule(cpus);
7130 + }
7131 + }
7132 +
7133 +}
7134 +
7135 +/* update_cpu_prio - Update cpu's priority
7136 + * When a cpu picks a new task, call this function
7137 + * to update cpu priorities.
7138 + */
7139 +static void update_cpu_prio(struct mc2_cpu_state *state)
7140 +{
7141 + struct cpu_entry *ce = &_lowest_prio_cpu.cpu_entries[state->cpu];
7142 + enum crit_level lv = get_task_crit_level(state->scheduled);
7143 +
7144 + if (!state->scheduled) {
7145 + /* cpu is idle. */
7146 + ce->scheduled = NULL;
7147 + ce->deadline = ULLONG_MAX;
7148 + ce->lv = NUM_CRIT_LEVELS;
7149 + } else if (lv == CRIT_LEVEL_C) {
7150 + ce->scheduled = state->scheduled;
7151 + ce->deadline = get_deadline(state->scheduled);
7152 + ce->lv = lv;
7153 + } else if (lv < CRIT_LEVEL_C) {
7154 + /* If cpu is running level A or B tasks, it is not eligible
7155 + to run level-C tasks */
7156 + ce->scheduled = state->scheduled;
7157 + ce->deadline = 0;
7158 + ce->lv = lv;
7159 + }
7160 +};
7161 +
7162 +/* on_scheduling_timer - timer event for partitioned tasks
7163 + */
7164 +static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
7165 +{
7166 + unsigned long flags;
7167 + enum hrtimer_restart restart = HRTIMER_NORESTART;
7168 + struct mc2_cpu_state *state;
7169 + lt_t update, now;
7170 + int global_schedule_now;
7171 + int reschedule[NR_CPUS];
7172 + int cpus;
7173 +
7174 + for (cpus = 0; cpus<NR_CPUS; cpus++)
7175 + reschedule[cpus] = 0;
7176 +
7177 + state = container_of(timer, struct mc2_cpu_state, timer);
7178 +
7179 + /* The scheduling timer should only fire on the local CPU, because
7180 + * otherwise deadlocks via timer_cancel() are possible.
7181 + * Note: this does not interfere with dedicated interrupt handling, as
7182 + * even under dedicated interrupt handling scheduling timers for
7183 + * budget enforcement must occur locally on each CPU.
7184 + */
7185 + BUG_ON(state->cpu != raw_smp_processor_id());
7186 +
7187 + TS_ISR_START;
7188 +
7189 + TRACE("Timer fired at %llu\n", litmus_clock());
7190 + raw_spin_lock_irqsave(&state->lock, flags);
7191 + now = litmus_clock();
7192 + sup_update_time(&state->sup_env, now);
7193 +
7194 + update = state->sup_env.next_scheduler_update;
7195 + now = state->sup_env.env.current_time;
7196 +
7197 + if (update <= now) {
7198 + litmus_reschedule_local();
7199 + } else if (update != SUP_NO_SCHEDULER_UPDATE) {
7200 + hrtimer_set_expires(timer, ns_to_ktime(update));
7201 + restart = HRTIMER_RESTART;
7202 + }
7203 +
7204 + raw_spin_lock(&_global_env.lock);
7205 + global_schedule_now = gmp_update_time(&_global_env, now);
7206 +
7207 + BUG_ON(global_schedule_now < 0 || global_schedule_now > 4);
7208 +
7209 + /* Find the lowest cpu, and call reschedule */
7210 + while (global_schedule_now--) {
7211 + int cpu = get_lowest_prio_cpu(0);
7212 + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) {
7213 + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
7214 + if (cpu == state->cpu && update > now)
7215 + ; //litmus_reschedule_local();
7216 + else
7217 + reschedule[cpu] = 1;
7218 + }
7219 + }
7220 + raw_spin_unlock(&_global_env.lock);
7221 + raw_spin_unlock_irqrestore(&state->lock, flags);
7222 +
7223 + TS_ISR_END;
7224 +
7225 + for (cpus = 0; cpus<NR_CPUS; cpus++) {
7226 + if (reschedule[cpus]) {
7227 + litmus_reschedule(cpus);
7228 + }
7229 + }
7230 +
7231 + return restart;
7232 +}
7233 +
7234 +/* mc2_complete_job - syscall backend for job completions
7235 + */
7236 +static long mc2_complete_job(void)
7237 +{
7238 + ktime_t next_release;
7239 + long err;
7240 +
7241 + tsk_rt(current)->completed = 1;
7242 +
7243 + /* If this the first job instance, we need to reset replenish
7244 + time to the next release time */
7245 + if (tsk_rt(current)->sporadic_release) {
7246 + struct mc2_cpu_state *state;
7247 + struct reservation_environment *env;
7248 + struct mc2_task_state *tinfo;
7249 + struct reservation *res = NULL;
7250 + unsigned long flags;
7251 + enum crit_level lv;
7252 +
7253 + //preempt_disable();
7254 + local_irq_save(flags);
7255 +
7256 + tinfo = get_mc2_state(current);
7257 + lv = get_task_crit_level(current);
7258 +
7259 + if (lv < CRIT_LEVEL_C) {
7260 + state = cpu_state_for(tinfo->cpu);
7261 + raw_spin_lock(&state->lock);
7262 + env = &(state->sup_env.env);
7263 + res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id);
7264 + env->time_zero = tsk_rt(current)->sporadic_release_time;
7265 + }
7266 + else if (lv == CRIT_LEVEL_C) {
7267 + state = local_cpu_state();
7268 + raw_spin_lock(&state->lock);
7269 + raw_spin_lock(&_global_env.lock);
7270 + res = gmp_find_by_id(&_global_env, tinfo->mc2_param.res_id);
7271 + _global_env.env.time_zero = tsk_rt(current)->sporadic_release_time;
7272 + }
7273 + else
7274 + BUG();
7275 +
7276 + /* set next_replenishtime to synchronous release time */
7277 + BUG_ON(!res);
7278 + res->next_replenishment = tsk_rt(current)->sporadic_release_time;
7279 + res->cur_budget = 0;
7280 + res->env->change_state(res->env, res, RESERVATION_DEPLETED);
7281 +
7282 + if (lv == CRIT_LEVEL_C)
7283 + raw_spin_unlock(&_global_env.lock);
7284 +
7285 + raw_spin_unlock(&state->lock);
7286 + local_irq_restore(flags);
7287 + //preempt_enable();
7288 + }
7289 +
7290 + sched_trace_task_completion(current, 0);
7291 + /* update the next release time and deadline */
7292 + sched_trace_task_release(current);
7293 + prepare_for_next_period(current);
7294 + next_release = ns_to_ktime(get_release(current));
7295 + preempt_disable();
7296 + TRACE_CUR("next_release=%llu\n", get_release(current));
7297 + if (get_release(current) > litmus_clock()) {
7298 + /* sleep until next_release */
7299 + set_current_state(TASK_INTERRUPTIBLE);
7300 + preempt_enable_no_resched();
7301 + err = schedule_hrtimeout(&next_release, HRTIMER_MODE_ABS);
7302 + } else {
7303 + /* release the next job immediately */
7304 + err = 0;
7305 + TRACE_CUR("TARDY: release=%llu now=%llu\n", get_release(current), litmus_clock());
7306 + preempt_enable();
7307 + }
7308 +
7309 + TRACE_CUR("mc2_complete_job returns at %llu\n", litmus_clock());
7310 +
7311 + tsk_rt(current)->completed = 0;
7312 + return err;
7313 +}
7314 +
7315 +/* mc2_dispatch - Select the next task to schedule.
7316 + */
7317 +struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, struct mc2_cpu_state* state)
7318 +{
7319 + struct reservation *res, *next;
7320 + struct task_struct *tsk = NULL;
7321 + struct crit_entry *ce;
7322 + enum crit_level lv;
7323 + lt_t time_slice;
7324 +
7325 + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) {
7326 + if (res->state == RESERVATION_ACTIVE) {
7327 + tsk = res->ops->dispatch_client(res, &time_slice);
7328 + if (likely(tsk)) {
7329 + lv = get_task_crit_level(tsk);
7330 + if (lv == NUM_CRIT_LEVELS) {
7331 + sup_scheduler_update_after(sup_env, res->cur_budget);
7332 + return tsk;
7333 + } else {
7334 + ce = &state->crit_entries[lv];
7335 + sup_scheduler_update_after(sup_env, res->cur_budget);
7336 + res->blocked_by_ghost = 0;
7337 + res->is_ghost = NO_CPU;
7338 + return tsk;
7339 + }
7340 + }
7341 + }
7342 + }
7343 +
7344 + return NULL;
7345 +}
7346 +
7347 +struct task_struct* mc2_global_dispatch(struct mc2_cpu_state* state)
7348 +{
7349 + struct reservation *res, *next;
7350 + struct task_struct *tsk = NULL;
7351 + enum crit_level lv;
7352 + lt_t time_slice;
7353 +
7354 + list_for_each_entry_safe(res, next, &_global_env.active_reservations, list) {
7355 + BUG_ON(!res);
7356 + if (res->state == RESERVATION_ACTIVE && res->scheduled_on == NO_CPU) {
7357 + tsk = res->ops->dispatch_client(res, &time_slice);
7358 + if (likely(tsk)) {
7359 + lv = get_task_crit_level(tsk);
7360 + if (lv != CRIT_LEVEL_C)
7361 + BUG();
7362 +#if BUDGET_ENFORCEMENT_AT_C
7363 + gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
7364 +#endif
7365 + res->event_added = 1;
7366 + res->blocked_by_ghost = 0;
7367 + res->is_ghost = NO_CPU;
7368 + res->scheduled_on = state->cpu;
7369 + return tsk;
7370 + }
7371 + }
7372 + }
7373 +
7374 + return NULL;
7375 +}
7376 +
7377 +static inline void pre_schedule(struct task_struct *prev, int cpu)
7378 +{
7379 + TS_SCHED_A_START;
7380 + TS_SCHED_C_START;
7381 +
7382 + if (!prev || !is_realtime(prev))
7383 + return;
7384 +
7385 + do_partition(CRIT_LEVEL_C, cpu);
7386 +}
7387 +
7388 +static inline void post_schedule(struct task_struct *next, int cpu)
7389 +{
7390 + enum crit_level lev;
7391 + if ((!next) || !is_realtime(next)) {
7392 + //do_partition(NUM_CRIT_LEVELS, -1);
7393 + return;
7394 + }
7395 +
7396 + lev = get_task_crit_level(next);
7397 + do_partition(lev, cpu);
7398 +
7399 + switch(lev) {
7400 + case CRIT_LEVEL_A:
7401 + case CRIT_LEVEL_B:
7402 + TS_SCHED_A_END(next);
7403 + break;
7404 + case CRIT_LEVEL_C:
7405 + TS_SCHED_C_END(next);
7406 + break;
7407 + default:
7408 + break;
7409 + }
7410 +
7411 +}
7412 +
7413 +/* mc2_schedule - main scheduler function. pick the next task to run
7414 + */
7415 +static struct task_struct* mc2_schedule(struct task_struct * prev)
7416 +{
7417 + int np, blocks, exists, cpu; //preempt, to_schedule;
7418 + /* next == NULL means "schedule background work". */
7419 + lt_t now = litmus_clock();
7420 + struct mc2_cpu_state *state = local_cpu_state();
7421 +
7422 + raw_spin_lock(&state->lock);
7423 +
7424 + pre_schedule(prev, state->cpu);
7425 +
7426 + if (state->scheduled && state->scheduled != prev)
7427 + printk(KERN_ALERT "BUG1!!!!!!!! %s %s\n", state->scheduled ? (state->scheduled)->comm : "null", prev ? (prev)->comm : "null");
7428 + if (state->scheduled && !is_realtime(prev))
7429 + printk(KERN_ALERT "BUG2!!!!!!!! \n");
7430 +
7431 + /* (0) Determine state */
7432 + exists = state->scheduled != NULL;
7433 + blocks = exists && !is_current_running();
7434 + np = exists && is_np(state->scheduled);
7435 +
7436 + /* update time */
7437 + state->sup_env.will_schedule = true;
7438 + sup_update_time(&state->sup_env, now);
7439 +
7440 + if (is_realtime(current) && blocks) {
7441 + if (get_task_crit_level(current) == CRIT_LEVEL_C)
7442 + raw_spin_lock(&_global_env.lock);
7443 + task_departs(current, is_completed(current));
7444 + if (get_task_crit_level(current) == CRIT_LEVEL_C)
7445 + raw_spin_unlock(&_global_env.lock);
7446 + }
7447 +
7448 + /* figure out what to schedule next */
7449 + if (!np)
7450 + state->scheduled = mc2_dispatch(&state->sup_env, state);
7451 +
7452 + if (!state->scheduled) {
7453 + raw_spin_lock(&_global_env.lock);
7454 + if (is_realtime(prev))
7455 + gmp_update_time(&_global_env, now);
7456 + state->scheduled = mc2_global_dispatch(state);
7457 + _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false;
7458 + update_cpu_prio(state);
7459 + raw_spin_unlock(&_global_env.lock);
7460 + } else {
7461 + raw_spin_lock(&_global_env.lock);
7462 + _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false;
7463 + update_cpu_prio(state);
7464 + raw_spin_unlock(&_global_env.lock);
7465 + }
7466 +
7467 + /* Notify LITMUS^RT core that we've arrived at a scheduling decision. */
7468 + sched_state_task_picked();
7469 +
7470 + /* program scheduler timer */
7471 + state->sup_env.will_schedule = false;
7472 +
7473 + /* NOTE: drops state->lock */
7474 + mc2_update_timer_and_unlock(state);
7475 +
7476 + raw_spin_lock(&state->lock);
7477 + if (prev != state->scheduled && is_realtime(prev)) {
7478 + struct mc2_task_state* tinfo = get_mc2_state(prev);
7479 + struct reservation* res = tinfo->res_info.client.reservation;
7480 + res->scheduled_on = NO_CPU;
7481 + TRACE_TASK(prev, "descheduled at %llu.\n", litmus_clock());
7482 + /* if prev is preempted and a global task, find the lowest cpu and reschedule */
7483 + if (tinfo->has_departed == false && get_task_crit_level(prev) == CRIT_LEVEL_C) {
7484 + int cpu;
7485 + raw_spin_lock(&_global_env.lock);
7486 + cpu = get_lowest_prio_cpu(res?res->priority:LITMUS_NO_PRIORITY);
7487 + //TRACE("LEVEL-C TASK PREEMPTED!! poking CPU %d to reschedule\n", cpu);
7488 + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) {
7489 + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
7490 + resched_cpu[cpu] = 1;
7491 + }
7492 + raw_spin_unlock(&_global_env.lock);
7493 + }
7494 + }
7495 +
7496 +/*
7497 + if (to_schedule != 0) {
7498 + raw_spin_lock(&_global_env.lock);
7499 + while (to_schedule--) {
7500 + int cpu = get_lowest_prio_cpu(0);
7501 + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) {
7502 + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
7503 + resched_cpu[cpu] = 1;
7504 + }
7505 + }
7506 + raw_spin_unlock(&_global_env.lock);
7507 + }
7508 +*/
7509 +
7510 + post_schedule(state->scheduled, state->cpu);
7511 +
7512 + raw_spin_unlock(&state->lock);
7513 + if (state->scheduled) {
7514 + TRACE_TASK(state->scheduled, "scheduled.\n");
7515 + }
7516 +
7517 + return state->scheduled;
7518 +}
7519 +
7520 +/* mc2_task_resume - Called when the state of tsk changes back to
7521 + * TASK_RUNNING. We need to requeue the task.
7522 + */
7523 +static void mc2_task_resume(struct task_struct *tsk)
7524 +{
7525 + unsigned long flags;
7526 + struct mc2_task_state* tinfo;
7527 + struct mc2_cpu_state *state;
7528 +
7529 + TRACE_TASK(tsk, "thread wakes up at %llu\n", litmus_clock());
7530 +
7531 + preempt_disable();
7532 + tinfo = get_mc2_state(tsk);
7533 + if (tinfo->cpu != -1)
7534 + state = cpu_state_for(tinfo->cpu);
7535 + else
7536 + state = local_cpu_state();
7537 + preempt_enable();
7538 +
7539 + /* Requeue only if self-suspension was already processed. */
7540 + if (tinfo->has_departed)
7541 + {
7542 +#ifdef CONFIG_SCHED_OVERHEAD_TRACE
7543 + switch(get_task_crit_level(tsk)) {
7544 + case CRIT_LEVEL_A:
7545 + TS_RELEASE_LATENCY_A(get_release(tsk));
7546 + break;
7547 + case CRIT_LEVEL_B:
7548 + TS_RELEASE_LATENCY_B(get_release(tsk));
7549 + break;
7550 + case CRIT_LEVEL_C:
7551 + TS_RELEASE_LATENCY_C(get_release(tsk));
7552 + break;
7553 + default:
7554 + break;
7555 + }
7556 +#endif
7557 +
7558 + raw_spin_lock_irqsave(&state->lock, flags);
7559 +
7560 + /* Assumption: litmus_clock() is synchronized across cores,
7561 + * since we might not actually be executing on tinfo->cpu
7562 + * at the moment. */
7563 + if (tinfo->cpu != -1) {
7564 + sup_update_time(&state->sup_env, litmus_clock());
7565 + task_arrives(state, tsk);
7566 + } else {
7567 + raw_spin_lock(&_global_env.lock);
7568 + gmp_update_time(&_global_env, litmus_clock());
7569 + task_arrives(state, tsk);
7570 + raw_spin_unlock(&_global_env.lock);
7571 + }
7572 +
7573 + /* NOTE: drops state->lock */
7574 + TRACE_TASK(tsk, "mc2_resume()\n");
7575 + raw_spin_unlock_irqrestore(&state->lock, flags);
7576 +
7577 + raw_spin_lock(&state->lock);
7578 + mc2_update_timer_and_unlock(state);
7579 + } else {
7580 + TRACE_TASK(tsk, "resume event ignored, still scheduled\n");
7581 + }
7582 +
7583 +}
7584 +
7585 +
7586 +/* mc2_admit_task - Setup mc2 task parameters
7587 + */
7588 +static long mc2_admit_task(struct task_struct *tsk)
7589 +{
7590 + long err = 0;
7591 + unsigned long flags;
7592 + struct reservation *res;
7593 + struct mc2_cpu_state *state;
7594 + struct mc2_task_state *tinfo = kzalloc(sizeof(*tinfo), GFP_ATOMIC);
7595 + struct mc2_task *mp = tsk_rt(tsk)->mc2_data;
7596 + enum crit_level lv;
7597 +
7598 + if (!tinfo)
7599 + return -ENOMEM;
7600 +
7601 + if (!mp) {
7602 + printk(KERN_ERR "mc2_admit_task: criticality level has not been set\n");
7603 + return -ESRCH;
7604 + }
7605 +
7606 + lv = mp->crit;
7607 +
7608 + if (lv < CRIT_LEVEL_C) {
7609 + state = cpu_state_for(task_cpu(tsk));
7610 + raw_spin_lock_irqsave(&state->lock, flags);
7611 +
7612 + res = sup_find_by_id(&state->sup_env, mp->res_id);
7613 +
7614 + /* found the appropriate reservation */
7615 + if (res) {
7616 + TRACE_TASK(tsk, "SUP FOUND RES ID\n");
7617 + tinfo->mc2_param.crit = mp->crit;
7618 + tinfo->mc2_param.res_id = mp->res_id;
7619 +
7620 + /* initial values */
7621 + err = mc2_task_client_init(&tinfo->res_info, &tinfo->mc2_param, tsk, res);
7622 + tinfo->cpu = task_cpu(tsk);
7623 + tinfo->has_departed = true;
7624 + tsk_rt(tsk)->plugin_state = tinfo;
7625 +
7626 + /* disable LITMUS^RT's per-thread budget enforcement */
7627 + tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT;
7628 + }
7629 + else {
7630 + err = -ESRCH;
7631 + }
7632 +
7633 + raw_spin_unlock_irqrestore(&state->lock, flags);
7634 + } else if (lv == CRIT_LEVEL_C) {
7635 + state = local_cpu_state();
7636 + raw_spin_lock_irqsave(&state->lock, flags);
7637 + raw_spin_lock(&_global_env.lock);
7638 +
7639 + res = gmp_find_by_id(&_global_env, mp->res_id);
7640 +
7641 + /* found the appropriate reservation (or vCPU) */
7642 + if (res) {
7643 + TRACE_TASK(tsk, "GMP FOUND RES ID\n");
7644 + tinfo->mc2_param.crit = mp->crit;
7645 + tinfo->mc2_param.res_id = mp->res_id;
7646 +
7647 + /* initial values */
7648 + err = mc2_task_client_init(&tinfo->res_info, &tinfo->mc2_param, tsk, res);
7649 + tinfo->cpu = -1;
7650 + tinfo->has_departed = true;
7651 + tsk_rt(tsk)->plugin_state = tinfo;
7652 +
7653 + /* disable LITMUS^RT's per-thread budget enforcement */
7654 + tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT;
7655 + }
7656 + else {
7657 + err = -ESRCH;
7658 + }
7659 +
7660 + raw_spin_unlock(&_global_env.lock);
7661 + raw_spin_unlock_irqrestore(&state->lock, flags);
7662 + }
7663 +
7664 + if (err)
7665 + kfree(tinfo);
7666 +
7667 + return err;
7668 +}
7669 +
7670 +/* mc2_task_new - A new real-time job is arrived. Release the next job
7671 + * at the next reservation replenish time
7672 + */
7673 +static void mc2_task_new(struct task_struct *tsk, int on_runqueue,
7674 + int is_running)
7675 +{
7676 + unsigned long flags;
7677 + struct mc2_task_state* tinfo = get_mc2_state(tsk);
7678 + struct mc2_cpu_state *state; // = cpu_state_for(tinfo->cpu);
7679 + struct reservation *res;
7680 + enum crit_level lv = get_task_crit_level(tsk);
7681 + lt_t release = 0;
7682 +
7683 + BUG_ON(lv < CRIT_LEVEL_A || lv > CRIT_LEVEL_C);
7684 +
7685 + TRACE_TASK(tsk, "new RT task %llu (on_rq:%d, running:%d)\n",
7686 + litmus_clock(), on_runqueue, is_running);
7687 +
7688 + if (tinfo->cpu == -1)
7689 + state = local_cpu_state();
7690 + else
7691 + state = cpu_state_for(tinfo->cpu);
7692 +
7693 +
7694 + if (is_running) {
7695 + state->scheduled = tsk;
7696 + /* make sure this task should actually be running */
7697 + litmus_reschedule_local();
7698 + }
7699 +
7700 + /* acquire the lock protecting the state and disable interrupts */
7701 + local_irq_save(flags);
7702 + raw_spin_lock(&state->lock);
7703 +
7704 + if (lv == CRIT_LEVEL_C) {
7705 + raw_spin_lock(&_global_env.lock);
7706 + res = gmp_find_by_id(&_global_env, tinfo->mc2_param.res_id);
7707 + }
7708 + else {
7709 + res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id);
7710 + }
7711 +
7712 + if (on_runqueue || is_running) {
7713 + /* Assumption: litmus_clock() is synchronized across cores
7714 + * [see comment in pres_task_resume()] */
7715 + if (lv == CRIT_LEVEL_C) {
7716 + gmp_update_time(&_global_env, litmus_clock());
7717 + }
7718 + else
7719 + sup_update_time(&state->sup_env, litmus_clock());
7720 +
7721 + task_arrives(state, tsk);
7722 + if (lv == CRIT_LEVEL_C)
7723 + raw_spin_unlock(&_global_env.lock);
7724 + /* NOTE: drops state->lock */
7725 + raw_spin_unlock(&state->lock);
7726 + local_irq_restore(flags);
7727 +
7728 + TRACE("mc2_new()\n");
7729 +
7730 + raw_spin_lock(&state->lock);
7731 + mc2_update_timer_and_unlock(state);
7732 + } else {
7733 + if (lv == CRIT_LEVEL_C)
7734 + raw_spin_unlock(&_global_env.lock);
7735 + raw_spin_unlock(&state->lock);
7736 + local_irq_restore(flags);
7737 + }
7738 + release = res->next_replenishment;
7739 +
7740 + if (!release) {
7741 + TRACE_TASK(tsk, "mc2_task_new() next_release = %llu\n", release);
7742 + BUG();
7743 + }
7744 + else
7745 + TRACE_TASK(tsk, "mc2_task_new() next_release = NULL\n");
7746 +}
7747 +
7748 +/* mc2_reservation_destroy - reservation_destroy system call backend
7749 + */
7750 +static long mc2_reservation_destroy(unsigned int reservation_id, int cpu)
7751 +{
7752 + long ret = -EINVAL;
7753 + struct mc2_cpu_state *state;
7754 + struct reservation *res = NULL, *next;
7755 + struct sup_reservation_environment *sup_env;
7756 + int found = 0;
7757 + unsigned long flags;
7758 +
7759 + if (cpu == -1) {
7760 + struct next_timer_event *event, *e_next;
7761 +
7762 + /* if the reservation is global reservation */
7763 +
7764 + local_irq_save(flags);
7765 + raw_spin_lock(&_global_env.lock);
7766 +
7767 + list_for_each_entry_safe(res, next, &_global_env.depleted_reservations, list) {
7768 + if (res->id == reservation_id) {
7769 + list_del(&res->list);
7770 + kfree(res);
7771 + found = 1;
7772 + ret = 0;
7773 + }
7774 + }
7775 + if (!found) {
7776 + list_for_each_entry_safe(res, next, &_global_env.inactive_reservations, list) {
7777 + if (res->id == reservation_id) {
7778 + list_del(&res->list);
7779 + kfree(res);
7780 + found = 1;
7781 + ret = 0;
7782 + }
7783 + }
7784 + }
7785 + if (!found) {
7786 + list_for_each_entry_safe(res, next, &_global_env.active_reservations, list) {
7787 + if (res->id == reservation_id) {
7788 + list_del(&res->list);
7789 + kfree(res);
7790 + found = 1;
7791 + ret = 0;
7792 + }
7793 + }
7794 + }
7795 + /* delete corresponding events */
7796 + list_for_each_entry_safe(event, e_next, &_global_env.next_events, list) {
7797 + if (event->id == reservation_id) {
7798 + list_del(&event->list);
7799 + kfree(event);
7800 + }
7801 + }
7802 +
7803 + raw_spin_unlock(&_global_env.lock);
7804 + local_irq_restore(flags);
7805 + } else {
7806 + /* if the reservation is partitioned reservation */
7807 + state = cpu_state_for(cpu);
7808 + local_irq_save(flags);
7809 + raw_spin_lock(&state->lock);
7810 +
7811 + sup_env = &state->sup_env;
7812 + list_for_each_entry_safe(res, next, &sup_env->depleted_reservations, list) {
7813 + if (res->id == reservation_id) {
7814 + list_del(&res->list);
7815 + kfree(res);
7816 + found = 1;
7817 + ret = 0;
7818 + }
7819 + }
7820 + if (!found) {
7821 + list_for_each_entry_safe(res, next, &sup_env->inactive_reservations, list) {
7822 + if (res->id == reservation_id) {
7823 + list_del(&res->list);
7824 + kfree(res);
7825 + found = 1;
7826 + ret = 0;
7827 + }
7828 + }
7829 + }
7830 + if (!found) {
7831 + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) {
7832 + if (res->id == reservation_id) {
7833 + list_del(&res->list);
7834 + kfree(res);
7835 + found = 1;
7836 + ret = 0;
7837 + }
7838 + }
7839 + }
7840 +
7841 + raw_spin_unlock(&state->lock);
7842 + local_irq_restore(flags);
7843 + }
7844 +
7845 + TRACE("Rerservation destroyed ret = %d\n", ret);
7846 + return ret;
7847 +}
7848 +
7849 +/* mc2_task_exit - Task became a normal task (not real-time task)
7850 + */
7851 +static void mc2_task_exit(struct task_struct *tsk)
7852 +{
7853 + unsigned long flags;
7854 + struct mc2_task_state* tinfo = get_mc2_state(tsk);
7855 + struct mc2_cpu_state *state;
7856 + enum crit_level lv = tinfo->mc2_param.crit;
7857 + struct crit_entry* ce;
7858 + int cpu;
7859 +
7860 + local_irq_save(flags);
7861 + if (tinfo->cpu != -1)
7862 + state = cpu_state_for(tinfo->cpu);
7863 + else
7864 + state = local_cpu_state();
7865 +
7866 + raw_spin_lock(&state->lock);
7867 +
7868 + if (state->scheduled == tsk)
7869 + state->scheduled = NULL;
7870 +
7871 + ce = &state->crit_entries[lv];
7872 + if (ce->running == tsk)
7873 + ce->running = NULL;
7874 +
7875 + /* remove from queues */
7876 + if (is_running(tsk)) {
7877 + /* Assumption: litmus_clock() is synchronized across cores
7878 + * [see comment in pres_task_resume()] */
7879 +
7880 + /* update both global and partitioned */
7881 + if (lv < CRIT_LEVEL_C) {
7882 + sup_update_time(&state->sup_env, litmus_clock());
7883 + }
7884 + else if (lv == CRIT_LEVEL_C) {
7885 + raw_spin_lock(&_global_env.lock);
7886 + gmp_update_time(&_global_env, litmus_clock());
7887 + }
7888 + task_departs(tsk, 0);
7889 + if (lv == CRIT_LEVEL_C)
7890 + raw_spin_unlock(&_global_env.lock);
7891 +
7892 + /* NOTE: drops state->lock */
7893 + TRACE("mc2_exit()\n");
7894 +
7895 + mc2_update_timer_and_unlock(state);
7896 + } else {
7897 + raw_spin_unlock(&state->lock);
7898 + }
7899 +
7900 + if (lv == CRIT_LEVEL_C) {
7901 + for_each_online_cpu(cpu) {
7902 + state = cpu_state_for(cpu);
7903 + if (state == local_cpu_state())
7904 + continue;
7905 + raw_spin_lock(&state->lock);
7906 +
7907 + if (state->scheduled == tsk)
7908 + state->scheduled = NULL;
7909 +
7910 + ce = &state->crit_entries[lv];
7911 + if (ce->running == tsk)
7912 + ce->running = NULL;
7913 +
7914 + raw_spin_unlock(&state->lock);
7915 + }
7916 + }
7917 +
7918 + local_irq_restore(flags);
7919 +
7920 + kfree(tsk_rt(tsk)->plugin_state);
7921 + tsk_rt(tsk)->plugin_state = NULL;
7922 + kfree(tsk_rt(tsk)->mc2_data);
7923 + tsk_rt(tsk)->mc2_data = NULL;
7924 +}
7925 +
7926 +/* create_polling_reservation - create a new polling reservation
7927 + */
7928 +static long create_polling_reservation(
7929 + int res_type,
7930 + struct reservation_config *config)
7931 +{
7932 + struct mc2_cpu_state *state;
7933 + struct reservation* res;
7934 + struct polling_reservation *pres;
7935 + unsigned long flags;
7936 + int use_edf = config->priority == LITMUS_NO_PRIORITY;
7937 + int periodic = res_type == PERIODIC_POLLING;
7938 + long err = -EINVAL;
7939 +
7940 + /* sanity checks */
7941 + if (config->polling_params.budget >
7942 + config->polling_params.period) {
7943 + printk(KERN_ERR "invalid polling reservation (%u): "
7944 + "budget > period\n", config->id);
7945 + return -EINVAL;
7946 + }
7947 + if (config->polling_params.budget >
7948 + config->polling_params.relative_deadline
7949 + && config->polling_params.relative_deadline) {
7950 + printk(KERN_ERR "invalid polling reservation (%u): "
7951 + "budget > deadline\n", config->id);
7952 + return -EINVAL;
7953 + }
7954 + if (config->polling_params.offset >
7955 + config->polling_params.period) {
7956 + printk(KERN_ERR "invalid polling reservation (%u): "
7957 + "offset > period\n", config->id);
7958 + return -EINVAL;
7959 + }
7960 +
7961 + /* Allocate before we grab a spin lock.
7962 + * Todo: would be nice to use a core-local allocation.
7963 + */
7964 + pres = kzalloc(sizeof(*pres), GFP_KERNEL);
7965 + if (!pres)
7966 + return -ENOMEM;
7967 +
7968 + if (config->cpu != -1) {
7969 + state = cpu_state_for(config->cpu);
7970 + raw_spin_lock_irqsave(&state->lock, flags);
7971 +
7972 + res = sup_find_by_id(&state->sup_env, config->id);
7973 + if (!res) {
7974 + polling_reservation_init(pres, use_edf, periodic,
7975 + config->polling_params.budget,
7976 + config->polling_params.period,
7977 + config->polling_params.relative_deadline,
7978 + config->polling_params.offset);
7979 + pres->res.id = config->id;
7980 + pres->res.blocked_by_ghost = 0;
7981 + pres->res.is_ghost = NO_CPU;
7982 + if (!use_edf)
7983 + pres->res.priority = config->priority;
7984 + sup_add_new_reservation(&state->sup_env, &pres->res);
7985 + err = config->id;
7986 + TRACE_CUR("reservation created R%d priority : %llu\n", config->id, pres->res.priority);
7987 + } else {
7988 + err = -EEXIST;
7989 + }
7990 +
7991 + raw_spin_unlock_irqrestore(&state->lock, flags);
7992 +
7993 + } else {
7994 + raw_spin_lock_irqsave(&_global_env.lock, flags);
7995 +
7996 + res = gmp_find_by_id(&_global_env, config->id);
7997 + if (!res) {
7998 + polling_reservation_init(pres, use_edf, periodic,
7999 + config->polling_params.budget,
8000 + config->polling_params.period,
8001 + config->polling_params.relative_deadline,
8002 + config->polling_params.offset);
8003 + pres->res.id = config->id;
8004 + pres->res.blocked_by_ghost = 0;
8005 + pres->res.scheduled_on = NO_CPU;
8006 + pres->res.is_ghost = NO_CPU;
8007 + if (!use_edf)
8008 + pres->res.priority = config->priority;
8009 + gmp_add_new_reservation(&_global_env, &pres->res);
8010 + err = config->id;
8011 + } else {
8012 + err = -EEXIST;
8013 + }
8014 + raw_spin_unlock_irqrestore(&_global_env.lock, flags);
8015 + }
8016 +
8017 + if (err < 0)
8018 + kfree(pres);
8019 +
8020 + return err;
8021 +}
8022 +
8023 +#define MAX_INTERVALS 1024
8024 +
8025 +/* create_table_driven_reservation - create a table_driven reservation
8026 + */
8027 +static long create_table_driven_reservation(
8028 + struct reservation_config *config)
8029 +{
8030 + struct mc2_cpu_state *state;
8031 + struct reservation* res;
8032 + struct table_driven_reservation *td_res = NULL;
8033 + struct lt_interval *slots = NULL;
8034 + size_t slots_size;
8035 + unsigned int i, num_slots;
8036 + unsigned long flags;
8037 + long err = -EINVAL;
8038 +
8039 +
8040 + if (!config->table_driven_params.num_intervals) {
8041 + printk(KERN_ERR "invalid table-driven reservation (%u): "
8042 + "no intervals\n", config->id);
8043 + return -EINVAL;
8044 + }
8045 +
8046 + if (config->table_driven_params.num_intervals > MAX_INTERVALS) {
8047 + printk(KERN_ERR "invalid table-driven reservation (%u): "
8048 + "too many intervals (max: %d)\n", config->id, MAX_INTERVALS);
8049 + return -EINVAL;
8050 + }
8051 +
8052 + num_slots = config->table_driven_params.num_intervals;
8053 + slots_size = sizeof(slots[0]) * num_slots;
8054 + slots = kzalloc(slots_size, GFP_KERNEL);
8055 + if (!slots)
8056 + return -ENOMEM;
8057 +
8058 + td_res = kzalloc(sizeof(*td_res), GFP_KERNEL);
8059 + if (!td_res)
8060 + err = -ENOMEM;
8061 + else
8062 + err = copy_from_user(slots,
8063 + config->table_driven_params.intervals, slots_size);
8064 +
8065 + if (!err) {
8066 + /* sanity checks */
8067 + for (i = 0; !err && i < num_slots; i++)
8068 + if (slots[i].end <= slots[i].start) {
8069 + printk(KERN_ERR
8070 + "invalid table-driven reservation (%u): "
8071 + "invalid interval %u => [%llu, %llu]\n",
8072 + config->id, i,
8073 + slots[i].start, slots[i].end);
8074 + err = -EINVAL;
8075 + }
8076 +
8077 + for (i = 0; !err && i + 1 < num_slots; i++)
8078 + if (slots[i + 1].start <= slots[i].end) {
8079 + printk(KERN_ERR
8080 + "invalid table-driven reservation (%u): "
8081 + "overlapping intervals %u, %u\n",
8082 + config->id, i, i + 1);
8083 + err = -EINVAL;
8084 + }
8085 +
8086 + if (slots[num_slots - 1].end >
8087 + config->table_driven_params.major_cycle_length) {
8088 + printk(KERN_ERR
8089 + "invalid table-driven reservation (%u): last "
8090 + "interval ends past major cycle %llu > %llu\n",
8091 + config->id,
8092 + slots[num_slots - 1].end,
8093 + config->table_driven_params.major_cycle_length);
8094 + err = -EINVAL;
8095 + }
8096 + }
8097 +
8098 + if (!err) {
8099 + state = cpu_state_for(config->cpu);
8100 + raw_spin_lock_irqsave(&state->lock, flags);
8101 +
8102 + res = sup_find_by_id(&state->sup_env, config->id);
8103 + if (!res) {
8104 + table_driven_reservation_init(td_res,
8105 + config->table_driven_params.major_cycle_length,
8106 + slots, num_slots);
8107 + td_res->res.id = config->id;
8108 + td_res->res.priority = config->priority;
8109 + td_res->res.blocked_by_ghost = 0;
8110 + sup_add_new_reservation(&state->sup_env, &td_res->res);
8111 + err = config->id;
8112 + } else {
8113 + err = -EEXIST;
8114 + }
8115 +
8116 + raw_spin_unlock_irqrestore(&state->lock, flags);
8117 + }
8118 +
8119 + if (err < 0) {
8120 + kfree(slots);
8121 + kfree(td_res);
8122 + }
8123 +
8124 + return err;
8125 +}
8126 +
8127 +/* mc2_reservation_create - reservation_create system call backend
8128 + */
8129 +static long mc2_reservation_create(int res_type, void* __user _config)
8130 +{
8131 + long ret = -EINVAL;
8132 + struct reservation_config config;
8133 +
8134 + TRACE("Attempt to create reservation (%d)\n", res_type);
8135 +
8136 + if (copy_from_user(&config, _config, sizeof(config)))
8137 + return -EFAULT;
8138 +
8139 + if (config.cpu != -1) {
8140 + if (config.cpu < 0 || !cpu_online(config.cpu)) {
8141 + printk(KERN_ERR "invalid polling reservation (%u): "
8142 + "CPU %d offline\n", config.id, config.cpu);
8143 + return -EINVAL;
8144 + }
8145 + }
8146 +
8147 + switch (res_type) {
8148 + case PERIODIC_POLLING:
8149 + case SPORADIC_POLLING:
8150 + ret = create_polling_reservation(res_type, &config);
8151 + break;
8152 +
8153 + case TABLE_DRIVEN:
8154 + ret = create_table_driven_reservation(&config);
8155 + break;
8156 +
8157 + default:
8158 + return -EINVAL;
8159 + };
8160 +
8161 + return ret;
8162 +}
8163 +
8164 +static struct domain_proc_info mc2_domain_proc_info;
8165 +
8166 +static long mc2_get_domain_proc_info(struct domain_proc_info **ret)
8167 +{
8168 + *ret = &mc2_domain_proc_info;
8169 + return 0;
8170 +}
8171 +
8172 +static void mc2_setup_domain_proc(void)
8173 +{
8174 + int i, cpu;
8175 + int num_rt_cpus = num_online_cpus();
8176 +
8177 + struct cd_mapping *cpu_map, *domain_map;
8178 +
8179 + memset(&mc2_domain_proc_info, sizeof(mc2_domain_proc_info), 0);
8180 + init_domain_proc_info(&mc2_domain_proc_info, num_rt_cpus, num_rt_cpus);
8181 + mc2_domain_proc_info.num_cpus = num_rt_cpus;
8182 + mc2_domain_proc_info.num_domains = num_rt_cpus;
8183 +
8184 + i = 0;
8185 + for_each_online_cpu(cpu) {
8186 + cpu_map = &mc2_domain_proc_info.cpu_to_domains[i];
8187 + domain_map = &mc2_domain_proc_info.domain_to_cpus[i];
8188 +
8189 + cpu_map->id = cpu;
8190 + domain_map->id = i;
8191 + cpumask_set_cpu(i, cpu_map->mask);
8192 + cpumask_set_cpu(cpu, domain_map->mask);
8193 + ++i;
8194 + }
8195 +}
8196 +
8197 +static long mc2_activate_plugin(void)
8198 +{
8199 + int cpu, lv;
8200 + struct mc2_cpu_state *state;
8201 + struct cpu_entry *ce;
8202 +
8203 + gmp_init(&_global_env);
8204 + //raw_spin_lock_init(&_lowest_prio_cpu.lock);
8205 +
8206 + for_each_online_cpu(cpu) {
8207 + TRACE("Initializing CPU%d...\n", cpu);
8208 +
8209 + resched_cpu[cpu] = 0;
8210 + level_a_priorities[cpu] = 0;
8211 + state = cpu_state_for(cpu);
8212 + ce = &_lowest_prio_cpu.cpu_entries[cpu];
8213 +
8214 + ce->cpu = cpu;
8215 + ce->scheduled = NULL;
8216 + ce->deadline = ULLONG_MAX;
8217 + ce->lv = NUM_CRIT_LEVELS;
8218 + ce->will_schedule = false;
8219 +
8220 + raw_spin_lock_init(&state->lock);
8221 + state->cpu = cpu;
8222 + state->scheduled = NULL;
8223 + for (lv = 0; lv < NUM_CRIT_LEVELS; lv++) {
8224 + struct crit_entry *cr_entry = &state->crit_entries[lv];
8225 + cr_entry->level = lv;
8226 + cr_entry->running = NULL;
8227 + }
8228 + sup_init(&state->sup_env);
8229 +
8230 + hrtimer_init(&state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
8231 + state->timer.function = on_scheduling_timer;
8232 + }
8233 +
8234 + mc2_setup_domain_proc();
8235 +
8236 + return 0;
8237 +}
8238 +
8239 +static void mc2_finish_switch(struct task_struct *prev)
8240 +{
8241 + int cpus;
8242 + enum crit_level lv = get_task_crit_level(prev);
8243 + struct mc2_cpu_state *state = local_cpu_state();
8244 +
8245 + state->scheduled = is_realtime(current) ? current : NULL;
8246 + if (lv == CRIT_LEVEL_C) {
8247 + for (cpus = 0; cpus<NR_CPUS; cpus++) {
8248 + if (resched_cpu[cpus] && state->cpu != cpus) {
8249 + resched_cpu[cpus] = 0;
8250 + litmus_reschedule(cpus);
8251 + }
8252 + }
8253 + }
8254 +}
8255 +
8256 +static long mc2_deactivate_plugin(void)
8257 +{
8258 + int cpu;
8259 + struct mc2_cpu_state *state;
8260 + struct reservation *res;
8261 + struct next_timer_event *event;
8262 + struct cpu_entry *ce;
8263 +
8264 + for_each_online_cpu(cpu) {
8265 + state = cpu_state_for(cpu);
8266 + raw_spin_lock(&state->lock);
8267 +
8268 + hrtimer_cancel(&state->timer);
8269 +
8270 + ce = &_lowest_prio_cpu.cpu_entries[cpu];
8271 +
8272 + ce->cpu = cpu;
8273 + ce->scheduled = NULL;
8274 + ce->deadline = ULLONG_MAX;
8275 + ce->lv = NUM_CRIT_LEVELS;
8276 + ce->will_schedule = false;
8277 +
8278 + /* Delete all reservations --- assumes struct reservation
8279 + * is prefix of containing struct. */
8280 +
8281 + while (!list_empty(&state->sup_env.active_reservations)) {
8282 + res = list_first_entry(
8283 + &state->sup_env.active_reservations,
8284 + struct reservation, list);
8285 + list_del(&res->list);
8286 + kfree(res);
8287 + }
8288 +
8289 + while (!list_empty(&state->sup_env.inactive_reservations)) {
8290 + res = list_first_entry(
8291 + &state->sup_env.inactive_reservations,
8292 + struct reservation, list);
8293 + list_del(&res->list);
8294 + kfree(res);
8295 + }
8296 +
8297 + while (!list_empty(&state->sup_env.depleted_reservations)) {
8298 + res = list_first_entry(
8299 + &state->sup_env.depleted_reservations,
8300 + struct reservation, list);
8301 + list_del(&res->list);
8302 + kfree(res);
8303 + }
8304 +
8305 + raw_spin_unlock(&state->lock);
8306 + }
8307 +
8308 + raw_spin_lock(&_global_env.lock);
8309 +
8310 + while (!list_empty(&_global_env.active_reservations)) {
8311 + res = list_first_entry(
8312 + &_global_env.active_reservations,
8313 + struct reservation, list);
8314 + list_del(&res->list);
8315 + kfree(res);
8316 + }
8317 +
8318 + while (!list_empty(&_global_env.inactive_reservations)) {
8319 + res = list_first_entry(
8320 + &_global_env.inactive_reservations,
8321 + struct reservation, list);
8322 + list_del(&res->list);
8323 + kfree(res);
8324 + }
8325 +
8326 + while (!list_empty(&_global_env.depleted_reservations)) {
8327 + res = list_first_entry(
8328 + &_global_env.depleted_reservations,
8329 + struct reservation, list);
8330 + list_del(&res->list);
8331 + kfree(res);
8332 + }
8333 +
8334 + while (!list_empty(&_global_env.next_events)) {
8335 + event = list_first_entry(
8336 + &_global_env.next_events,
8337 + struct next_timer_event, list);
8338 + list_del(&event->list);
8339 + kfree(event);
8340 + }
8341 +
8342 + raw_spin_unlock(&_global_env.lock);
8343 +
8344 + destroy_domain_proc_info(&mc2_domain_proc_info);
8345 + return 0;
8346 +}
8347 +
8348 +static struct sched_plugin mc2_plugin = {
8349 + .plugin_name = "MC2",
8350 + .schedule = mc2_schedule,
8351 + .finish_switch = mc2_finish_switch,
8352 + .task_wake_up = mc2_task_resume,
8353 + .admit_task = mc2_admit_task,
8354 + .task_new = mc2_task_new,
8355 + .task_exit = mc2_task_exit,
8356 + .complete_job = mc2_complete_job,
8357 + .get_domain_proc_info = mc2_get_domain_proc_info,
8358 + .activate_plugin = mc2_activate_plugin,
8359 + .deactivate_plugin = mc2_deactivate_plugin,
8360 + .reservation_create = mc2_reservation_create,
8361 + .reservation_destroy = mc2_reservation_destroy,
8362 +};
8363 +
8364 +static int __init init_mc2(void)
8365 +{
8366 + return register_sched_plugin(&mc2_plugin);
8367 +}
8368 +
8369 +module_init(init_mc2);
8370 diff --git litmus/sched_plugin.c litmus/sched_plugin.c
8371 index edd91e9..7b1eba0 100644
8372 --- litmus/sched_plugin.c
8373 +++ litmus/sched_plugin.c
8374 @@ -13,6 +13,7 @@
8375 #include <litmus/sched_plugin.h>
8376 #include <litmus/preempt.h>
8377 #include <litmus/jobs.h>
8378 +#include <litmus/budget.h>
8379
8380 /*
8381 * Generic function to trigger preemption on either local or remote cpu
8382 @@ -197,6 +198,9 @@ int register_sched_plugin(struct sched_plugin* plugin)
8383 if (!plugin->wait_for_release_at)
8384 plugin->wait_for_release_at = default_wait_for_release_at;
8385
8386 + if (!plugin->current_budget)
8387 + plugin->current_budget = litmus_current_budget;
8388 +
8389 raw_spin_lock(&sched_plugins_lock);
8390 list_add(&plugin->list, &sched_plugins);
8391 raw_spin_unlock(&sched_plugins_lock);
8392 diff --git litmus/sched_psn_edf.c litmus/sched_psn_edf.c
8393 index 2549a3f..216b9f3 100644
8394 --- litmus/sched_psn_edf.c
8395 +++ litmus/sched_psn_edf.c
8396 @@ -23,6 +23,10 @@
8397 #include <litmus/sched_trace.h>
8398 #include <litmus/trace.h>
8399
8400 +#ifdef CONFIG_PGMRT_SUPPORT
8401 +#include <litmus/pgm.h>
8402 +#endif
8403 +
8404 /* to set up domain/cpu mappings */
8405 #include <litmus/litmus_proc.h>
8406
8407 @@ -199,6 +203,62 @@ static struct task_struct* psnedf_schedule(struct task_struct * prev)
8408 */
8409 resched = preempt;
8410
8411 +#ifdef CONFIG_PGMRT_SUPPORT
8412 + if (exists) {
8413 + if (is_pgm_sending(pedf->scheduled)) {
8414 + if (!is_pgm_satisfied(pedf->scheduled)) {
8415 + if (!is_priority_boosted(pedf->scheduled)) {
8416 + TRACE_TASK(pedf->scheduled, "is sending PGM tokens and needs boosting.\n");
8417 + BUG_ON(is_pgm_satisfied(pedf->scheduled));
8418 +
8419 + /* We are either sending tokens or waiting for tokes.
8420 + If waiting: Boost priority so we'll be scheduled
8421 + immediately when needed tokens arrive.
8422 + If sending: Boost priority so no one (specifically, our
8423 + consumers) will preempt us while signalling the token
8424 + transmission.
8425 + */
8426 + tsk_rt(pedf->scheduled)->priority_boosted = 1;
8427 + tsk_rt(pedf->scheduled)->boost_start_time = litmus_clock();
8428 +
8429 + if (likely(!blocks)) {
8430 + requeue(pedf->scheduled, edf);
8431 + /* we may regain the processor */
8432 + if (preempt) {
8433 + preempt = edf_preemption_needed(edf, prev);
8434 + if (!preempt) {
8435 + TRACE_TASK(pedf->scheduled, "blocked preemption by lazy boosting.\n");
8436 + }
8437 + }
8438 + }
8439 + }
8440 + }
8441 + else { /* sending is satisfied */
8442 + tsk_rt(pedf->scheduled)->ctrl_page->pgm_sending = 0;
8443 + tsk_rt(pedf->scheduled)->ctrl_page->pgm_satisfied = 0;
8444 +
8445 + if (is_priority_boosted(pedf->scheduled)) {
8446 + TRACE_TASK(pedf->scheduled,
8447 + "is done sending PGM tokens must relinquish boosting.\n");
8448 + /* clear boosting */
8449 + tsk_rt(pedf->scheduled)->priority_boosted = 0;
8450 + if(likely(!blocks)) {
8451 + /* recheck priority */
8452 + requeue(pedf->scheduled, edf);
8453 + /* we may lose the processor */
8454 + if (!preempt) {
8455 + preempt = edf_preemption_needed(edf, prev);
8456 + if (preempt) {
8457 + TRACE_TASK(pedf->scheduled, "preempted by lazy unboosting.\n");
8458 + }
8459 + }
8460 + }
8461 + }
8462 + }
8463 + }
8464 + }
8465 +#endif
8466 +
8467 /* If a task blocks we have no choice but to reschedule.
8468 */
8469 if (blocks)
8470 @@ -243,7 +303,7 @@ static struct task_struct* psnedf_schedule(struct task_struct * prev)
8471 if (next) {
8472 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
8473 } else {
8474 - TRACE("becoming idle at %llu\n", litmus_clock());
8475 + ; //TRACE("becoming idle at %llu\n", litmus_clock());
8476 }
8477
8478 pedf->scheduled = next;
8479 @@ -644,10 +704,14 @@ static long psnedf_admit_task(struct task_struct* tsk)
8480 /* don't allow tasks on release master CPU */
8481 && task_cpu(tsk) != remote_edf(task_cpu(tsk))->release_master
8482 #endif
8483 - )
8484 + ) {
8485 + TRACE_TASK(tsk, "admitted\n");
8486 return 0;
8487 - else
8488 + }
8489 + else {
8490 + TRACE_TASK(tsk, "not admitted\n");
8491 return -EINVAL;
8492 + }
8493 }
8494
8495 /* Plugin object */
8496 diff --git litmus/sched_task_trace.c litmus/sched_task_trace.c
8497 index c154ec4..c25bc5a 100644
8498 --- litmus/sched_task_trace.c
8499 +++ litmus/sched_task_trace.c
8500 @@ -132,7 +132,6 @@ feather_callback void do_sched_trace_task_param(unsigned long id, unsigned long
8501 rec->data.param.period = get_rt_period(t);
8502 rec->data.param.phase = get_rt_phase(t);
8503 rec->data.param.partition = get_partition(t);
8504 - rec->data.param.class = get_class(t);
8505 put_record(rec);
8506 }
8507 }
8508 @@ -189,6 +188,7 @@ feather_callback void do_sched_trace_task_completion(unsigned long id,
8509 if (rec) {
8510 rec->data.completion.when = now();
8511 rec->data.completion.forced = forced;
8512 + rec->data.completion.exec_time = get_exec_time(t);
8513 put_record(rec);
8514 }
8515 }
8516 diff --git litmus/uncachedev.c litmus/uncachedev.c
8517 index 06a6a7c..8687581 100644
8518 --- litmus/uncachedev.c
8519 +++ litmus/uncachedev.c
8520 @@ -28,8 +28,8 @@ int litmus_uncache_vm_fault(struct vm_area_struct* vma,
8521 /* modeled after SG DMA video4linux, but without DMA. */
8522 /* (see drivers/media/video/videobuf-dma-sg.c) */
8523 struct page *page;
8524 -
8525 - page = alloc_page(GFP_USER);
8526 +
8527 + page = alloc_page(GFP_USER|GFP_COLOR);
8528 if (!page)
8529 return VM_FAULT_OOM;
8530
8531 diff --git mm/dmapool.c mm/dmapool.c
8532 index fd5fe43..b69dc13 100644
8533 --- mm/dmapool.c
8534 +++ mm/dmapool.c
8535 @@ -333,7 +333,7 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
8536
8537 /* pool_alloc_page() might sleep, so temporarily drop &pool->lock */
8538 spin_unlock_irqrestore(&pool->lock, flags);
8539 -
8540 + printk(KERN_INFO "dma_pool_alloc(): called with %x flags\n", mem_flags);
8541 page = pool_alloc_page(pool, mem_flags);
8542 if (!page)
8543 return NULL;
8544 diff --git mm/migrate.c mm/migrate.c
8545 index f53838f..29b69cd 100644
8546 --- mm/migrate.c
8547 +++ mm/migrate.c
8548 @@ -38,6 +38,8 @@
8549 #include <linux/balloon_compaction.h>
8550 #include <linux/mmu_notifier.h>
8551
8552 +#include <litmus/litmus.h> // for TRACE_TASK
8553 +
8554 #include <asm/tlbflush.h>
8555
8556 #define CREATE_TRACE_POINTS
8557 @@ -391,6 +393,65 @@ int migrate_page_move_mapping(struct address_space *mapping,
8558 }
8559
8560 /*
8561 + * Replace the page in the mapping.
8562 + *
8563 + * The number of remaining references must be:
8564 + * 1 for anonymous pages without a mapping
8565 + * 2 for pages with a mapping
8566 + * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
8567 + */
8568 +int replicate_page_move_mapping(struct address_space *mapping,
8569 + struct page *newpage, struct page *page,
8570 + struct buffer_head *head, enum migrate_mode mode,
8571 + int extra_count)
8572 +{
8573 + int prev_count = page_count(page);
8574 + void **pslot;
8575 +
8576 + BUG_ON(!mapping);
8577 +
8578 + spin_lock_irq(&mapping->tree_lock);
8579 +
8580 + pslot = radix_tree_lookup_slot(&mapping->page_tree, page_index(page));
8581 +
8582 + /*
8583 + * Now we know that no one else is looking at the page.
8584 + */
8585 + get_page(newpage); /* add cache reference */
8586 + if (PageSwapCache(page)) {
8587 + SetPageSwapCache(newpage);
8588 + set_page_private(newpage, page_private(page));
8589 + }
8590 +
8591 + /*
8592 + * Drop cache reference from old page by unfreezing
8593 + * to the previous reference.
8594 + * We know this isn't the last reference.
8595 + */
8596 + page_unfreeze_refs(page, prev_count);
8597 +
8598 + /*
8599 + * If moved to a different zone then also account
8600 + * the page for that zone. Other VM counters will be
8601 + * taken care of when we establish references to the
8602 + * new page and drop references to the old page.
8603 + *
8604 + * Note that anonymous pages are accounted for
8605 + * via NR_FILE_PAGES and NR_ANON_PAGES if they
8606 + * are mapped to swap space.
8607 + */
8608 + __dec_zone_page_state(page, NR_FILE_PAGES);
8609 + __inc_zone_page_state(newpage, NR_FILE_PAGES);
8610 + if (!PageSwapCache(page) && PageSwapBacked(page)) {
8611 + __dec_zone_page_state(page, NR_SHMEM);
8612 + __inc_zone_page_state(newpage, NR_SHMEM);
8613 + }
8614 + spin_unlock_irq(&mapping->tree_lock);
8615 +
8616 + return MIGRATEPAGE_SUCCESS;
8617 +}
8618 +
8619 +/*
8620 * The expected number of remaining references is the same as that
8621 * of migrate_page_move_mapping().
8622 */
8623 @@ -550,6 +611,73 @@ void migrate_page_copy(struct page *newpage, struct page *page)
8624 end_page_writeback(newpage);
8625 }
8626
8627 +/*
8628 + * Copy the page to its new location
8629 + */
8630 +void replicate_page_copy(struct page *newpage, struct page *page)
8631 +{
8632 + if (PageHuge(page) || PageTransHuge(page))
8633 + copy_huge_page(newpage, page);
8634 + else
8635 + copy_highpage(newpage, page);
8636 +
8637 + if (PageError(page))
8638 + SetPageError(newpage);
8639 + if (PageReferenced(page))
8640 + SetPageReferenced(newpage);
8641 + if (PageUptodate(page))
8642 + SetPageUptodate(newpage);
8643 + if (PageActive(page)) {
8644 + VM_BUG_ON_PAGE(PageUnevictable(page), page);
8645 + SetPageActive(newpage);
8646 + } else if (PageUnevictable(page))
8647 + SetPageUnevictable(newpage);
8648 + if (PageChecked(page))
8649 + SetPageChecked(newpage);
8650 + if (PageMappedToDisk(page))
8651 + SetPageMappedToDisk(newpage);
8652 +
8653 + if (PageDirty(page)) {
8654 + clear_page_dirty_for_io(page);
8655 + /*
8656 + * Want to mark the page and the radix tree as dirty, and
8657 + * redo the accounting that clear_page_dirty_for_io undid,
8658 + * but we can't use set_page_dirty because that function
8659 + * is actually a signal that all of the page has become dirty.
8660 + * Whereas only part of our page may be dirty.
8661 + */
8662 + if (PageSwapBacked(page))
8663 + SetPageDirty(newpage);
8664 + else
8665 + __set_page_dirty_nobuffers(newpage);
8666 + }
8667 +
8668 + /*
8669 + * Copy NUMA information to the new page, to prevent over-eager
8670 + * future migrations of this same page.
8671 + */
8672 +#ifdef CONFIG_NUMA_BALANCING
8673 + BUG();
8674 +#endif
8675 +
8676 + if (PageMlocked(page)) {
8677 + unsigned long flags;
8678 + int nr_pages = hpage_nr_pages(page);
8679 +
8680 + local_irq_save(flags);
8681 + SetPageMlocked(newpage);
8682 + __mod_zone_page_state(page_zone(newpage), NR_MLOCK, nr_pages);
8683 + local_irq_restore(flags);
8684 + }
8685 +
8686 + /*
8687 + * If any waiters have accumulated on the new page then
8688 + * wake them up.
8689 + */
8690 + if (PageWriteback(newpage))
8691 + end_page_writeback(newpage);
8692 +}
8693 +
8694 /************************************************************
8695 * Migration functions
8696 ***********************************************************/
8697 @@ -578,6 +706,23 @@ int migrate_page(struct address_space *mapping,
8698 }
8699 EXPORT_SYMBOL(migrate_page);
8700
8701 +int replicate_page(struct address_space *mapping,
8702 + struct page *newpage, struct page *page,
8703 + enum migrate_mode mode, int has_replica)
8704 +{
8705 + int rc, extra_count = 0;
8706 +
8707 + BUG_ON(PageWriteback(page)); /* Writeback must be complete */
8708 +
8709 + rc = replicate_page_move_mapping(mapping, newpage, page, NULL, mode, extra_count);
8710 + if (rc != MIGRATEPAGE_SUCCESS)
8711 + return rc;
8712 +
8713 + if (has_replica == 0)
8714 + replicate_page_copy(newpage, page);
8715 + return MIGRATEPAGE_SUCCESS;
8716 +}
8717 +
8718 #ifdef CONFIG_BLOCK
8719 /*
8720 * Migration function for pages with buffers. This function can only be used
8721 @@ -638,6 +783,8 @@ int buffer_migrate_page(struct address_space *mapping,
8722 EXPORT_SYMBOL(buffer_migrate_page);
8723 #endif
8724
8725 +extern struct list_head shared_lib_pages;
8726 +
8727 /*
8728 * Writeback a page to clean the dirty state
8729 */
8730 @@ -763,6 +910,64 @@ static int move_to_new_page(struct page *newpage, struct page *page,
8731 return rc;
8732 }
8733
8734 +/*
8735 + * Copy a page to a newly allocated page
8736 + * The page is locked and all ptes have been successfully removed.
8737 + *
8738 + * The new page will have replaced the old page if this function
8739 + * is successful.
8740 + *
8741 + * Return value:
8742 + * < 0 - error code
8743 + * MIGRATEPAGE_SUCCESS - success
8744 + */
8745 +static int copy_to_new_page(struct page *newpage, struct page *page,
8746 + int page_was_mapped, enum migrate_mode mode,
8747 + int has_replica)
8748 +{
8749 + struct address_space *mapping;
8750 + int rc;
8751 +
8752 + /*
8753 + * Block others from accessing the page when we get around to
8754 + * establishing additional references. We are the only one
8755 + * holding a reference to the new page at this point.
8756 + */
8757 + if (!trylock_page(newpage))
8758 + BUG();
8759 +
8760 + /* Prepare mapping for the new page.*/
8761 + newpage->index = page->index;
8762 + newpage->mapping = page->mapping;
8763 + if (PageSwapBacked(page))
8764 + SetPageSwapBacked(newpage);
8765 +
8766 + mapping = page_mapping(page);
8767 + if (!mapping) {
8768 + /* a shared library page must have a mapping. */
8769 + BUG();
8770 + }
8771 + else if (mapping->a_ops->migratepage) {
8772 + rc = replicate_page(mapping, newpage, page, mode, has_replica);
8773 + }
8774 + else {
8775 + rc = fallback_migrate_page(mapping, newpage, page, mode);
8776 + }
8777 +
8778 + if (rc != MIGRATEPAGE_SUCCESS) {
8779 + newpage->mapping = NULL;
8780 + } else {
8781 + if (page_was_mapped) {
8782 + remove_migration_ptes(page, newpage);
8783 + }
8784 + }
8785 +
8786 + unlock_page(newpage);
8787 +
8788 + return rc;
8789 +}
8790 +
8791 +
8792 static int __unmap_and_move(struct page *page, struct page *newpage,
8793 int force, enum migrate_mode mode)
8794 {
8795 @@ -901,6 +1106,106 @@ out:
8796 return rc;
8797 }
8798
8799 +static int __unmap_and_copy(struct page *page, struct page *newpage,
8800 + int force, enum migrate_mode mode, int has_replica)
8801 +{
8802 + int rc = -EAGAIN;
8803 + int ttu_ret = SWAP_AGAIN;
8804 + int page_was_mapped = 0;
8805 + struct anon_vma *anon_vma = NULL;
8806 +
8807 + if (!trylock_page(page)) {
8808 + if (!force || mode == MIGRATE_ASYNC)
8809 + goto out;
8810 +
8811 + /*
8812 + * It's not safe for direct compaction to call lock_page.
8813 + * For example, during page readahead pages are added locked
8814 + * to the LRU. Later, when the IO completes the pages are
8815 + * marked uptodate and unlocked. However, the queueing
8816 + * could be merging multiple pages for one bio (e.g.
8817 + * mpage_readpages). If an allocation happens for the
8818 + * second or third page, the process can end up locking
8819 + * the same page twice and deadlocking. Rather than
8820 + * trying to be clever about what pages can be locked,
8821 + * avoid the use of lock_page for direct compaction
8822 + * altogether.
8823 + */
8824 + if (current->flags & PF_MEMALLOC)
8825 + goto out;
8826 +
8827 + lock_page(page);
8828 + }
8829 +
8830 + if (PageWriteback(page)) {
8831 + /*
8832 + * The code of shared library cannot be written.
8833 + */
8834 + BUG();
8835 + }
8836 +
8837 + if (PageAnon(page) && !PageKsm(page)) {
8838 + /* The shared library pages must be backed by a file. */
8839 + BUG();
8840 + }
8841 +
8842 + if (unlikely(isolated_balloon_page(page))) {
8843 + BUG();
8844 + }
8845 +
8846 + /*
8847 + * Corner case handling:
8848 + * 1. When a new swap-cache page is read into, it is added to the LRU
8849 + * and treated as swapcache but it has no rmap yet.
8850 + * Calling try_to_unmap() against a page->mapping==NULL page will
8851 + * trigger a BUG. So handle it here.
8852 + * 2. An orphaned page (see truncate_complete_page) might have
8853 + * fs-private metadata. The page can be picked up due to memory
8854 + * offlining. Everywhere else except page reclaim, the page is
8855 + * invisible to the vm, so the page can not be migrated. So try to
8856 + * free the metadata, so the page can be freed.
8857 + */
8858 + if (!page->mapping) {
8859 + VM_BUG_ON_PAGE(PageAnon(page), page);
8860 + if (page_has_private(page)) {
8861 + try_to_free_buffers(page);
8862 + goto out_unlock;
8863 + }
8864 + goto skip_unmap;
8865 + }
8866 +
8867 + /* Establish migration ptes or remove ptes */
8868 + if (page_mapped(page)) {
8869 + struct rmap_walk_control rwc = {
8870 + .rmap_one = try_to_unmap_one_only,
8871 + .arg = (void *)(TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS),
8872 + };
8873 + ttu_ret = rmap_walk(page, &rwc);
8874 +
8875 + page_was_mapped = 1;
8876 + }
8877 +
8878 +skip_unmap:
8879 + if (ttu_ret == SWAP_SUCCESS) {
8880 + rc = copy_to_new_page(newpage, page, page_was_mapped, mode, has_replica);
8881 + } else if (ttu_ret == SWAP_AGAIN)
8882 + printk(KERN_ERR "rmap_walk returned SWAP_AGAIN\n");
8883 + else
8884 + printk(KERN_ERR "rmap_walk failed\n");
8885 +
8886 + if (rc && page_was_mapped)
8887 + remove_migration_ptes(page, page);
8888 +
8889 + /* Drop an anon_vma reference if we took one */
8890 + if (anon_vma)
8891 + put_anon_vma(anon_vma);
8892 +
8893 +out_unlock:
8894 + unlock_page(page);
8895 +out:
8896 + return rc;
8897 +}
8898 +
8899 /*
8900 * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move(). Work
8901 * around it.
8902 @@ -976,6 +1281,97 @@ out:
8903 }
8904
8905 /*
8906 + * Obtain the lock on page, remove all ptes.
8907 + * 1) If r_pfn == INVALID_PFN, then copy the page to the newly allocated page in newpage.
8908 + * 2) If r_pfn != INVALID_PFN, then unmap and modify ptes.
8909 + */
8910 +#include <litmus/replicate_lib.h>
8911 +
8912 +static ICE_noinline int unmap_and_copy(new_page_t get_new_page,
8913 + free_page_t put_new_page,
8914 + unsigned long private, struct page *page,
8915 + int force, enum migrate_mode mode)
8916 +{
8917 + int rc = 0;
8918 + int *result = NULL;
8919 + struct page *newpage;
8920 + struct shared_lib_page *lib_page;
8921 + int master_exist_in_psl = 0, has_replica = 0, cpu = private/2;
8922 +
8923 + /* check if this page is in the PSL list */
8924 + rcu_read_lock();
8925 + list_for_each_entry(lib_page, &shared_lib_pages, list)
8926 + {
8927 + if (page_to_pfn(page) == lib_page->master_pfn) {
8928 + master_exist_in_psl = 1;
8929 + break;
8930 + }
8931 + }
8932 + rcu_read_unlock();
8933 +
8934 + if (lib_page->r_page[cpu] == NULL) {
8935 + newpage = get_new_page(page, private, &result);
8936 + if (!newpage)
8937 + return -ENOMEM;
8938 + } else {
8939 + newpage = lib_page->r_page[cpu];
8940 + has_replica = 1;
8941 + }
8942 +
8943 + if (page_count(page) == 1) {
8944 + /* page was freed from under us. So we are done. */
8945 + goto out;
8946 + }
8947 +
8948 + if (unlikely(PageTransHuge(page)))
8949 + if (unlikely(split_huge_page(page)))
8950 + goto out;
8951 +
8952 + rc = __unmap_and_copy(page, newpage, force, mode, has_replica);
8953 +
8954 + if (has_replica == 0 && rc == MIGRATEPAGE_SUCCESS) {
8955 + lib_page->r_page[cpu] = newpage;
8956 + lib_page->r_pfn[cpu] = page_to_pfn(newpage);
8957 + }
8958 +
8959 +out:
8960 + if (rc != -EAGAIN) {
8961 + /*
8962 + * A page that has been migrated has all references
8963 + * removed and will be freed. A page that has not been
8964 + * migrated will have kepts its references and be
8965 + * restored.
8966 + */
8967 + list_del(&page->lru);
8968 + dec_zone_page_state(page, NR_ISOLATED_ANON +
8969 + page_is_file_cache(page));
8970 + putback_lru_page(page);
8971 + }
8972 +
8973 + /*
8974 + * If migration was not successful and there's a freeing callback, use
8975 + * it. Otherwise, putback_lru_page() will drop the reference grabbed
8976 + * during isolation.
8977 + */
8978 + if (rc != MIGRATEPAGE_SUCCESS && put_new_page) {
8979 + ClearPageSwapBacked(newpage);
8980 + put_new_page(newpage, private);
8981 + } else if (unlikely(__is_movable_balloon_page(newpage))) {
8982 + /* drop our reference, page already in the balloon */
8983 + put_page(newpage);
8984 + } else
8985 + putback_lru_page(newpage);
8986 +
8987 + if (result) {
8988 + if (rc)
8989 + *result = rc;
8990 + else
8991 + *result = page_to_nid(newpage);
8992 + }
8993 + return rc;
8994 +}
8995 +
8996 +/*
8997 * Counterpart of unmap_and_move_page() for hugepage migration.
8998 *
8999 * This function doesn't wait the completion of hugepage I/O
9000 @@ -1159,6 +1555,85 @@ out:
9001 return rc;
9002 }
9003
9004 +/*
9005 + * replicate_pages - replicate the pages specified in a list
9006 + *
9007 + * @from: The list of pages to be migrated.
9008 + * @get_new_page: The function used to allocate free pages to be used
9009 + * if there is no replicated page.
9010 + * @put_new_page: The function used to free target pages if migration
9011 + * fails, or NULL if no special handling is necessary.
9012 + * @private: Private data to be passed on to get_new_page()
9013 + * @mode: The migration mode that specifies the constraints for
9014 + * page migration, if any.
9015 + * @reason: The reason for page migration.
9016 + *
9017 + * The function returns after 10 attempts or if no pages are movable any more
9018 + * because the list has become empty or no retryable pages exist any more.
9019 + * The caller should call putback_lru_pages() to return pages to the LRU
9020 + * or free list only if ret != 0.
9021 + *
9022 + * Returns the number of pages that were not migrated, or an error code.
9023 + */
9024 +int replicate_pages(struct list_head *from, new_page_t get_new_page,
9025 + free_page_t put_new_page, unsigned long private,
9026 + enum migrate_mode mode, int reason)
9027 +{
9028 + int retry = 1;
9029 + int nr_failed = 0;
9030 + int nr_succeeded = 0;
9031 + int pass = 0;
9032 + struct page *page;
9033 + struct page *page2;
9034 + int swapwrite = current->flags & PF_SWAPWRITE;
9035 + int rc;
9036 +
9037 + if (!swapwrite)
9038 + current->flags |= PF_SWAPWRITE;
9039 +
9040 + for(pass = 0; pass < 10 && retry; pass++) {
9041 + retry = 0;
9042 +
9043 + list_for_each_entry_safe(page, page2, from, lru) {
9044 + cond_resched();
9045 +
9046 + rc = unmap_and_copy(get_new_page, put_new_page, private, page, pass > 2, mode);
9047 +
9048 + switch(rc) {
9049 + case -ENOMEM:
9050 + goto out;
9051 + case -EAGAIN:
9052 + retry++;
9053 + break;
9054 + case MIGRATEPAGE_SUCCESS:
9055 + nr_succeeded++;
9056 + break;
9057 + default:
9058 + /*
9059 + * Permanent failure (-EBUSY, -ENOSYS, etc.):
9060 + * unlike -EAGAIN case, the failed page is
9061 + * removed from migration page list and not
9062 + * retried in the next outer loop.
9063 + */
9064 + nr_failed++;
9065 + break;
9066 + }
9067 + }
9068 + }
9069 + rc = nr_failed + retry;
9070 +out:
9071 + if (nr_succeeded)
9072 + count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
9073 + if (nr_failed)
9074 + count_vm_events(PGMIGRATE_FAIL, nr_failed);
9075 + trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
9076 +
9077 + if (!swapwrite)
9078 + current->flags &= ~PF_SWAPWRITE;
9079 +
9080 + return rc;
9081 +}
9082 +
9083 #ifdef CONFIG_NUMA
9084 /*
9085 * Move a list of individual pages
9086 diff --git mm/page_alloc.c mm/page_alloc.c
9087 index 950c002..574eb21 100644
9088 --- mm/page_alloc.c
9089 +++ mm/page_alloc.c
9090 @@ -65,11 +65,32 @@
9091 #include <asm/sections.h>
9092
9093 #include <litmus/litmus.h> /* for is_realtime() */
9094 +#include <litmus/page_dev.h> /* for coloring pages */
9095
9096 #include <asm/tlbflush.h>
9097 #include <asm/div64.h>
9098 #include "internal.h"
9099
9100 +// This Address Decoding is used in imx6-sabredsd platform
9101 +#define BANK_MASK 0x38000000
9102 +#define BANK_SHIFT 27
9103 +
9104 +#define CACHE_MASK 0x0000f000
9105 +#define CACHE_SHIFT 12
9106 +#define MAX_COLOR_NODE 128
9107 +
9108 +/* Decoding page color, 0~15 */
9109 +static inline unsigned int page_color(struct page *page)
9110 +{
9111 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
9112 +}
9113 +
9114 +/* Decoding page bank number, 0~7 */
9115 +static inline unsigned int page_bank(struct page *page)
9116 +{
9117 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
9118 +}
9119 +
9120 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
9121 static DEFINE_MUTEX(pcp_batch_high_lock);
9122 #define MIN_PERCPU_PAGELIST_FRACTION (8)
9123 @@ -464,7 +485,7 @@ static inline void set_page_guard(struct zone *zone, struct page *page,
9124 INIT_LIST_HEAD(&page->lru);
9125 set_page_private(page, order);
9126 /* Guard pages are not available for any usage */
9127 - __mod_zone_freepage_state(zone, -(1 << order), migratetype);
9128 + __mod_zone_freepage_state(zone, -(1 << order), migratetype, bank_to_partition(page_bank(page)));
9129 }
9130
9131 static inline void clear_page_guard(struct zone *zone, struct page *page,
9132 @@ -480,7 +501,7 @@ static inline void clear_page_guard(struct zone *zone, struct page *page,
9133
9134 set_page_private(page, 0);
9135 if (!is_migrate_isolate(migratetype))
9136 - __mod_zone_freepage_state(zone, (1 << order), migratetype);
9137 + __mod_zone_freepage_state(zone, (1 << order), migratetype, bank_to_partition(page_bank(page)));
9138 }
9139 #else
9140 struct page_ext_operations debug_guardpage_ops = { NULL, };
9141 @@ -582,76 +603,151 @@ static inline void __free_one_page(struct page *page,
9142 unsigned long combined_idx;
9143 unsigned long uninitialized_var(buddy_idx);
9144 struct page *buddy;
9145 - int max_order = MAX_ORDER;
9146 + int max_order, parti_no;
9147 +
9148 + parti_no = bank_to_partition(page_bank(page));
9149 + BUG_ON(parti_no < 0 || parti_no > NR_CPUS);
9150
9151 - VM_BUG_ON(!zone_is_initialized(zone));
9152 - VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
9153 + if (parti_no < NR_CPUS) {
9154 + max_order = MAX_PARTITIONED_ORDER;
9155
9156 - VM_BUG_ON(migratetype == -1);
9157 - if (is_migrate_isolate(migratetype)) {
9158 - /*
9159 - * We restrict max order of merging to prevent merge
9160 - * between freepages on isolate pageblock and normal
9161 - * pageblock. Without this, pageblock isolation
9162 - * could cause incorrect freepage accounting.
9163 - */
9164 - max_order = min(MAX_ORDER, pageblock_order + 1);
9165 - } else {
9166 - __mod_zone_freepage_state(zone, 1 << order, migratetype);
9167 - }
9168 + VM_BUG_ON(!zone_is_initialized(zone));
9169 + VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
9170
9171 - page_idx = pfn & ((1 << max_order) - 1);
9172 + VM_BUG_ON(migratetype == -1);
9173 + if (is_migrate_isolate(migratetype)) {
9174 + max_order = min(MAX_PARTITIONED_ORDER, pageblock_order + 1);
9175 + } else {
9176 + __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no);
9177 + }
9178
9179 - VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
9180 - VM_BUG_ON_PAGE(bad_range(zone, page), page);
9181 + page_idx = pfn & ((1 << max_order) - 1);
9182
9183 - while (order < max_order - 1) {
9184 - buddy_idx = __find_buddy_index(page_idx, order);
9185 - buddy = page + (buddy_idx - page_idx);
9186 - if (!page_is_buddy(page, buddy, order))
9187 - break;
9188 - /*
9189 - * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
9190 - * merge with it and move up one order.
9191 - */
9192 - if (page_is_guard(buddy)) {
9193 - clear_page_guard(zone, buddy, order, migratetype);
9194 + VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
9195 + VM_BUG_ON_PAGE(bad_range(zone, page), page);
9196 +
9197 + while (order < max_order - 1) {
9198 + buddy_idx = __find_buddy_index(page_idx, order);
9199 + buddy = page + (buddy_idx - page_idx);
9200 + if (!page_is_buddy(page, buddy, order))
9201 + break;
9202 +
9203 + if (page_is_guard(buddy)) {
9204 + clear_page_guard(zone, buddy, order, migratetype);
9205 + } else {
9206 + list_del(&buddy->lru);
9207 + zone->free_area_d[parti_no][order].nr_free--;
9208 + rmv_page_order(buddy);
9209 + }
9210 + combined_idx = buddy_idx & page_idx;
9211 + page = page + (combined_idx - page_idx);
9212 + page_idx = combined_idx;
9213 + order++;
9214 + }
9215 + set_page_order(page, order);
9216 +
9217 + if ((order < MAX_PARTITIONED_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
9218 + struct page *higher_page, *higher_buddy;
9219 + combined_idx = buddy_idx & page_idx;
9220 + higher_page = page + (combined_idx - page_idx);
9221 + buddy_idx = __find_buddy_index(combined_idx, order + 1);
9222 + higher_buddy = higher_page + (buddy_idx - combined_idx);
9223 + if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
9224 + list_add_tail(&page->lru,
9225 + &zone->free_area_d[parti_no][order].free_list[migratetype]);
9226 + zone->free_area_d[parti_no][order].nr_free++;
9227 + return;
9228 + }
9229 + }
9230 +
9231 + if (order >= MAX_PARTITIONED_ORDER) {
9232 + int n_idx = 0;
9233 + struct page *lower_page;
9234 + for (n_idx = 0 ; n_idx < (1 << (order - MAX_PARTITIONED_ORDER + 1)); n_idx++) {
9235 + lower_page = page + (n_idx << (MAX_PARTITIONED_ORDER - 1));
9236 + if (lower_page->flags & PAGE_FLAGS_CHECK_AT_PREP)
9237 + lower_page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
9238 + set_page_order(lower_page, MAX_PARTITIONED_ORDER-1);
9239 + list_add(&lower_page->lru, &zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].free_list[migratetype]);
9240 + zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].nr_free++;
9241 + }
9242 } else {
9243 - list_del(&buddy->lru);
9244 - zone->free_area[order].nr_free--;
9245 - rmv_page_order(buddy);
9246 + list_add(&page->lru, &zone->free_area_d[parti_no][order].free_list[migratetype]);
9247 + zone->free_area_d[parti_no][order].nr_free++;
9248 }
9249 - combined_idx = buddy_idx & page_idx;
9250 - page = page + (combined_idx - page_idx);
9251 - page_idx = combined_idx;
9252 - order++;
9253 }
9254 - set_page_order(page, order);
9255 + else {
9256 + max_order = MAX_ORDER;
9257
9258 - /*
9259 - * If this is not the largest possible page, check if the buddy
9260 - * of the next-highest order is free. If it is, it's possible
9261 - * that pages are being freed that will coalesce soon. In case,
9262 - * that is happening, add the free page to the tail of the list
9263 - * so it's less likely to be used soon and more likely to be merged
9264 - * as a higher order page
9265 - */
9266 - if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
9267 - struct page *higher_page, *higher_buddy;
9268 - combined_idx = buddy_idx & page_idx;
9269 - higher_page = page + (combined_idx - page_idx);
9270 - buddy_idx = __find_buddy_index(combined_idx, order + 1);
9271 - higher_buddy = higher_page + (buddy_idx - combined_idx);
9272 - if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
9273 - list_add_tail(&page->lru,
9274 - &zone->free_area[order].free_list[migratetype]);
9275 - goto out;
9276 + VM_BUG_ON(!zone_is_initialized(zone));
9277 + VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
9278 +
9279 + VM_BUG_ON(migratetype == -1);
9280 + if (is_migrate_isolate(migratetype)) {
9281 + /*
9282 + * We restrict max order of merging to prevent merge
9283 + * between freepages on isolate pageblock and normal
9284 + * pageblock. Without this, pageblock isolation
9285 + * could cause incorrect freepage accounting.
9286 + */
9287 + max_order = min(MAX_ORDER, pageblock_order + 1);
9288 + } else {
9289 + __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no);
9290 + }
9291 +
9292 + page_idx = pfn & ((1 << max_order) - 1);
9293 +
9294 + VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
9295 + VM_BUG_ON_PAGE(bad_range(zone, page), page);
9296 +
9297 + while (order < max_order - 1) {
9298 + buddy_idx = __find_buddy_index(page_idx, order);
9299 + buddy = page + (buddy_idx - page_idx);
9300 + if (!page_is_buddy(page, buddy, order))
9301 + break;
9302 + /*
9303 + * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
9304 + * merge with it and move up one order.
9305 + */
9306 + if (page_is_guard(buddy)) {
9307 + clear_page_guard(zone, buddy, order, migratetype);
9308 + } else {
9309 + list_del(&buddy->lru);
9310 + zone->free_area[order].nr_free--;
9311 + rmv_page_order(buddy);
9312 + }
9313 + combined_idx = buddy_idx & page_idx;
9314 + page = page + (combined_idx - page_idx);
9315 + page_idx = combined_idx;
9316 + order++;
9317 + }
9318 + set_page_order(page, order);
9319 +
9320 + /*
9321 + * If this is not the largest possible page, check if the buddy
9322 + * of the next-highest order is free. If it is, it's possible
9323 + * that pages are being freed that will coalesce soon. In case,
9324 + * that is happening, add the free page to the tail of the list
9325 + * so it's less likely to be used soon and more likely to be merged
9326 + * as a higher order page
9327 + */
9328 + if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
9329 + struct page *higher_page, *higher_buddy;
9330 + combined_idx = buddy_idx & page_idx;
9331 + higher_page = page + (combined_idx - page_idx);
9332 + buddy_idx = __find_buddy_index(combined_idx, order + 1);
9333 + higher_buddy = higher_page + (buddy_idx - combined_idx);
9334 + if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
9335 + list_add_tail(&page->lru,
9336 + &zone->free_area[order].free_list[migratetype]);
9337 + goto out;
9338 + }
9339 }
9340 - }
9341
9342 - list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
9343 + list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
9344 out:
9345 - zone->free_area[order].nr_free++;
9346 + zone->free_area[order].nr_free++;
9347 + }
9348 }
9349
9350 static inline int free_pages_check(struct page *page)
9351 @@ -829,7 +925,10 @@ static void __free_pages_ok(struct page *page, unsigned int order)
9352
9353 migratetype = get_pfnblock_migratetype(page, pfn);
9354 local_irq_save(flags);
9355 - __count_vm_events(PGFREE, 1 << order);
9356 + if (bank_to_partition(page_bank(page)) == NR_CPUS)
9357 + __count_vm_events(PGFREE, 1 << order);
9358 + else if (bank_to_partition(page_bank(page)) < NR_CPUS)
9359 + __count_vm_events(PGFREE_HC, 1 << order);
9360 set_freepage_migratetype(page, migratetype);
9361 free_one_page(page_zone(page), page, pfn, order, migratetype);
9362 local_irq_restore(flags);
9363 @@ -930,6 +1029,43 @@ static inline void expand(struct zone *zone, struct page *page,
9364 }
9365 }
9366
9367 +static inline int expand_middle(struct zone *zone, struct page *page,
9368 + int offset, int low, int high, struct free_area *area,
9369 + int migratetype)
9370 +{
9371 + unsigned long size = 1 << high;
9372 +
9373 + while ((size>>1) > offset) {
9374 + area--;
9375 + high--;
9376 + size >>= 1;
9377 + VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
9378 +
9379 + list_add(&page[size].lru, &area->free_list[migratetype]);
9380 + area->nr_free++;
9381 + set_page_order(&page[size], high);
9382 + }
9383 + area--;
9384 + high--;
9385 + size >>= 1;
9386 + VM_BUG_ON_PAGE(bad_range(zone, page), page);
9387 + list_add(&page[0].lru, &area->free_list[migratetype]);
9388 + area->nr_free++;
9389 + set_page_order(&page[0], high);
9390 +
9391 + if (offset == size)
9392 + return high;
9393 +
9394 + area--;
9395 + high--;
9396 + VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
9397 + list_add(&page[size].lru, &area->free_list[migratetype]);
9398 + area->nr_free++;
9399 + set_page_order(&page[size], high);
9400 +
9401 + return high;
9402 +}
9403 +
9404 /*
9405 * This page is about to be returned from the page allocator
9406 */
9407 @@ -996,34 +1132,176 @@ static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
9408 return 0;
9409 }
9410
9411 +/* Kernel page coloring */
9412 +
9413 +/* build colored page list */
9414 +#if 0
9415 +static void build_colored_pages(struct zone *zone, struct page *page, int order)
9416 +{
9417 + int i, color, bank;
9418 +
9419 + list_del(&page->lru);
9420 + zone->free_area[order].nr_free--;
9421 +
9422 + /* insert pages to zone->color_list[] */
9423 + for (i = 0; i < (1<<order); i++) {
9424 + int node;
9425 + color = page_color(&page[i]);
9426 + bank = page_bank(&page[i]);
9427 + node = bank*MAX_NUM_COLOR+color;
9428 +
9429 + INIT_LIST_HEAD(&page[i].lru);
9430 + list_add_tail(&page[i].lru, &zone->color_list[node]);
9431 + bitmap_set(zone->color_map, node, 1);
9432 + zone->free_area[0].nr_free++;
9433 + rmv_page_order(&page[i]);
9434 + }
9435 +}
9436 +
9437 +int color_seq_index[9] = {
9438 + 0, /* Core 0, and Level A*/
9439 + 0, /* Core 0, and Level B*/
9440 + 0, /* Core 1, and Level A*/
9441 + 0, /* Core 1, and Level B*/
9442 + 0, /* Core 2, and Level A*/
9443 + 0, /* Core 2, and Level B*/
9444 + 0, /* Core 3, and Level A*/
9445 + 0, /* Core 3, and Level B*/
9446 + 0, /* Level C */
9447 +};
9448 +
9449 +/* return a colored page */
9450 +static inline struct page *get_colored_page(struct zone *zone, unsigned long req_color_map[BITS_TO_LONGS(MAX_COLOR_NODE)], int order, int partition)
9451 +{
9452 + struct page *page;
9453 + unsigned int color, bank, index;
9454 + int i;
9455 + DECLARE_BITMAP(candidate_bit, MAX_COLOR_NODE);
9456 +
9457 + /* if req_color_map does not exist in zone, return NULL */
9458 + if (!bitmap_intersects(zone->color_map, req_color_map, MAX_COLOR_NODE))
9459 + return NULL;
9460 +
9461 + bitmap_and(candidate_bit, zone->color_map, req_color_map, MAX_COLOR_NODE);
9462 + index = color_seq_index[partition];
9463 +
9464 + for_each_set_bit(i, candidate_bit, MAX_COLOR_NODE) {
9465 + if (index-- <= 0)
9466 + break;
9467 + }
9468 +
9469 + BUG_ON(i >= MAX_COLOR_NODE);
9470 + BUG_ON(list_empty(&zone->color_list[i]));
9471 +
9472 + page = list_entry(zone->color_list[i].next, struct page, lru);
9473 +
9474 + list_del(&page->lru);
9475 +
9476 + if (list_empty(&zone->color_list[i]))
9477 + bitmap_clear(zone->color_map, i, 1);
9478 +
9479 + zone->free_area[0].nr_free--;
9480 + color = page_color(page);
9481 + bank = page_bank(page);
9482 +
9483 + return page;
9484 +}
9485 +#endif
9486 +
9487 /*
9488 * Go through the free lists for the given migratetype and remove
9489 * the smallest available page from the freelists
9490 */
9491 static inline
9492 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
9493 - int migratetype)
9494 + int migratetype, int color_req)
9495 {
9496 unsigned int current_order;
9497 struct free_area *area;
9498 struct page *page;
9499 + int cpu = raw_smp_processor_id();
9500 +
9501 + if (color_req != 0) {
9502 + int found = 0;
9503 + int area_index;
9504 + unsigned long s_pfn = zone->zone_start_pfn;
9505 + unsigned long e_pfn = zone_end_pfn(zone);
9506 +
9507 + area_index = get_area_index(cpu);
9508 +
9509 + /* Find a page of the appropriate size in the preferred list */
9510 + for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) {
9511 + int offset = 0;
9512 + area = &(zone->free_area_d[area_index][current_order]);
9513 + if (list_empty(&area->free_list[migratetype])) {
9514 + continue;
9515 + }
9516 +
9517 + if (order >= MAX_CONTIG_ORDER) {
9518 + page = list_entry(area->free_list[migratetype].next, struct page, lru);
9519 + found = 1;
9520 + } else {
9521 + list_for_each_entry(page, &area->free_list[migratetype], lru) {
9522 + if (current_order < MAX_CONTIG_ORDER) {
9523 + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) {
9524 + offset = 0;
9525 + found = 1;
9526 + break;
9527 + }
9528 + } else {
9529 + int size = 1 << current_order;
9530 + for (offset = 0; offset < size; offset++) {
9531 + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) {
9532 + found = 1;
9533 + break;
9534 + }
9535 + }
9536 + if (found)
9537 + break;
9538 + }
9539 + }
9540 + }
9541 +
9542 +
9543 + if (!found)
9544 + continue;
9545 +
9546 + list_del(&page->lru);
9547 + rmv_page_order(page);
9548 + area->nr_free--;
9549 +
9550 + if (offset == 0) {
9551 + expand(zone, page, order, current_order, area, migratetype);
9552 + } else {
9553 + int frac = expand_middle(zone, page, offset, order, current_order, area, migratetype);
9554 + page = &page[offset];
9555 + area = &(zone->free_area_d[area_index][frac]);
9556 + expand(zone, page, order, frac, area, migratetype);
9557 + }
9558 +
9559 + set_freepage_migratetype(page, migratetype);
9560
9561 - /* Find a page of the appropriate size in the preferred list */
9562 - for (current_order = order; current_order < MAX_ORDER; ++current_order) {
9563 - area = &(zone->free_area[current_order]);
9564 - if (list_empty(&area->free_list[migratetype]))
9565 - continue;
9566 + return page;
9567 + }
9568 + } else {
9569 + /* Buddy allocator */
9570 + /* Find a page of the appropriate size in the preferred list */
9571 + for (current_order = order; current_order < MAX_ORDER; ++current_order) {
9572 + area = &(zone->free_area[current_order]);
9573 + if (list_empty(&area->free_list[migratetype]))
9574 + continue;
9575
9576 - page = list_entry(area->free_list[migratetype].next,
9577 - struct page, lru);
9578 - list_del(&page->lru);
9579 - rmv_page_order(page);
9580 - area->nr_free--;
9581 - expand(zone, page, order, current_order, area, migratetype);
9582 - set_freepage_migratetype(page, migratetype);
9583 - return page;
9584 + page = list_entry(area->free_list[migratetype].next,
9585 + struct page, lru);
9586 + list_del(&page->lru);
9587 + rmv_page_order(page);
9588 + area->nr_free--;
9589 + expand(zone, page, order, current_order, area, migratetype);
9590 + set_freepage_migratetype(page, migratetype);
9591 + return page;
9592 + }
9593 }
9594 -
9595 +
9596 return NULL;
9597 }
9598
9599 @@ -1049,7 +1327,7 @@ static int fallbacks[MIGRATE_TYPES][4] = {
9600 static struct page *__rmqueue_cma_fallback(struct zone *zone,
9601 unsigned int order)
9602 {
9603 - return __rmqueue_smallest(zone, order, MIGRATE_CMA);
9604 + return __rmqueue_smallest(zone, order, MIGRATE_CMA, 0);
9605 }
9606 #else
9607 static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
9608 @@ -1236,7 +1514,7 @@ int find_suitable_fallback(struct free_area *area, unsigned int order,
9609
9610 /* Remove an element from the buddy allocator from the fallback list */
9611 static inline struct page *
9612 -__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
9613 +__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, int color_req)
9614 {
9615 struct free_area *area;
9616 unsigned int current_order;
9617 @@ -1244,44 +1522,128 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
9618 int fallback_mt;
9619 bool can_steal;
9620
9621 - /* Find the largest possible block of pages in the other list */
9622 - for (current_order = MAX_ORDER-1;
9623 - current_order >= order && current_order <= MAX_ORDER-1;
9624 - --current_order) {
9625 - area = &(zone->free_area[current_order]);
9626 - fallback_mt = find_suitable_fallback(area, current_order,
9627 - start_migratetype, false, &can_steal);
9628 - if (fallback_mt == -1)
9629 - continue;
9630 + if (color_req != 0) {
9631 + int cpu = raw_smp_processor_id();
9632 + int found = 0;
9633 + int area_index;
9634 + unsigned long s_pfn = zone->zone_start_pfn;
9635 + unsigned long e_pfn = zone_end_pfn(zone);
9636 +
9637 + area_index = get_area_index(cpu);
9638 +
9639 + /* Find the largest possible block of pages in the other list */
9640 + for (current_order = MAX_PARTITIONED_ORDER-1;
9641 + current_order >= order && current_order <= MAX_PARTITIONED_ORDER-1;
9642 + --current_order) {
9643 + int offset = 0;
9644 + area = &(zone->free_area_d[area_index][current_order]);
9645 + fallback_mt = find_suitable_fallback(area, current_order,
9646 + start_migratetype, false, &can_steal);
9647 + if (fallback_mt == -1)
9648 + continue;
9649 +
9650 + if (order >= MAX_CONTIG_ORDER) {
9651 + page = list_entry(area->free_list[fallback_mt].next, struct page, lru);
9652 + found = 1;
9653 + } else {
9654 + list_for_each_entry(page, &area->free_list[fallback_mt], lru) {
9655 + if (current_order < MAX_CONTIG_ORDER) {
9656 + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) {
9657 + found = 1;
9658 + offset = 0;
9659 + break;
9660 + }
9661 + } else {
9662 + int size = 1 << current_order;
9663 + for (offset = 0; offset < size; offset++) {
9664 + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) {
9665 + found = 1;
9666 + break;
9667 + }
9668 + }
9669 + if (found)
9670 + break;
9671 + }
9672 + }
9673 + }
9674
9675 - page = list_entry(area->free_list[fallback_mt].next,
9676 - struct page, lru);
9677 - if (can_steal)
9678 - steal_suitable_fallback(zone, page, start_migratetype);
9679 + if (!found)
9680 + continue;
9681 +
9682 + if (can_steal)
9683 + steal_suitable_fallback(zone, page, start_migratetype);
9684
9685 - /* Remove the page from the freelists */
9686 - area->nr_free--;
9687 - list_del(&page->lru);
9688 - rmv_page_order(page);
9689 + /* Remove the page from the freelists */
9690 + area->nr_free--;
9691 + list_del(&page->lru);
9692 + rmv_page_order(page);
9693 +
9694 + if (offset == 0)
9695 + expand(zone, page, order, current_order, area, start_migratetype);
9696 + else {
9697 + int frac = expand_middle(zone, page, offset, order, current_order, area, start_migratetype);
9698 + page = &page[offset];
9699 + area = &(zone->free_area_d[area_index][frac]);
9700 + expand(zone, page, order, frac, area, start_migratetype);
9701 +
9702 + }
9703 +
9704
9705 - expand(zone, page, order, current_order, area,
9706 - start_migratetype);
9707 - /*
9708 - * The freepage_migratetype may differ from pageblock's
9709 - * migratetype depending on the decisions in
9710 - * try_to_steal_freepages(). This is OK as long as it
9711 - * does not differ for MIGRATE_CMA pageblocks. For CMA
9712 - * we need to make sure unallocated pages flushed from
9713 - * pcp lists are returned to the correct freelist.
9714 - */
9715 - set_freepage_migratetype(page, start_migratetype);
9716 + /*
9717 + * The freepage_migratetype may differ from pageblock's
9718 + * migratetype depending on the decisions in
9719 + * try_to_steal_freepages(). This is OK as long as it
9720 + * does not differ for MIGRATE_CMA pageblocks. For CMA
9721 + * we need to make sure unallocated pages flushed from
9722 + * pcp lists are returned to the correct freelist.
9723 + */
9724 + set_freepage_migratetype(page, start_migratetype);
9725
9726 - trace_mm_page_alloc_extfrag(page, order, current_order,
9727 - start_migratetype, fallback_mt);
9728 + trace_mm_page_alloc_extfrag(page, order, current_order,
9729 + start_migratetype, fallback_mt);
9730
9731 - return page;
9732 - }
9733 + return page;
9734 + }
9735 + } else {
9736 + /* Find the largest possible block of pages in the other list */
9737 + for (current_order = MAX_ORDER-1;
9738 + current_order >= order && current_order <= MAX_ORDER-1;
9739 + --current_order) {
9740 + area = &(zone->free_area[current_order]);
9741 + fallback_mt = find_suitable_fallback(area, current_order,
9742 + start_migratetype, false, &can_steal);
9743 + if (fallback_mt == -1)
9744 + continue;
9745 +
9746 + page = list_entry(area->free_list[fallback_mt].next,
9747 + struct page, lru);
9748 + if (can_steal)
9749 + steal_suitable_fallback(zone, page, start_migratetype);
9750
9751 + /* Remove the page from the freelists */
9752 + area->nr_free--;
9753 + list_del(&page->lru);
9754 + rmv_page_order(page);
9755 +
9756 + expand(zone, page, order, current_order, area,
9757 + start_migratetype);
9758 + /*
9759 + * The freepage_migratetype may differ from pageblock's
9760 + * migratetype depending on the decisions in
9761 + * try_to_steal_freepages(). This is OK as long as it
9762 + * does not differ for MIGRATE_CMA pageblocks. For CMA
9763 + * we need to make sure unallocated pages flushed from
9764 + * pcp lists are returned to the correct freelist.
9765 + */
9766 + set_freepage_migratetype(page, start_migratetype);
9767 +
9768 + trace_mm_page_alloc_extfrag(page, order, current_order,
9769 + start_migratetype, fallback_mt);
9770 +
9771 + return page;
9772 + }
9773 + }
9774 +
9775 return NULL;
9776 }
9777
9778 @@ -1290,26 +1652,27 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
9779 * Call me with the zone->lock already held.
9780 */
9781 static struct page *__rmqueue(struct zone *zone, unsigned int order,
9782 - int migratetype)
9783 + int migratetype, int color_req)
9784 {
9785 struct page *page;
9786
9787 retry_reserve:
9788 - page = __rmqueue_smallest(zone, order, migratetype);
9789 + page = __rmqueue_smallest(zone, order, migratetype, color_req);
9790
9791 if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
9792 if (migratetype == MIGRATE_MOVABLE)
9793 page = __rmqueue_cma_fallback(zone, order);
9794
9795 - if (!page)
9796 - page = __rmqueue_fallback(zone, order, migratetype);
9797 + if (!page) {
9798 + page = __rmqueue_fallback(zone, order, migratetype, color_req);
9799 + }
9800
9801 /*
9802 * Use MIGRATE_RESERVE rather than fail an allocation. goto
9803 * is used because __rmqueue_smallest is an inline function
9804 * and we want just one call site
9805 */
9806 - if (!page) {
9807 + if (!page && !color_req) {
9808 migratetype = MIGRATE_RESERVE;
9809 goto retry_reserve;
9810 }
9811 @@ -1332,7 +1695,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
9812
9813 spin_lock(&zone->lock);
9814 for (i = 0; i < count; ++i) {
9815 - struct page *page = __rmqueue(zone, order, migratetype);
9816 + struct page *page = __rmqueue(zone, order, migratetype, 0);
9817 if (unlikely(page == NULL))
9818 break;
9819
9820 @@ -1549,8 +1912,16 @@ void free_hot_cold_page(struct page *page, bool cold)
9821 migratetype = get_pfnblock_migratetype(page, pfn);
9822 set_freepage_migratetype(page, migratetype);
9823 local_irq_save(flags);
9824 - __count_vm_event(PGFREE);
9825 -
9826 +
9827 +
9828 + if (bank_to_partition(page_bank(page)) == NR_CPUS)
9829 + __count_vm_event(PGFREE);
9830 + else if (bank_to_partition(page_bank(page)) < NR_CPUS) {
9831 + __count_vm_event(PGFREE_HC);
9832 + free_one_page(zone, page, pfn, 0, migratetype);
9833 + goto out;
9834 + }
9835 +
9836 /*
9837 * We only track unmovable, reclaimable and movable on pcp lists.
9838 * Free ISOLATE pages back to the allocator because they are being
9839 @@ -1565,19 +1936,18 @@ void free_hot_cold_page(struct page *page, bool cold)
9840 }
9841 migratetype = MIGRATE_MOVABLE;
9842 }
9843 -
9844 - pcp = &this_cpu_ptr(zone->pageset)->pcp;
9845 - if (!cold)
9846 - list_add(&page->lru, &pcp->lists[migratetype]);
9847 - else
9848 - list_add_tail(&page->lru, &pcp->lists[migratetype]);
9849 - pcp->count++;
9850 - if (pcp->count >= pcp->high) {
9851 - unsigned long batch = READ_ONCE(pcp->batch);
9852 - free_pcppages_bulk(zone, batch, pcp);
9853 - pcp->count -= batch;
9854 - }
9855 -
9856 +
9857 + pcp = &this_cpu_ptr(zone->pageset)->pcp;
9858 + if (!cold)
9859 + list_add(&page->lru, &pcp->lists[migratetype]);
9860 + else
9861 + list_add_tail(&page->lru, &pcp->lists[migratetype]);
9862 + pcp->count++;
9863 + if (pcp->count >= pcp->high) {
9864 + unsigned long batch = READ_ONCE(pcp->batch);
9865 + free_pcppages_bulk(zone, batch, pcp);
9866 + pcp->count -= batch;
9867 + }
9868 out:
9869 local_irq_restore(flags);
9870 }
9871 @@ -1590,8 +1960,12 @@ void free_hot_cold_page_list(struct list_head *list, bool cold)
9872 struct page *page, *next;
9873
9874 list_for_each_entry_safe(page, next, list, lru) {
9875 + int parti_no = bank_to_partition(page_bank(page));
9876 trace_mm_page_free_batched(page, cold);
9877 - free_hot_cold_page(page, cold);
9878 + if (parti_no == NR_CPUS)
9879 + free_hot_cold_page(page, cold);
9880 + else
9881 + __free_pages_ok(page, 0);
9882 }
9883 }
9884
9885 @@ -1644,7 +2018,7 @@ int __isolate_free_page(struct page *page, unsigned int order)
9886 if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
9887 return 0;
9888
9889 - __mod_zone_freepage_state(zone, -(1UL << order), mt);
9890 + __mod_zone_freepage_state(zone, -(1UL << order), mt, bank_to_partition(page_bank(page)));
9891 }
9892
9893 /* Remove page from free list */
9894 @@ -1705,8 +2079,9 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
9895 unsigned long flags;
9896 struct page *page;
9897 bool cold = ((gfp_flags & __GFP_COLD) != 0);
9898 + int colored_req = ((gfp_flags & __GFP_COLOR) != 0);
9899
9900 - if (likely(order == 0)) {
9901 + if (likely(order == 0) && !colored_req) {
9902 struct per_cpu_pages *pcp;
9903 struct list_head *list;
9904
9905 @@ -1743,12 +2118,12 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
9906 WARN_ON_ONCE(order > 1);
9907 }
9908 spin_lock_irqsave(&zone->lock, flags);
9909 - page = __rmqueue(zone, order, migratetype);
9910 + page = __rmqueue(zone, order, migratetype, colored_req);
9911 spin_unlock(&zone->lock);
9912 if (!page)
9913 goto failed;
9914 __mod_zone_freepage_state(zone, -(1 << order),
9915 - get_freepage_migratetype(page));
9916 + get_freepage_migratetype(page), bank_to_partition(page_bank(page)));
9917 }
9918
9919 __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
9920 @@ -2950,8 +3325,9 @@ EXPORT_SYMBOL(get_zeroed_page);
9921
9922 void __free_pages(struct page *page, unsigned int order)
9923 {
9924 + int parti_no = bank_to_partition(page_bank(page));
9925 if (put_page_testzero(page)) {
9926 - if (order == 0)
9927 + if (order == 0 && parti_no == NR_CPUS)
9928 free_hot_cold_page(page, false);
9929 else
9930 __free_pages_ok(page, order);
9931 @@ -4230,10 +4606,22 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
9932 static void __meminit zone_init_free_lists(struct zone *zone)
9933 {
9934 unsigned int order, t;
9935 + int cpu;
9936 +
9937 for_each_migratetype_order(order, t) {
9938 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
9939 zone->free_area[order].nr_free = 0;
9940 }
9941 +
9942 + /* Initialize per-partition free_area data structures */
9943 + for (cpu = 0; cpu < NR_CPUS; cpu++) {
9944 + for (order = 0; order < MAX_PARTITIONED_ORDER; order++) {
9945 + for (t = 0; t < MIGRATE_TYPES; t++) {
9946 + INIT_LIST_HEAD(&zone->free_area_d[cpu][order].free_list[t]);
9947 + zone->free_area_d[cpu][order].nr_free = 0;
9948 + }
9949 + }
9950 + }
9951 }
9952
9953 #ifndef __HAVE_ARCH_MEMMAP_INIT
9954 diff --git mm/page_isolation.c mm/page_isolation.c
9955 index 303c908..ed05910 100644
9956 --- mm/page_isolation.c
9957 +++ mm/page_isolation.c
9958 @@ -58,12 +58,13 @@ out:
9959 if (!ret) {
9960 unsigned long nr_pages;
9961 int migratetype = get_pageblock_migratetype(page);
9962 + int partno = bank_to_partition(page_bank(page));
9963
9964 set_pageblock_migratetype(page, MIGRATE_ISOLATE);
9965 zone->nr_isolate_pageblock++;
9966 nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE);
9967
9968 - __mod_zone_freepage_state(zone, -nr_pages, migratetype);
9969 + __mod_zone_freepage_state(zone, -nr_pages, migratetype, partno);
9970 }
9971
9972 spin_unlock_irqrestore(&zone->lock, flags);
9973 @@ -117,8 +118,9 @@ void unset_migratetype_isolate(struct page *page, unsigned migratetype)
9974 * pageblock scanning for freepage moving.
9975 */
9976 if (!isolated_page) {
9977 + int partno = bank_to_partition(page_bank(page));
9978 nr_pages = move_freepages_block(zone, page, migratetype);
9979 - __mod_zone_freepage_state(zone, nr_pages, migratetype);
9980 + __mod_zone_freepage_state(zone, nr_pages, migratetype, partno);
9981 }
9982 set_pageblock_migratetype(page, migratetype);
9983 zone->nr_isolate_pageblock--;
9984 diff --git mm/rmap.c mm/rmap.c
9985 index 24dd3f9..0613210 100644
9986 --- mm/rmap.c
9987 +++ mm/rmap.c
9988 @@ -1317,6 +1317,170 @@ out_mlock:
9989 return ret;
9990 }
9991
9992 +/*
9993 + * @arg: enum ttu_flags will be passed to this argument
9994 + */
9995 +static int try_to_unmap_one_entry(struct page *page, struct vm_area_struct *vma,
9996 + unsigned long address, void *arg)
9997 +{
9998 + struct mm_struct *mm = vma->vm_mm;
9999 + pte_t *pte;
10000 + pte_t pteval;
10001 + spinlock_t *ptl;
10002 + int ret = SWAP_AGAIN;
10003 + enum ttu_flags flags = (enum ttu_flags)arg;
10004 +
10005 + pte = page_check_address(page, mm, address, &ptl, 0);
10006 + if (!pte)
10007 + goto out;
10008 +
10009 + /*
10010 + * If the page is mlock()d, we cannot swap it out.
10011 + * If it's recently referenced (perhaps page_referenced
10012 + * skipped over this mm) then we should reactivate it.
10013 + */
10014 + if (!(flags & TTU_IGNORE_MLOCK)) {
10015 + if (vma->vm_flags & VM_LOCKED)
10016 + goto out_mlock;
10017 +
10018 + if (flags & TTU_MUNLOCK)
10019 + goto out_unmap;
10020 + }
10021 + if (!(flags & TTU_IGNORE_ACCESS)) {
10022 + if (ptep_clear_flush_young_notify(vma, address, pte)) {
10023 + ret = SWAP_FAIL;
10024 + goto out_unmap;
10025 + }
10026 + }
10027 +
10028 + /* Nuke the page table entry. */
10029 + flush_cache_page(vma, address, page_to_pfn(page));
10030 + pteval = ptep_clear_flush(vma, address, pte);
10031 +
10032 + /* Move the dirty bit to the physical page now the pte is gone. */
10033 + if (pte_dirty(pteval))
10034 + set_page_dirty(page);
10035 +
10036 + /* Update high watermark before we lower rss */
10037 + update_hiwater_rss(mm);
10038 +
10039 + if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
10040 + if (!PageHuge(page)) {
10041 + if (PageAnon(page))
10042 + dec_mm_counter(mm, MM_ANONPAGES);
10043 + else
10044 + dec_mm_counter(mm, MM_FILEPAGES);
10045 + }
10046 + set_pte_at(mm, address, pte,
10047 + swp_entry_to_pte(make_hwpoison_entry(page)));
10048 + } else if (pte_unused(pteval)) {
10049 + /*
10050 + * The guest indicated that the page content is of no
10051 + * interest anymore. Simply discard the pte, vmscan
10052 + * will take care of the rest.
10053 + */
10054 + if (PageAnon(page))
10055 + dec_mm_counter(mm, MM_ANONPAGES);
10056 + else
10057 + dec_mm_counter(mm, MM_FILEPAGES);
10058 + } else if (PageAnon(page)) {
10059 + swp_entry_t entry = { .val = page_private(page) };
10060 + pte_t swp_pte;
10061 +
10062 + if (PageSwapCache(page)) {
10063 + /*
10064 + * Store the swap location in the pte.
10065 + * See handle_pte_fault() ...
10066 + */
10067 + if (swap_duplicate(entry) < 0) {
10068 + set_pte_at(mm, address, pte, pteval);
10069 + ret = SWAP_FAIL;
10070 + goto out_unmap;
10071 + }
10072 + if (list_empty(&mm->mmlist)) {
10073 + spin_lock(&mmlist_lock);
10074 + if (list_empty(&mm->mmlist))
10075 + list_add(&mm->mmlist, &init_mm.mmlist);
10076 + spin_unlock(&mmlist_lock);
10077 + }
10078 + dec_mm_counter(mm, MM_ANONPAGES);
10079 + inc_mm_counter(mm, MM_SWAPENTS);
10080 + } else if (IS_ENABLED(CONFIG_MIGRATION)) {
10081 + /*
10082 + * Store the pfn of the page in a special migration
10083 + * pte. do_swap_page() will wait until the migration
10084 + * pte is removed and then restart fault handling.
10085 + */
10086 + BUG_ON(!(flags & TTU_MIGRATION));
10087 + entry = make_migration_entry(page, pte_write(pteval));
10088 + }
10089 + swp_pte = swp_entry_to_pte(entry);
10090 + if (pte_soft_dirty(pteval))
10091 + swp_pte = pte_swp_mksoft_dirty(swp_pte);
10092 + set_pte_at(mm, address, pte, swp_pte);
10093 + } else if (IS_ENABLED(CONFIG_MIGRATION) &&
10094 + (flags & TTU_MIGRATION)) {
10095 + /* Establish migration entry for a file page */
10096 + swp_entry_t entry;
10097 + entry = make_migration_entry(page, pte_write(pteval));
10098 + set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
10099 + } else
10100 + dec_mm_counter(mm, MM_FILEPAGES);
10101 +
10102 + page_remove_rmap(page);
10103 + page_cache_release(page);
10104 +
10105 +out_unmap:
10106 + pte_unmap_unlock(pte, ptl);
10107 + if (ret != SWAP_FAIL && !(flags & TTU_MUNLOCK)) {
10108 + mmu_notifier_invalidate_page(mm, address);
10109 + ret = SWAP_SUCCESS;
10110 + }
10111 +out:
10112 + return ret;
10113 +
10114 +out_mlock:
10115 + pte_unmap_unlock(pte, ptl);
10116 +
10117 +
10118 + /*
10119 + * We need mmap_sem locking, Otherwise VM_LOCKED check makes
10120 + * unstable result and race. Plus, We can't wait here because
10121 + * we now hold anon_vma->rwsem or mapping->i_mmap_rwsem.
10122 + * if trylock failed, the page remain in evictable lru and later
10123 + * vmscan could retry to move the page to unevictable lru if the
10124 + * page is actually mlocked.
10125 + */
10126 + if (down_read_trylock(&vma->vm_mm->mmap_sem)) {
10127 + if (vma->vm_flags & VM_LOCKED) {
10128 + mlock_vma_page(page);
10129 + ret = SWAP_MLOCK;
10130 + }
10131 + up_read(&vma->vm_mm->mmap_sem);
10132 + }
10133 + return ret;
10134 +}
10135 +
10136 +int try_to_unmap_one_only(struct page *page, struct vm_area_struct *vma,
10137 + unsigned long address, void *arg)
10138 +{
10139 + struct mm_struct *mm = vma->vm_mm;
10140 + struct mm_struct *current_mm;
10141 +
10142 + rcu_read_lock();
10143 + get_task_struct(current);
10144 + rcu_read_unlock();
10145 + current_mm = get_task_mm(current);
10146 + put_task_struct(current);
10147 + if (!current_mm)
10148 + BUG();
10149 +
10150 + if (mm == current_mm) {
10151 + return try_to_unmap_one_entry(page, vma, address, arg);
10152 + }
10153 + return SWAP_AGAIN;
10154 +}
10155 +
10156 bool is_vma_temporary_stack(struct vm_area_struct *vma)
10157 {
10158 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
10159 diff --git mm/slab.h mm/slab.h
10160 index 4c3ac12..48be14c 100644
10161 --- mm/slab.h
10162 +++ mm/slab.h
10163 @@ -114,7 +114,7 @@ static inline unsigned long kmem_cache_flags(unsigned long object_size,
10164
10165 /* Legal flag mask for kmem_cache_create(), for various configurations */
10166 #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
10167 - SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )
10168 + SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS | SLAB_NO_MERGE)
10169
10170 #if defined(CONFIG_DEBUG_SLAB)
10171 #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
10172 diff --git mm/slab_common.c mm/slab_common.c
10173 index 999bb34..350acdb 100644
10174 --- mm/slab_common.c
10175 +++ mm/slab_common.c
10176 @@ -30,12 +30,18 @@ LIST_HEAD(slab_caches);
10177 DEFINE_MUTEX(slab_mutex);
10178 struct kmem_cache *kmem_cache;
10179
10180 +#ifdef CONFIG_UVC_ALLOC_LEVEL_AB
10181 +#define KMEM_FLAG (GFP_COLOR)
10182 +#else
10183 +#define KMEM_FLAG (0)
10184 +#endif
10185 +
10186 /*
10187 * Set of flags that will prevent slab merging
10188 */
10189 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
10190 SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \
10191 - SLAB_FAILSLAB)
10192 + SLAB_FAILSLAB | SLAB_NO_MERGE)
10193
10194 #define SLAB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
10195 SLAB_CACHE_DMA | SLAB_NOTRACK)
10196 @@ -303,7 +309,7 @@ do_kmem_cache_create(const char *name, size_t object_size, size_t size,
10197 int err;
10198
10199 err = -ENOMEM;
10200 - s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
10201 + s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL|KMEM_FLAG);
10202 if (!s)
10203 goto out;
10204
10205 @@ -703,7 +709,9 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
10206 panic("Out of memory when creating slab %s\n", name);
10207
10208 create_boot_cache(s, name, size, flags);
10209 +
10210 list_add(&s->list, &slab_caches);
10211 +
10212 s->refcount = 1;
10213 return s;
10214 }
10215 @@ -711,6 +719,11 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
10216 struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
10217 EXPORT_SYMBOL(kmalloc_caches);
10218
10219 +/* for per-cpu kmalloc objects */
10220 +struct kmem_cache *hc_kmalloc_caches[NR_CPUS][KMALLOC_SHIFT_HIGH + 1];
10221 +//struct kmem_cache *hc_kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
10222 +EXPORT_SYMBOL(hc_kmalloc_caches);
10223 +
10224 #ifdef CONFIG_ZONE_DMA
10225 struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
10226 EXPORT_SYMBOL(kmalloc_dma_caches);
10227 @@ -780,7 +793,13 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
10228 return kmalloc_dma_caches[index];
10229
10230 #endif
10231 - return kmalloc_caches[index];
10232 +
10233 + if (flags & GFP_COLOR) {
10234 + int cpu = raw_smp_processor_id();
10235 + return hc_kmalloc_caches[cpu][index];
10236 + }
10237 + else
10238 + return kmalloc_caches[index];
10239 }
10240
10241 /*
10242 @@ -790,7 +809,7 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
10243 */
10244 void __init create_kmalloc_caches(unsigned long flags)
10245 {
10246 - int i;
10247 + int i, cpu = 0;
10248
10249 /*
10250 * Patch up the size_index table if we have strange large alignment
10251 @@ -844,11 +863,36 @@ void __init create_kmalloc_caches(unsigned long flags)
10252 * These have to be created immediately after the
10253 * earlier power of two caches
10254 */
10255 - if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6)
10256 + if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6) {
10257 kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
10258 + }
10259
10260 - if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7)
10261 + if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7) {
10262 kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
10263 + }
10264 + }
10265 +
10266 +/* per-cpu kmalloc caches */
10267 + for (cpu = 0; cpu < NR_CPUS; cpu++) {
10268 + for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
10269 + char *n;
10270 + n = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(i));
10271 +
10272 + hc_kmalloc_caches[cpu][i] = create_kmalloc_cache(n, 1 << i, SLAB_NO_MERGE|flags);
10273 + hc_kmalloc_caches[cpu][i]->cpu_id = cpu;
10274 +
10275 + if (KMALLOC_MIN_SIZE <= 32 && !hc_kmalloc_caches[cpu][1] && i == 6) {
10276 + char *nm;
10277 + nm = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(1));
10278 + hc_kmalloc_caches[cpu][1] = create_kmalloc_cache(nm, 96, SLAB_NO_MERGE|flags);
10279 + }
10280 +
10281 + if (KMALLOC_MIN_SIZE <= 64 && !hc_kmalloc_caches[cpu][2] && i == 7) {
10282 + char *nm;
10283 + nm = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(2));
10284 + hc_kmalloc_caches[cpu][2] = create_kmalloc_cache(nm, 192, SLAB_NO_MERGE|flags);
10285 + }
10286 + }
10287 }
10288
10289 /* Kmalloc array is now usable */
10290 diff --git mm/slub.c mm/slub.c
10291 index 54c0876..dc2e9eb 100644
10292 --- mm/slub.c
10293 +++ mm/slub.c
10294 @@ -115,6 +115,26 @@
10295 * the fast path and disables lockless freelists.
10296 */
10297
10298 +// This Address Decoding is used in imx6-sabredsd platform
10299 +#define BANK_MASK 0x38000000
10300 +#define BANK_SHIFT 27
10301 +
10302 +#define CACHE_MASK 0x0000f000
10303 +#define CACHE_SHIFT 12
10304 +#define MAX_COLOR_NODE 128
10305 +
10306 +/* Decoding page color, 0~15 */
10307 +static inline unsigned int page_color(struct page *page)
10308 +{
10309 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
10310 +}
10311 +
10312 +/* Decoding page bank number, 0~7 */
10313 +static inline unsigned int page_bank(struct page *page)
10314 +{
10315 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
10316 +}
10317 +
10318 static inline int kmem_cache_debug(struct kmem_cache *s)
10319 {
10320 #ifdef CONFIG_SLUB_DEBUG
10321 @@ -1320,8 +1340,9 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s,
10322 if (memcg_charge_slab(s, flags, order))
10323 return NULL;
10324
10325 - if (node == NUMA_NO_NODE)
10326 + if (node == NUMA_NO_NODE) {
10327 page = alloc_pages(flags, order);
10328 + }
10329 else
10330 page = alloc_pages_exact_node(node, flags, order);
10331
10332 @@ -1336,7 +1357,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
10333 struct page *page;
10334 struct kmem_cache_order_objects oo = s->oo;
10335 gfp_t alloc_gfp;
10336 -
10337 +
10338 flags &= gfp_allowed_mask;
10339
10340 if (flags & __GFP_WAIT)
10341 @@ -1419,7 +1440,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
10342 }
10343
10344 page = allocate_slab(s,
10345 - flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
10346 + flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK | GFP_COLOR), node);
10347 if (!page)
10348 goto out;
10349
10350 @@ -2223,6 +2244,7 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
10351 return freelist;
10352
10353 page = new_slab(s, flags, node);
10354 +
10355 if (page) {
10356 c = raw_cpu_ptr(s->cpu_slab);
10357 if (c->page)
10358 @@ -2770,7 +2792,8 @@ EXPORT_SYMBOL(kmem_cache_free);
10359 * take the list_lock.
10360 */
10361 static int slub_min_order;
10362 -static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
10363 +//static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
10364 +static int slub_max_order = 2;
10365 static int slub_min_objects;
10366
10367 /*
10368 @@ -3311,7 +3334,7 @@ void *__kmalloc(size_t size, gfp_t flags)
10369 return kmalloc_large(size, flags);
10370
10371 s = kmalloc_slab(size, flags);
10372 -
10373 +
10374 if (unlikely(ZERO_OR_NULL_PTR(s)))
10375 return s;
10376
10377 @@ -5139,6 +5162,10 @@ static char *create_unique_id(struct kmem_cache *s)
10378 * are matched during merging to guarantee that the id is
10379 * unique.
10380 */
10381 + if (s->flags & SLAB_NO_MERGE) {
10382 + *p++ = 'n';
10383 + p += sprintf(p, "%01d", s->cpu_id);
10384 + }
10385 if (s->flags & SLAB_CACHE_DMA)
10386 *p++ = 'd';
10387 if (s->flags & SLAB_RECLAIM_ACCOUNT)
10388 diff --git mm/vmalloc.c mm/vmalloc.c
10389 index 2faaa29..5b1c46b 100644
10390 --- mm/vmalloc.c
10391 +++ mm/vmalloc.c
10392 @@ -1746,6 +1746,22 @@ void *vmalloc(unsigned long size)
10393 EXPORT_SYMBOL(vmalloc);
10394
10395 /**
10396 + * vmalloc_color - allocate virtually contiguous memory from a dedicated free_area
10397 + * @size: allocation size
10398 + * Allocate enough pages to cover @size from the page level
10399 + * allocator and map them into contiguous kernel virtual space.
10400 + *
10401 + * For tight control over page level allocator and protection flags
10402 + * use __vmalloc() instead.
10403 + */
10404 +void *vmalloc_color(unsigned long size)
10405 +{
10406 + return __vmalloc_node_flags(size, NUMA_NO_NODE,
10407 + GFP_KERNEL | __GFP_HIGHMEM | GFP_COLOR);
10408 +}
10409 +EXPORT_SYMBOL(vmalloc_color);
10410 +
10411 +/**
10412 * vzalloc - allocate virtually contiguous memory with zero fill
10413 * @size: allocation size
10414 * Allocate enough pages to cover @size from the page level
10415 @@ -1787,6 +1803,30 @@ void *vmalloc_user(unsigned long size)
10416 EXPORT_SYMBOL(vmalloc_user);
10417
10418 /**
10419 + * vmalloc_color_user - allocate zeroed virtually contiguous memory for userspace
10420 + * @size: allocation size
10421 + *
10422 + * The resulting memory area is zeroed so it can be mapped to userspace
10423 + * without leaking data.
10424 + */
10425 +void *vmalloc_color_user(unsigned long size)
10426 +{
10427 + struct vm_struct *area;
10428 + void *ret;
10429 +
10430 + ret = __vmalloc_node(size, SHMLBA,
10431 + GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | GFP_COLOR,
10432 + PAGE_KERNEL, NUMA_NO_NODE,
10433 + __builtin_return_address(0));
10434 + if (ret) {
10435 + area = find_vm_area(ret);
10436 + area->flags |= VM_USERMAP;
10437 + }
10438 + return ret;
10439 +}
10440 +EXPORT_SYMBOL(vmalloc_color_user);
10441 +
10442 +/**
10443 * vmalloc_node - allocate memory on a specific node
10444 * @size: allocation size
10445 * @node: numa node
10446 diff --git mm/vmstat.c mm/vmstat.c
10447 index 4f5cd97..b09cfa5 100644
10448 --- mm/vmstat.c
10449 +++ mm/vmstat.c
10450 @@ -739,6 +739,7 @@ const char * const vmstat_text[] = {
10451 "workingset_nodereclaim",
10452 "nr_anon_transparent_hugepages",
10453 "nr_free_cma",
10454 + "nr_free_hc_pages",
10455
10456 /* enum writeback_stat_item counters */
10457 "nr_dirty_threshold",
10458 @@ -754,6 +755,7 @@ const char * const vmstat_text[] = {
10459 TEXTS_FOR_ZONES("pgalloc")
10460
10461 "pgfree",
10462 + "pgfree_hc",
10463 "pgactivate",
10464 "pgdeactivate",
10465
10466 @@ -956,6 +958,67 @@ static void pagetypeinfo_showfree_print(struct seq_file *m,
10467 }
10468 }
10469
10470 +static void pagetypeinfo_showpartitionfree_print(struct seq_file *m,
10471 + pg_data_t *pgdat, struct zone *zone, int cpu)
10472 +{
10473 + int order, mtype;
10474 +
10475 + for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) {
10476 + seq_printf(m, "Node %4d, zone %8s, type %12s ",
10477 + pgdat->node_id,
10478 + zone->name,
10479 + migratetype_names[mtype]);
10480 + for (order = 0; order < MAX_PARTITIONED_ORDER; ++order) {
10481 + unsigned long freecount = 0;
10482 + struct free_area *area;
10483 + struct list_head *curr;
10484 +
10485 + area = &(zone->free_area_d[cpu][order]);
10486 +
10487 + list_for_each(curr, &area->free_list[mtype])
10488 + freecount++;
10489 + seq_printf(m, "%6lu ", freecount);
10490 + }
10491 + seq_putc(m, '\n');
10492 + }
10493 +}
10494 +
10495 +static void walk_zones_in_node_in_partition(struct seq_file *m, pg_data_t *pgdat,
10496 + int cpu, void (*print)(struct seq_file *m, pg_data_t *, struct zone *, int))
10497 +{
10498 + struct zone *zone;
10499 + struct zone *node_zones = pgdat->node_zones;
10500 + unsigned long flags;
10501 +
10502 + for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
10503 + if (!populated_zone(zone))
10504 + continue;
10505 +
10506 + spin_lock_irqsave(&zone->lock, flags);
10507 + print(m, pgdat, zone, cpu);
10508 + spin_unlock_irqrestore(&zone->lock, flags);
10509 + }
10510 +}
10511 +
10512 +/* Print out the free pages at each order for each migatetype and partition */
10513 +static int pagetypeinfo_showpartitioned(struct seq_file *m, void *arg)
10514 +{
10515 + int order, cpu;
10516 + pg_data_t *pgdat = (pg_data_t *)arg;
10517 +
10518 + for_each_online_cpu(cpu) {
10519 + /* Print header */
10520 + seq_putc(m, '\n');
10521 + seq_printf(m, "CPU%d %-43s ", cpu, "free pages count per migrate type at order");
10522 + for (order = 0; order < MAX_PARTITIONED_ORDER; ++order)
10523 + seq_printf(m, "%6d ", order);
10524 + seq_putc(m, '\n');
10525 +
10526 + walk_zones_in_node_in_partition(m, pgdat, cpu, pagetypeinfo_showpartitionfree_print);
10527 + }
10528 + return 0;
10529 +}
10530 +
10531 /* Print out the free pages at each order for each migatetype */
10532 static int pagetypeinfo_showfree(struct seq_file *m, void *arg)
10533 {
10534 @@ -1138,7 +1201,7 @@ static int pagetypeinfo_show(struct seq_file *m, void *arg)
10535 pagetypeinfo_showfree(m, pgdat);
10536 pagetypeinfo_showblockcount(m, pgdat);
10537 pagetypeinfo_showmixedcount(m, pgdat);
10538 -
10539 + pagetypeinfo_showpartitioned(m, pgdat);
10540 return 0;
10541 }
10542
10543 @@ -1180,10 +1243,26 @@ static const struct file_operations pagetypeinfo_file_ops = {
10544 .release = seq_release,
10545 };
10546
10547 +#define BANK_MASK 0x38000000
10548 +#define BANK_SHIFT 27
10549 +#define CACHE_MASK 0x0000f000
10550 +#define CACHE_SHIFT 12
10551 +/* Decoding page bank number, 0~7 */
10552 +static inline unsigned int page_bank(struct page *page)
10553 +{
10554 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
10555 +}
10556 +/* Decoding page color, 0~15 */
10557 +static inline unsigned int page_color(struct page *page)
10558 +{
10559 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
10560 +}
10561 +
10562 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
10563 struct zone *zone)
10564 {
10565 int i;
10566 +
10567 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
10568 seq_printf(m,
10569 "\n pages free %lu"
10570 @@ -1232,6 +1311,16 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
10571 seq_printf(m, "\n vm stats threshold: %d",
10572 pageset->stat_threshold);
10573 #endif
10574 + /* pcp test */
10575 +/* seq_printf(m, "\n");
10576 + for (mtype = 0; mtype < MIGRATE_PCPTYPES; mtype++) {
10577 + struct page *p;
10578 + list_for_each_entry(p, &pageset->pcp.lists[mtype], lru) {
10579 + if (p)
10580 + seq_printf(m, "page bank = %d color = %d\n", page_bank(p), page_color(p));
10581 + }
10582 + }
10583 +*/
10584 }
10585 seq_printf(m,
10586 "\n all_unreclaimable: %u"
Attached Files
To refer to attachments on a page, use attachment:filename, as shown below in the list of files. Do NOT use the URL of the [get] link, since this is subject to change and can break easily.You are not allowed to attach a file to this page.