Attachment 'litmus-rt-os-isolation.patch'
Download 1 diff --git arch/arm/boot/compressed/Makefile arch/arm/boot/compressed/Makefile
2 index 6e1fb2b..e2284fe 100644
3 --- arch/arm/boot/compressed/Makefile
4 +++ arch/arm/boot/compressed/Makefile
5 @@ -107,6 +107,7 @@ ifeq ($(CONFIG_FUNCTION_TRACER),y)
6 ORIG_CFLAGS := $(KBUILD_CFLAGS)
7 KBUILD_CFLAGS = $(subst -pg, , $(ORIG_CFLAGS))
8 endif
9 +KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
10
11 ccflags-y := -fpic -mno-single-pic-base -fno-builtin -I$(obj)
12 asflags-y := -DZIMAGE
13 diff --git arch/arm/include/asm/dma-mapping.h arch/arm/include/asm/dma-mapping.h
14 index b52101d..fef26e0 100644
15 --- arch/arm/include/asm/dma-mapping.h
16 +++ arch/arm/include/asm/dma-mapping.h
17 @@ -219,6 +219,13 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size,
18 void *cpu_addr;
19 BUG_ON(!ops);
20
21 +#ifdef CONFIG_SCHED_DEBUG_TRACE
22 + if (flag&GFP_COLOR) {
23 + printk(KERN_INFO "dma_alloc_attrs() \n");
24 + printk(KERN_INFO "func: %pF at address: %p\n", ops->alloc, ops->alloc);
25 + }
26 +#endif
27 +
28 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
29 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
30 return cpu_addr;
31 diff --git arch/arm/include/asm/unistd.h arch/arm/include/asm/unistd.h
32 index 0c462a9..5291b70 100644
33 --- arch/arm/include/asm/unistd.h
34 +++ arch/arm/include/asm/unistd.h
35 @@ -19,7 +19,8 @@
36 * This may need to be greater than __NR_last_syscall+1 in order to
37 * account for the padding in the syscall table
38 */
39 -#define __NR_syscalls (388 + NR_litmus_syscalls)
40 +#define __NR_syscalls (388 + NR_litmus_syscalls + 0)
41 +
42
43 /*
44 * *NOTE*: This is a ghost syscall private to the kernel. Only the
45 diff --git arch/arm/kernel/calls.S arch/arm/kernel/calls.S
46 index f4738a8..55dc863 100644
47 --- arch/arm/kernel/calls.S
48 +++ arch/arm/kernel/calls.S
49 @@ -409,6 +409,14 @@
50 CALL(sys_wait_for_ts_release)
51 CALL(sys_release_ts)
52 CALL(sys_null_call)
53 +/* 400 */ CALL(sys_get_current_budget)
54 + CALL(sys_reservation_create)
55 + CALL(sys_reservation_destroy)
56 + CALL(sys_set_mc2_task_param)
57 + CALL(sys_set_page_color)
58 +/* 405 */ CALL(sys_test_call)
59 + CALL(sys_run_test)
60 + CALL(sys_lock_buffer)
61
62 #ifndef syscalls_counted
63 .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls
64 diff --git arch/arm/kernel/irq.c arch/arm/kernel/irq.c
65 index 350f188..a9ba6e5 100644
66 --- arch/arm/kernel/irq.c
67 +++ arch/arm/kernel/irq.c
68 @@ -44,6 +44,9 @@
69 #include <asm/mach/irq.h>
70 #include <asm/mach/time.h>
71
72 +#include <litmus/cache_proc.h>
73 +#include <litmus/litmus.h>
74 +
75 unsigned long irq_err_count;
76
77 int arch_show_interrupts(struct seq_file *p, int prec)
78 @@ -66,7 +69,9 @@ int arch_show_interrupts(struct seq_file *p, int prec)
79 */
80 void handle_IRQ(unsigned int irq, struct pt_regs *regs)
81 {
82 + enter_irq_mode();
83 __handle_domain_irq(NULL, irq, false, regs);
84 + exit_irq_mode();
85 }
86
87 /*
88 diff --git arch/arm/mach-imx/mmdc.c arch/arm/mach-imx/mmdc.c
89 index 0411f06..03dce5b 100644
90 --- arch/arm/mach-imx/mmdc.c
91 +++ arch/arm/mach-imx/mmdc.c
92 @@ -40,6 +40,7 @@ static int imx_mmdc_probe(struct platform_device *pdev)
93 reg = mmdc_base + MMDC_MDMISC;
94 /* Get ddr type */
95 val = readl_relaxed(reg);
96 + pr_info("MMDC_MDMISC reg: 0x%08x\n", val);
97 ddr_type = (val & BM_MMDC_MDMISC_DDR_TYPE) >>
98 BP_MMDC_MDMISC_DDR_TYPE;
99
100 diff --git arch/arm/mm/cache-l2x0.c arch/arm/mm/cache-l2x0.c
101 index e309c8f..969da4a 100644
102 --- arch/arm/mm/cache-l2x0.c
103 +++ arch/arm/mm/cache-l2x0.c
104 @@ -33,6 +33,8 @@
105 #include "cache-tauros3.h"
106 #include "cache-aurora-l2.h"
107
108 +#include <litmus/cache_proc.h>
109 +
110 struct l2c_init_data {
111 const char *type;
112 unsigned way_size_0;
113 @@ -651,6 +653,11 @@ static void __init l2c310_enable(void __iomem *base, u32 aux, unsigned num_lock)
114 */
115 aux |= L310_AUX_CTRL_NS_LOCKDOWN;
116
117 + /*
118 + * Always enable non-secure interrupt access control registers
119 + */
120 + aux |= L220_AUX_CTRL_NS_INT_CTRL;
121 +
122 l2c_enable(base, aux, num_lock);
123
124 /* Read back resulting AUX_CTRL value as it could have been altered. */
125 @@ -726,7 +733,6 @@ static void __init l2c310_fixup(void __iomem *base, u32 cache_id,
126
127 if (n) {
128 unsigned i;
129 -
130 pr_info("L2C-310 errat%s", n > 1 ? "a" : "um");
131 for (i = 0; i < n; i++)
132 pr_cont(" %s", errata[i]);
133 @@ -774,6 +780,11 @@ static const struct l2c_init_data l2c310_init_fns __initconst = {
134 },
135 };
136
137 +void l2c310_flush_all(void)
138 +{
139 + l2c210_flush_all();
140 +};
141 +
142 static int __init __l2c_init(const struct l2c_init_data *data,
143 u32 aux_val, u32 aux_mask, u32 cache_id)
144 {
145 @@ -876,6 +887,8 @@ static int __init __l2c_init(const struct l2c_init_data *data,
146 pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n",
147 data->type, cache_id, aux);
148
149 + litmus_setup_lockdown(l2x0_base, cache_id);
150 +
151 return 0;
152 }
153
154 diff --git arch/arm/mm/dma-mapping.c arch/arm/mm/dma-mapping.c
155 index 7e7583d..8a297ad 100644
156 --- arch/arm/mm/dma-mapping.c
157 +++ arch/arm/mm/dma-mapping.c
158 @@ -259,7 +259,8 @@ static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gf
159 page = alloc_pages(gfp, order);
160 if (!page)
161 return NULL;
162 -
163 + if (gfp&GFP_COLOR)
164 + printk(KERN_INFO "__dma_alloc_buffer(): size %d, order %ld requested\n", size, order);
165 /*
166 * Now split the huge page and free the excess pages
167 */
168 @@ -341,6 +342,24 @@ void __init init_dma_coherent_pool_size(unsigned long size)
169 atomic_pool_size = size;
170 }
171
172 +#define BANK_MASK 0x38000000
173 +#define BANK_SHIFT 27
174 +
175 +#define CACHE_MASK 0x0000f000
176 +#define CACHE_SHIFT 12
177 +
178 +/* Decoding page color, 0~15 */
179 +static inline unsigned int page_color(struct page *page)
180 +{
181 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
182 +}
183 +
184 +/* Decoding page bank number, 0~7 */
185 +static inline unsigned int page_bank(struct page *page)
186 +{
187 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
188 +}
189 +
190 /*
191 * Initialise the coherent pool for atomic allocations.
192 */
193 @@ -375,6 +394,7 @@ static int __init atomic_pool_init(void)
194 (void *)PAGE_SHIFT);
195 pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n",
196 atomic_pool_size / 1024);
197 + pr_info("DMA: coherent pool located in 0x%p phys %08x color %d bank %d\n", ptr, page_to_phys(page), page_color(page), page_bank(page));
198 return 0;
199 }
200
201 @@ -644,15 +664,17 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
202 *handle = DMA_ERROR_CODE;
203 size = PAGE_ALIGN(size);
204 want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs);
205 -
206 +
207 if (is_coherent || nommu())
208 addr = __alloc_simple_buffer(dev, size, gfp, &page);
209 else if (!(gfp & __GFP_WAIT))
210 addr = __alloc_from_pool(size, &page);
211 - else if (!dev_get_cma_area(dev))
212 + else if (!dev_get_cma_area(dev)) {
213 addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller, want_vaddr);
214 - else
215 + //printk(KERN_INFO "__alloc_remap_buffer returned %p page, size %d, color %d, bank %d, pfn %05lx\n", page, size, page_color(page), page_bank(page), page_to_pfn(page));
216 + } else {
217 addr = __alloc_from_contiguous(dev, size, prot, &page, caller, want_vaddr);
218 + }
219
220 if (page)
221 *handle = pfn_to_dma(dev, page_to_pfn(page));
222 @@ -670,6 +692,17 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
223 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
224 void *memory;
225
226 + /*
227 + if ((gfp&GFP_COLOR) && (size > PAGE_SIZE*4)) {
228 +#ifdef CONFIG_SCHED_DEBUG_TRACE
229 + printk(KERN_INFO "arm_dma_alloc(): original prot %08x\n", prot);
230 +#endif
231 + //prot = pgprot_noncached(prot);
232 +#ifdef CONFIG_SCHED_DEBUG_TRACE
233 + printk(KERN_INFO "arm_dma_alloc(): set as uncacheable prot %08x\n", prot);
234 +#endif
235 + }
236 + */
237 if (dma_alloc_from_coherent(dev, size, handle, &memory))
238 return memory;
239
240 diff --git arch/x86/syscalls/syscall_32.tbl arch/x86/syscalls/syscall_32.tbl
241 index 34680a5..b303a9b 100644
242 --- arch/x86/syscalls/syscall_32.tbl
243 +++ arch/x86/syscalls/syscall_32.tbl
244 @@ -377,3 +377,11 @@
245 368 i386 wait_for_ts_release sys_wait_for_ts_release
246 369 i386 release_ts sys_release_ts
247 370 i386 null_call sys_null_call
248 +371 i386 get_current_budget sys_get_current_budget
249 +372 i386 reservation_create sys_reservation_create
250 +373 i386 reservation_destroy sys_reservation_destroy
251 +374 i386 set_mc2_task_param sys_set_mc2_task_param
252 +375 i386 set_page_color sys_set_page_color
253 +376 i386 test_call sys_test_call
254 +377 i386 run_test sys_run_test
255 +378 i386 lock_buffer sys_lock_buffer
256 diff --git arch/x86/syscalls/syscall_64.tbl arch/x86/syscalls/syscall_64.tbl
257 index cbd1b6b..5f24a80 100644
258 --- arch/x86/syscalls/syscall_64.tbl
259 +++ arch/x86/syscalls/syscall_64.tbl
260 @@ -342,6 +342,14 @@
261 360 common wait_for_ts_release sys_wait_for_ts_release
262 361 common release_ts sys_release_ts
263 362 common null_call sys_null_call
264 +363 common get_current_budget sys_get_current_budget
265 +364 common reservation_create sys_reservation_create
266 +365 common reservation_destroy sys_reservation_destroy
267 +366 common set_mc2_task_param sys_set_mc2_task_param
268 +367 common set_page_color sys_set_page_color
269 +368 common test_call sys_test_call
270 +369 common run_test sys_run_test
271 +370 common lock_buffer sys_lock_buffer
272
273 #
274 # x32-specific system call numbers start at 512 to avoid cache impact
275 diff --git drivers/media/usb/uvc/uvc_v4l2.c drivers/media/usb/uvc/uvc_v4l2.c
276 index c4b1ac6..e40daf9 100644
277 --- drivers/media/usb/uvc/uvc_v4l2.c
278 +++ drivers/media/usb/uvc/uvc_v4l2.c
279 @@ -1437,7 +1437,9 @@ static int uvc_v4l2_mmap(struct file *file, struct vm_area_struct *vma)
280 struct uvc_streaming *stream = handle->stream;
281
282 uvc_trace(UVC_TRACE_CALLS, "uvc_v4l2_mmap\n");
283 -
284 +#if 0
285 + printk(KERN_INFO "uvc_mmap entry point\n");
286 +#endif
287 return uvc_queue_mmap(&stream->queue, vma);
288 }
289
290 diff --git drivers/media/usb/uvc/uvc_video.c drivers/media/usb/uvc/uvc_video.c
291 index 20ccc9d..d4a64fc 100644
292 --- drivers/media/usb/uvc/uvc_video.c
293 +++ drivers/media/usb/uvc/uvc_video.c
294 @@ -26,6 +26,13 @@
295
296 #include "uvcvideo.h"
297
298 +#define USE_LEVEL_A_BANK 1
299 +#ifdef USE_LEVEL_A_BANK
300 +#define UVC_FLAG (GFP_COLOR)
301 +#else
302 +#define UVC_FLAG (0)
303 +#endif
304 +
305 /* ------------------------------------------------------------------------
306 * UVC Controls
307 */
308 @@ -167,7 +174,7 @@ static int uvc_get_video_ctrl(struct uvc_streaming *stream,
309 query == UVC_GET_DEF)
310 return -EIO;
311
312 - data = kmalloc(size, GFP_KERNEL);
313 + data = kmalloc(size, GFP_KERNEL|UVC_FLAG);
314 if (data == NULL)
315 return -ENOMEM;
316
317 @@ -251,7 +258,7 @@ static int uvc_set_video_ctrl(struct uvc_streaming *stream,
318 int ret;
319
320 size = stream->dev->uvc_version >= 0x0110 ? 34 : 26;
321 - data = kzalloc(size, GFP_KERNEL);
322 + data = kzalloc(size, GFP_KERNEL|UVC_FLAG);
323 if (data == NULL)
324 return -ENOMEM;
325
326 @@ -494,7 +501,7 @@ static int uvc_video_clock_init(struct uvc_streaming *stream)
327 clock->size = 32;
328
329 clock->samples = kmalloc(clock->size * sizeof(*clock->samples),
330 - GFP_KERNEL);
331 + GFP_KERNEL|UVC_FLAG);
332 if (clock->samples == NULL)
333 return -ENOMEM;
334
335 @@ -1343,7 +1350,7 @@ static void uvc_video_complete(struct urb *urb)
336
337 stream->decode(urb, stream, buf);
338
339 - if ((ret = usb_submit_urb(urb, GFP_ATOMIC)) < 0) {
340 + if ((ret = usb_submit_urb(urb, GFP_ATOMIC|UVC_FLAG)) < 0) {
341 uvc_printk(KERN_ERR, "Failed to resubmit video URB (%d).\n",
342 ret);
343 }
344 @@ -1406,10 +1413,10 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream,
345 #ifndef CONFIG_DMA_NONCOHERENT
346 stream->urb_buffer[i] = usb_alloc_coherent(
347 stream->dev->udev, stream->urb_size,
348 - gfp_flags | __GFP_NOWARN, &stream->urb_dma[i]);
349 + gfp_flags | __GFP_NOWARN | UVC_FLAG, &stream->urb_dma[i]);
350 #else
351 stream->urb_buffer[i] =
352 - kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN);
353 + kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN | UVC_FLAG;
354 #endif
355 if (!stream->urb_buffer[i]) {
356 uvc_free_urb_buffers(stream);
357 @@ -1492,14 +1499,14 @@ static int uvc_init_video_isoc(struct uvc_streaming *stream,
358 psize = uvc_endpoint_max_bpi(stream->dev->udev, ep);
359 size = stream->ctrl.dwMaxVideoFrameSize;
360
361 - npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags);
362 + npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags|UVC_FLAG);
363 if (npackets == 0)
364 return -ENOMEM;
365
366 size = npackets * psize;
367
368 for (i = 0; i < UVC_URBS; ++i) {
369 - urb = usb_alloc_urb(npackets, gfp_flags);
370 + urb = usb_alloc_urb(npackets, gfp_flags|UVC_FLAG);
371 if (urb == NULL) {
372 uvc_uninit_video(stream, 1);
373 return -ENOMEM;
374 @@ -1548,7 +1555,7 @@ static int uvc_init_video_bulk(struct uvc_streaming *stream,
375 size = stream->ctrl.dwMaxPayloadTransferSize;
376 stream->bulk.max_payload_size = size;
377
378 - npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags);
379 + npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags|UVC_FLAG);
380 if (npackets == 0)
381 return -ENOMEM;
382
383 @@ -1565,7 +1572,7 @@ static int uvc_init_video_bulk(struct uvc_streaming *stream,
384 size = 0;
385
386 for (i = 0; i < UVC_URBS; ++i) {
387 - urb = usb_alloc_urb(0, gfp_flags);
388 + urb = usb_alloc_urb(0, gfp_flags|UVC_FLAG);
389 if (urb == NULL) {
390 uvc_uninit_video(stream, 1);
391 return -ENOMEM;
392 @@ -1654,7 +1661,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags)
393 if (ret < 0)
394 return ret;
395
396 - ret = uvc_init_video_isoc(stream, best_ep, gfp_flags);
397 + ret = uvc_init_video_isoc(stream, best_ep, gfp_flags|UVC_FLAG);
398 } else {
399 /* Bulk endpoint, proceed to URB initialization. */
400 ep = uvc_find_endpoint(&intf->altsetting[0],
401 @@ -1662,7 +1669,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags)
402 if (ep == NULL)
403 return -EIO;
404
405 - ret = uvc_init_video_bulk(stream, ep, gfp_flags);
406 + ret = uvc_init_video_bulk(stream, ep, gfp_flags|UVC_FLAG);
407 }
408
409 if (ret < 0)
410 @@ -1670,7 +1677,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags)
411
412 /* Submit the URBs. */
413 for (i = 0; i < UVC_URBS; ++i) {
414 - ret = usb_submit_urb(stream->urb[i], gfp_flags);
415 + ret = usb_submit_urb(stream->urb[i], gfp_flags|UVC_FLAG);
416 if (ret < 0) {
417 uvc_printk(KERN_ERR, "Failed to submit URB %u "
418 "(%d).\n", i, ret);
419 @@ -1741,7 +1748,7 @@ int uvc_video_resume(struct uvc_streaming *stream, int reset)
420 if (ret < 0)
421 return ret;
422
423 - return uvc_init_video(stream, GFP_NOIO);
424 + return uvc_init_video(stream, GFP_NOIO|UVC_FLAG);
425 }
426
427 /* ------------------------------------------------------------------------
428 @@ -1892,7 +1899,7 @@ int uvc_video_enable(struct uvc_streaming *stream, int enable)
429 if (ret < 0)
430 goto error_commit;
431
432 - ret = uvc_init_video(stream, GFP_KERNEL);
433 + ret = uvc_init_video(stream, GFP_KERNEL|UVC_FLAG);
434 if (ret < 0)
435 goto error_video;
436
437 diff --git drivers/media/v4l2-core/videobuf2-core.c drivers/media/v4l2-core/videobuf2-core.c
438 index 66ada01..ef75f1f 100644
439 --- drivers/media/v4l2-core/videobuf2-core.c
440 +++ drivers/media/v4l2-core/videobuf2-core.c
441 @@ -30,6 +30,13 @@
442 #include <media/v4l2-common.h>
443 #include <media/videobuf2-core.h>
444
445 +#define USE_LEVEL_A_BANK 1
446 +#ifdef USE_LEVEL_A_BANK
447 +#define VB2_CORE_FLAG (GFP_COLOR)
448 +#else
449 +#define VB2_CORE_FLAG (0)
450 +#endif
451 +
452 static int debug;
453 module_param(debug, int, 0644);
454
455 @@ -200,7 +207,7 @@ static int __vb2_buf_mem_alloc(struct vb2_buffer *vb)
456 */
457 for (plane = 0; plane < vb->num_planes; ++plane) {
458 unsigned long size = PAGE_ALIGN(q->plane_sizes[plane]);
459 -
460 + printk(KERN_INFO "__vb2_buf_mem_alloc(): size %ld, func %pF GFP_COLOR? %d\n", size, vb->vb2_queue->mem_ops->alloc, q->gfp_flags&GFP_COLOR);
461 mem_priv = call_ptr_memop(vb, alloc, q->alloc_ctx[plane],
462 size, dma_dir, q->gfp_flags);
463 if (IS_ERR_OR_NULL(mem_priv))
464 @@ -352,7 +359,7 @@ static int __vb2_queue_alloc(struct vb2_queue *q, enum v4l2_memory memory,
465
466 for (buffer = 0; buffer < num_buffers; ++buffer) {
467 /* Allocate videobuf buffer structures */
468 - vb = kzalloc(q->buf_struct_size, GFP_KERNEL);
469 + vb = kzalloc(q->buf_struct_size, GFP_KERNEL|VB2_CORE_FLAG);
470 if (!vb) {
471 dprintk(1, "memory alloc for buffer struct failed\n");
472 break;
473 @@ -402,7 +409,8 @@ static int __vb2_queue_alloc(struct vb2_queue *q, enum v4l2_memory memory,
474
475 dprintk(1, "allocated %d buffers, %d plane(s) each\n",
476 buffer, num_planes);
477 -
478 + printk(KERN_INFO "allocated %d buffers, %d plane(s) each\n",
479 + buffer, num_planes);
480 return buffer;
481 }
482
483 @@ -2237,6 +2245,7 @@ static int vb2_internal_streamon(struct vb2_queue *q, enum v4l2_buf_type type)
484 * Tell driver to start streaming provided sufficient buffers
485 * are available.
486 */
487 +printk(KERN_INFO "vb2_internal_streamon()\n");
488 if (q->queued_count >= q->min_buffers_needed) {
489 ret = vb2_start_streaming(q);
490 if (ret) {
491 @@ -2525,7 +2534,7 @@ int vb2_mmap(struct vb2_queue *q, struct vm_area_struct *vma)
492 "MMAP invalid, as it would overflow buffer length\n");
493 return -EINVAL;
494 }
495 -
496 +printk(KERN_INFO "memop mmap %pF\n", vb->vb2_queue->mem_ops->mmap);
497 mutex_lock(&q->mmap_lock);
498 ret = call_memop(vb, mmap, vb->planes[plane].mem_priv, vma);
499 mutex_unlock(&q->mmap_lock);
500 @@ -2830,7 +2839,7 @@ static int __vb2_init_fileio(struct vb2_queue *q, int read)
501 (read) ? "read" : "write", count, q->fileio_read_once,
502 q->fileio_write_immediately);
503
504 - fileio = kzalloc(sizeof(struct vb2_fileio_data), GFP_KERNEL);
505 + fileio = kzalloc(sizeof(struct vb2_fileio_data), GFP_KERNEL|VB2_CORE_FLAG);
506 if (fileio == NULL)
507 return -ENOMEM;
508
509 @@ -3223,7 +3232,7 @@ int vb2_thread_start(struct vb2_queue *q, vb2_thread_fnc fnc, void *priv,
510 if (WARN_ON(q->fileio))
511 return -EBUSY;
512
513 - threadio = kzalloc(sizeof(*threadio), GFP_KERNEL);
514 + threadio = kzalloc(sizeof(*threadio), GFP_KERNEL|VB2_CORE_FLAG);
515 if (threadio == NULL)
516 return -ENOMEM;
517 threadio->fnc = fnc;
518 diff --git drivers/media/v4l2-core/videobuf2-vmalloc.c drivers/media/v4l2-core/videobuf2-vmalloc.c
519 index 657ab30..b9a6457 100644
520 --- drivers/media/v4l2-core/videobuf2-vmalloc.c
521 +++ drivers/media/v4l2-core/videobuf2-vmalloc.c
522 @@ -21,6 +21,14 @@
523 #include <media/videobuf2-vmalloc.h>
524 #include <media/videobuf2-memops.h>
525
526 +
527 +#define USE_LEVEL_A_BANK 1
528 +#ifdef USE_LEVEL_A_BANK
529 +#define VB2_FLAG (GFP_COLOR)
530 +#else
531 +#define VB2_FLAG (0)
532 +#endif
533 +
534 struct vb2_vmalloc_buf {
535 void *vaddr;
536 struct page **pages;
537 @@ -39,13 +47,18 @@ static void *vb2_vmalloc_alloc(void *alloc_ctx, unsigned long size,
538 enum dma_data_direction dma_dir, gfp_t gfp_flags)
539 {
540 struct vb2_vmalloc_buf *buf;
541 -
542 - buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags);
543 +/* video buffer allocation */
544 +printk(KERN_INFO "vb2_vmalloc_alloc(): size %ld requested\n", size);
545 + buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags | VB2_FLAG);
546 if (!buf)
547 return NULL;
548
549 buf->size = size;
550 - buf->vaddr = vmalloc_user(buf->size);
551 +#ifdef ENABLE_WORST_CASE
552 + buf->vaddr = vmalloc_color_user_cpu1(buf->size);
553 +#else
554 + buf->vaddr = vmalloc_color_user(buf->size);
555 +#endif
556 buf->dma_dir = dma_dir;
557 buf->handler.refcount = &buf->refcount;
558 buf->handler.put = vb2_vmalloc_put;
559 @@ -81,7 +94,7 @@ static void *vb2_vmalloc_get_userptr(void *alloc_ctx, unsigned long vaddr,
560 struct vm_area_struct *vma;
561 dma_addr_t physp;
562
563 - buf = kzalloc(sizeof(*buf), GFP_KERNEL);
564 + buf = kzalloc(sizeof(*buf), GFP_KERNEL | VB2_FLAG);
565 if (!buf)
566 return NULL;
567
568 @@ -103,7 +116,7 @@ static void *vb2_vmalloc_get_userptr(void *alloc_ctx, unsigned long vaddr,
569 last = (vaddr + size - 1) >> PAGE_SHIFT;
570 buf->n_pages = last - first + 1;
571 buf->pages = kzalloc(buf->n_pages * sizeof(struct page *),
572 - GFP_KERNEL);
573 + GFP_KERNEL | VB2_FLAG);
574 if (!buf->pages)
575 goto fail_pages_array_alloc;
576
577 @@ -233,12 +246,12 @@ static int vb2_vmalloc_dmabuf_ops_attach(struct dma_buf *dbuf, struct device *de
578 int ret;
579 int i;
580
581 - attach = kzalloc(sizeof(*attach), GFP_KERNEL);
582 + attach = kzalloc(sizeof(*attach), GFP_KERNEL | VB2_FLAG);
583 if (!attach)
584 return -ENOMEM;
585
586 sgt = &attach->sgt;
587 - ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL);
588 + ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL | VB2_FLAG);
589 if (ret) {
590 kfree(attach);
591 return ret;
592 @@ -429,7 +442,7 @@ static void *vb2_vmalloc_attach_dmabuf(void *alloc_ctx, struct dma_buf *dbuf,
593 if (dbuf->size < size)
594 return ERR_PTR(-EFAULT);
595
596 - buf = kzalloc(sizeof(*buf), GFP_KERNEL);
597 + buf = kzalloc(sizeof(*buf), GFP_KERNEL | VB2_FLAG);
598 if (!buf)
599 return ERR_PTR(-ENOMEM);
600
601 diff --git drivers/net/ethernet/freescale/fec_main.c drivers/net/ethernet/freescale/fec_main.c
602 index 66d47e4..192b008 100644
603 --- drivers/net/ethernet/freescale/fec_main.c
604 +++ drivers/net/ethernet/freescale/fec_main.c
605 @@ -61,8 +61,17 @@
606
607 #include <asm/cacheflush.h>
608
609 +#include <litmus/cache_proc.h>
610 +
611 #include "fec.h"
612
613 +#define USE_LEVEL_A_BANK 1
614 +#ifdef USE_LEVEL_A_BANK
615 +#define FEC_FLAG (GFP_COLOR)
616 +#else
617 +#define FEC_FLAG (0)
618 +#endif
619 +
620 static void set_multicast_list(struct net_device *ndev);
621 static void fec_enet_itr_coal_init(struct net_device *ndev);
622
623 @@ -1587,6 +1596,8 @@ fec_enet_interrupt(int irq, void *dev_id)
624 writel(int_events, fep->hwp + FEC_IEVENT);
625 fec_enet_collect_events(fep, int_events);
626
627 + enter_irq_mode();
628 +
629 if ((fep->work_tx || fep->work_rx) && fep->link) {
630 ret = IRQ_HANDLED;
631
632 @@ -1605,6 +1616,7 @@ fec_enet_interrupt(int irq, void *dev_id)
633 if (fep->ptp_clock)
634 fec_ptp_check_pps_event(fep);
635
636 + exit_irq_mode();
637 return ret;
638 }
639
640 @@ -2623,7 +2635,7 @@ static int fec_enet_alloc_queue(struct net_device *ndev)
641 struct fec_enet_priv_tx_q *txq;
642
643 for (i = 0; i < fep->num_tx_queues; i++) {
644 - txq = kzalloc(sizeof(*txq), GFP_KERNEL);
645 + txq = kzalloc(sizeof(*txq), GFP_KERNEL|FEC_FLAG);
646 if (!txq) {
647 ret = -ENOMEM;
648 goto alloc_failed;
649 @@ -2640,7 +2652,7 @@ static int fec_enet_alloc_queue(struct net_device *ndev)
650 txq->tso_hdrs = dma_alloc_coherent(NULL,
651 txq->tx_ring_size * TSO_HEADER_SIZE,
652 &txq->tso_hdrs_dma,
653 - GFP_KERNEL);
654 + GFP_KERNEL|FEC_FLAG);
655 if (!txq->tso_hdrs) {
656 ret = -ENOMEM;
657 goto alloc_failed;
658 @@ -2649,7 +2661,7 @@ static int fec_enet_alloc_queue(struct net_device *ndev)
659
660 for (i = 0; i < fep->num_rx_queues; i++) {
661 fep->rx_queue[i] = kzalloc(sizeof(*fep->rx_queue[i]),
662 - GFP_KERNEL);
663 + GFP_KERNEL|FEC_FLAG);
664 if (!fep->rx_queue[i]) {
665 ret = -ENOMEM;
666 goto alloc_failed;
667 @@ -2718,7 +2730,7 @@ fec_enet_alloc_txq_buffers(struct net_device *ndev, unsigned int queue)
668 txq = fep->tx_queue[queue];
669 bdp = txq->tx_bd_base;
670 for (i = 0; i < txq->tx_ring_size; i++) {
671 - txq->tx_bounce[i] = kmalloc(FEC_ENET_TX_FRSIZE, GFP_KERNEL);
672 + txq->tx_bounce[i] = kmalloc(FEC_ENET_TX_FRSIZE, GFP_KERNEL|FEC_FLAG);
673 if (!txq->tx_bounce[i])
674 goto err_alloc;
675
676 @@ -3032,7 +3044,7 @@ static int fec_enet_init(struct net_device *ndev)
677
678 /* Allocate memory for buffer descriptors. */
679 cbd_base = dma_alloc_coherent(NULL, bd_size, &bd_dma,
680 - GFP_KERNEL);
681 + GFP_KERNEL|FEC_FLAG);
682 if (!cbd_base) {
683 return -ENOMEM;
684 }
685 diff --git drivers/usb/core/buffer.c drivers/usb/core/buffer.c
686 index 506b969..b9af514 100644
687 --- drivers/usb/core/buffer.c
688 +++ drivers/usb/core/buffer.c
689 @@ -128,6 +128,7 @@ void *hcd_buffer_alloc(
690 if (size <= pool_max[i])
691 return dma_pool_alloc(hcd->pool[i], mem_flags, dma);
692 }
693 +
694 return dma_alloc_coherent(hcd->self.controller, size, dma, mem_flags);
695 }
696
697 diff --git include/linux/gfp.h include/linux/gfp.h
698 index 15928f0..92643b8 100644
699 --- include/linux/gfp.h
700 +++ include/linux/gfp.h
701 @@ -35,6 +35,7 @@ struct vm_area_struct;
702 #define ___GFP_NO_KSWAPD 0x400000u
703 #define ___GFP_OTHER_NODE 0x800000u
704 #define ___GFP_WRITE 0x1000000u
705 +#define ___GFP_COLOR 0x2000000u
706 /* If the above are modified, __GFP_BITS_SHIFT may need updating */
707
708 /*
709 @@ -94,6 +95,7 @@ struct vm_area_struct;
710 #define __GFP_NO_KSWAPD ((__force gfp_t)___GFP_NO_KSWAPD)
711 #define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */
712 #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) /* Allocator intends to dirty page */
713 +#define __GFP_COLOR ((__force gfp_t)___GFP_COLOR) /* Colored page request */
714
715 /*
716 * This may seem redundant, but it's a way of annotating false positives vs.
717 @@ -101,7 +103,7 @@ struct vm_area_struct;
718 */
719 #define __GFP_NOTRACK_FALSE_POSITIVE (__GFP_NOTRACK)
720
721 -#define __GFP_BITS_SHIFT 25 /* Room for N __GFP_FOO bits */
722 +#define __GFP_BITS_SHIFT 26 /* Room for N __GFP_FOO bits */
723 #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
724
725 /* This equals 0, but use constants in case they ever change */
726 @@ -127,7 +129,7 @@ struct vm_area_struct;
727 /* Control page allocator reclaim behavior */
728 #define GFP_RECLAIM_MASK (__GFP_WAIT|__GFP_HIGH|__GFP_IO|__GFP_FS|\
729 __GFP_NOWARN|__GFP_REPEAT|__GFP_NOFAIL|\
730 - __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC)
731 + __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|__GFP_COLOR)
732
733 /* Control slab gfp mask during early boot */
734 #define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_WAIT|__GFP_IO|__GFP_FS))
735 @@ -146,6 +148,9 @@ struct vm_area_struct;
736 /* 4GB DMA on some platforms */
737 #define GFP_DMA32 __GFP_DMA32
738
739 +/* Colored page requests */
740 +#define GFP_COLOR __GFP_COLOR
741 +
742 /* Convert GFP flags to their corresponding migrate type */
743 static inline int gfpflags_to_migratetype(const gfp_t gfp_flags)
744 {
745 diff --git include/linux/migrate.h include/linux/migrate.h
746 index cac1c09..b16047b 100644
747 --- include/linux/migrate.h
748 +++ include/linux/migrate.h
749 @@ -33,6 +33,8 @@ extern int migrate_page(struct address_space *,
750 struct page *, struct page *, enum migrate_mode);
751 extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
752 unsigned long private, enum migrate_mode mode, int reason);
753 +extern int replicate_pages(struct list_head *l, new_page_t new, free_page_t free,
754 + unsigned long private, enum migrate_mode mode, int reason);
755
756 extern int migrate_prep(void);
757 extern int migrate_prep_local(void);
758 @@ -50,7 +52,11 @@ static inline int migrate_pages(struct list_head *l, new_page_t new,
759 free_page_t free, unsigned long private, enum migrate_mode mode,
760 int reason)
761 { return -ENOSYS; }
762 -
763 +static inline int replicate_pages(struct list_head *l, new_page_t new,
764 + free_page_t free, unsigned long private, enum migrate_mode mode,
765 + int reason)
766 + { return -ENOSYS; }
767 +
768 static inline int migrate_prep(void) { return -ENOSYS; }
769 static inline int migrate_prep_local(void) { return -ENOSYS; }
770
771 diff --git include/linux/mmzone.h include/linux/mmzone.h
772 index 54d74f6..f010005 100644
773 --- include/linux/mmzone.h
774 +++ include/linux/mmzone.h
775 @@ -35,6 +35,19 @@
776 */
777 #define PAGE_ALLOC_COSTLY_ORDER 3
778
779 +/* For page coloring - This address decoding is used in imx6-sabresd
780 + * platform without bank interleaving .
781 + */
782 +#define BANK_MASK 0x38000000
783 +#define BANK_SHIFT 27
784 +#define CACHE_MASK 0x0000f000
785 +#define CACHE_SHIFT 12
786 +#define MAX_NUM_COLOR 16
787 +#define MAX_NUM_BANK 8
788 +//#define MAX_PARTITIONED_ORDER 3
789 +#define MAX_PARTITIONED_ORDER 11
790 +#define MAX_CONTIG_ORDER 3
791 +
792 enum {
793 MIGRATE_UNMOVABLE,
794 MIGRATE_RECLAIMABLE,
795 @@ -157,6 +170,7 @@ enum zone_stat_item {
796 WORKINGSET_NODERECLAIM,
797 NR_ANON_TRANSPARENT_HUGEPAGES,
798 NR_FREE_CMA_PAGES,
799 + NR_FREE_HC_PAGES,
800 NR_VM_ZONE_STAT_ITEMS };
801
802 /*
803 @@ -476,7 +490,8 @@ struct zone {
804 ZONE_PADDING(_pad1_)
805 /* free areas of different sizes */
806 struct free_area free_area[MAX_ORDER];
807 -
808 + struct free_area free_area_d[NR_CPUS][MAX_PARTITIONED_ORDER];
809 +
810 /* zone flags, see below */
811 unsigned long flags;
812
813 @@ -523,7 +538,9 @@ struct zone {
814 /* Set to true when the PG_migrate_skip bits should be cleared */
815 bool compact_blockskip_flush;
816 #endif
817 -
818 +
819 + struct list_head color_list[MAX_NUM_COLOR * MAX_NUM_BANK];
820 + DECLARE_BITMAP(color_map, MAX_NUM_COLOR*MAX_NUM_BANK);
821 ZONE_PADDING(_pad3_)
822 /* Zone statistics */
823 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
824 diff --git include/linux/rmap.h include/linux/rmap.h
825 index c89c53a..7c90e02 100644
826 --- include/linux/rmap.h
827 +++ include/linux/rmap.h
828 @@ -188,7 +188,8 @@ int page_referenced(struct page *, int is_locked,
829 #define TTU_ACTION(x) ((x) & TTU_ACTION_MASK)
830
831 int try_to_unmap(struct page *, enum ttu_flags flags);
832 -
833 +int try_to_unmap_one_only(struct page *page, struct vm_area_struct *vma,
834 + unsigned long address, void *arg);
835 /*
836 * Used by uprobes to replace a userspace page safely
837 */
838 diff --git include/linux/slab.h include/linux/slab.h
839 index ffd24c8..6064df0 100644
840 --- include/linux/slab.h
841 +++ include/linux/slab.h
842 @@ -87,6 +87,8 @@
843 # define SLAB_FAILSLAB 0x00000000UL
844 #endif
845
846 +#define SLAB_NO_MERGE 0x04000000UL /* Do not merge with existing slab */
847 +
848 /* The following flags affect the page allocator grouping pages by mobility */
849 #define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */
850 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */
851 @@ -417,6 +419,12 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
852 */
853 static __always_inline void *kmalloc(size_t size, gfp_t flags)
854 {
855 +
856 +#ifdef CONFIG_SCHED_DEBUG_TRACE
857 + if (flags&GFP_COLOR)
858 + printk(KERN_INFO "kmalloc() is called with GFP_COLOR\n");
859 +#endif
860 +
861 if (__builtin_constant_p(size)) {
862 if (size > KMALLOC_MAX_CACHE_SIZE)
863 return kmalloc_large(size, flags);
864 diff --git include/linux/slub_def.h include/linux/slub_def.h
865 index 3388511..9400aa1 100644
866 --- include/linux/slub_def.h
867 +++ include/linux/slub_def.h
868 @@ -98,6 +98,8 @@ struct kmem_cache {
869 */
870 int remote_node_defrag_ratio;
871 #endif
872 + /* cpu id for higher-criticality slabs */
873 + int cpu_id;
874 struct kmem_cache_node *node[MAX_NUMNODES];
875 };
876
877 diff --git include/linux/vm_event_item.h include/linux/vm_event_item.h
878 index 9246d32..3f5a9da 100644
879 --- include/linux/vm_event_item.h
880 +++ include/linux/vm_event_item.h
881 @@ -23,7 +23,7 @@
882
883 enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
884 FOR_ALL_ZONES(PGALLOC),
885 - PGFREE, PGACTIVATE, PGDEACTIVATE,
886 + PGFREE, PGFREE_HC, PGACTIVATE, PGDEACTIVATE,
887 PGFAULT, PGMAJFAULT,
888 FOR_ALL_ZONES(PGREFILL),
889 FOR_ALL_ZONES(PGSTEAL_KSWAPD),
890 diff --git include/linux/vmalloc.h include/linux/vmalloc.h
891 index 0ec5983..2323fbe 100644
892 --- include/linux/vmalloc.h
893 +++ include/linux/vmalloc.h
894 @@ -67,8 +67,11 @@ static inline void vmalloc_init(void)
895 #endif
896
897 extern void *vmalloc(unsigned long size);
898 +extern void *vmalloc_color(unsigned long size);
899 extern void *vzalloc(unsigned long size);
900 extern void *vmalloc_user(unsigned long size);
901 +extern void *vmalloc_color_user(unsigned long size);
902 +extern void *vmalloc_color_user_cpu1(unsigned long size);
903 extern void *vmalloc_node(unsigned long size, int node);
904 extern void *vzalloc_node(unsigned long size, int node);
905 extern void *vmalloc_exec(unsigned long size);
906 diff --git include/linux/vmstat.h include/linux/vmstat.h
907 index 82e7db7..b6410f7 100644
908 --- include/linux/vmstat.h
909 +++ include/linux/vmstat.h
910 @@ -278,9 +278,12 @@ static inline void drain_zonestat(struct zone *zone,
911 #endif /* CONFIG_SMP */
912
913 static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
914 - int migratetype)
915 + int migratetype, int part_no)
916 {
917 - __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
918 + if (part_no == NR_CPUS)
919 + __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
920 + else
921 + __mod_zone_page_state(zone, NR_FREE_HC_PAGES, nr_pages);
922 if (is_migrate_cma(migratetype))
923 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
924 }
925 diff --git include/litmus/budget.h include/litmus/budget.h
926 index bd2d5c9..60eb814 100644
927 --- include/litmus/budget.h
928 +++ include/litmus/budget.h
929 @@ -33,4 +33,6 @@ static inline int requeue_preempted_job(struct task_struct* t)
930 (!budget_exhausted(t) || !budget_enforced(t));
931 }
932
933 +void litmus_current_budget(lt_t *used_so_far, lt_t *remaining);
934 +
935 #endif
936 diff --git include/litmus/cache_proc.h include/litmus/cache_proc.h
937 new file mode 100644
938 index 0000000..e9440de
939 --- /dev/null
940 +++ include/litmus/cache_proc.h
941 @@ -0,0 +1,17 @@
942 +#ifndef LITMUS_CACHE_PROC_H
943 +#define LITMUS_CACHE_PROC_H
944 +
945 +#ifdef __KERNEL__
946 +
947 +void litmus_setup_lockdown(void __iomem*, u32);
948 +void enter_irq_mode(void);
949 +void exit_irq_mode(void);
950 +void flush_cache(int all);
951 +void lock_cache(int cpu, u32 val);
952 +
953 +extern struct page *new_alloc_page_color(unsigned long color);
954 +
955 +#endif
956 +
957 +#endif
958 +
959 diff --git include/litmus/litmus.h include/litmus/litmus.h
960 index a6eb534..441210c 100644
961 --- include/litmus/litmus.h
962 +++ include/litmus/litmus.h
963 @@ -113,6 +113,13 @@ static inline lt_t litmus_clock(void)
964 ((current)->state == TASK_RUNNING || \
965 preempt_count() & PREEMPT_ACTIVE)
966
967 +#define is_running(t) \
968 + ((t)->state == TASK_RUNNING || \
969 + task_thread_info(t)->preempt_count & PREEMPT_ACTIVE)
970 +
971 +#define is_blocked(t) \
972 + (!is_running(t))
973 +
974 #define is_released(t, now) \
975 (lt_before_eq(get_release(t), now))
976 #define is_tardy(t, now) \
977 diff --git include/litmus/mc2_common.h include/litmus/mc2_common.h
978 new file mode 100644
979 index 0000000..e3c0af2
980 --- /dev/null
981 +++ include/litmus/mc2_common.h
982 @@ -0,0 +1,31 @@
983 +/*
984 + * MC^2 common data structures
985 + */
986 +
987 +#ifndef __UNC_MC2_COMMON_H__
988 +#define __UNC_MC2_COMMON_H__
989 +
990 +enum crit_level {
991 + CRIT_LEVEL_A = 0,
992 + CRIT_LEVEL_B = 1,
993 + CRIT_LEVEL_C = 2,
994 + NUM_CRIT_LEVELS = 3,
995 +};
996 +
997 +struct mc2_task {
998 + enum crit_level crit;
999 + unsigned int res_id;
1000 +};
1001 +
1002 +#ifdef __KERNEL__
1003 +
1004 +#include <litmus/reservation.h>
1005 +
1006 +#define tsk_mc2_data(t) (tsk_rt(t)->mc2_data)
1007 +
1008 +long mc2_task_client_init(struct task_client *tc, struct mc2_task *mc2_param, struct task_struct *tsk,
1009 + struct reservation *res);
1010 +
1011 +#endif /* __KERNEL__ */
1012 +
1013 +#endif
1014 \ No newline at end of file
1015 diff --git include/litmus/page_dev.h include/litmus/page_dev.h
1016 new file mode 100644
1017 index 0000000..c6874ac
1018 --- /dev/null
1019 +++ include/litmus/page_dev.h
1020 @@ -0,0 +1,28 @@
1021 +/*
1022 + * page_dev.h - Implementation of the page coloring for cache and bank partition.
1023 + * Author: Namhoon Kim (namhoonk@cs.unc.edu)
1024 + */
1025 +
1026 +#ifndef _LITMUS_PAGE_DEV_H
1027 +#define _LITMUS_PAGE_DEV_H
1028 +
1029 +#include <linux/init.h>
1030 +#include <linux/types.h>
1031 +#include <linux/kernel.h>
1032 +#include <linux/module.h>
1033 +#include <linux/sysctl.h>
1034 +#include <linux/slab.h>
1035 +#include <linux/io.h>
1036 +#include <linux/mutex.h>
1037 +
1038 +#include <litmus/sched_trace.h>
1039 +#include <litmus/litmus.h>
1040 +
1041 +int llc_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos);
1042 +int dram_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos);
1043 +int bank_to_partition(unsigned int bank);
1044 +int get_area_index(int cpu);
1045 +int is_in_correct_bank(struct page* page, int cpu);
1046 +int is_in_llc_partition(struct page* page, int cpu);
1047 +
1048 +#endif /* _LITMUS_PAGE_DEV_H */
1049 \ No newline at end of file
1050 diff --git include/litmus/polling_reservations.h include/litmus/polling_reservations.h
1051 new file mode 100644
1052 index 0000000..66c9b1e
1053 --- /dev/null
1054 +++ include/litmus/polling_reservations.h
1055 @@ -0,0 +1,36 @@
1056 +#ifndef LITMUS_POLLING_RESERVATIONS_H
1057 +#define LITMUS_POLLING_RESERVATIONS_H
1058 +
1059 +#include <litmus/reservation.h>
1060 +
1061 +struct polling_reservation {
1062 + /* extend basic reservation */
1063 + struct reservation res;
1064 +
1065 + lt_t max_budget;
1066 + lt_t period;
1067 + lt_t deadline;
1068 + lt_t offset;
1069 +};
1070 +
1071 +void polling_reservation_init(struct polling_reservation *pres, int use_edf_prio,
1072 + int use_periodic_polling, lt_t budget, lt_t period, lt_t deadline, lt_t offset);
1073 +
1074 +struct table_driven_reservation {
1075 + /* extend basic reservation */
1076 + struct reservation res;
1077 +
1078 + lt_t major_cycle;
1079 + unsigned int next_interval;
1080 + unsigned int num_intervals;
1081 + struct lt_interval *intervals;
1082 +
1083 + /* info about current scheduling slot */
1084 + struct lt_interval cur_interval;
1085 + lt_t major_cycle_start;
1086 +};
1087 +
1088 +void table_driven_reservation_init(struct table_driven_reservation *tdres,
1089 + lt_t major_cycle, struct lt_interval *intervals, unsigned int num_intervals);
1090 +
1091 +#endif
1092 diff --git include/litmus/replicate_lib.h include/litmus/replicate_lib.h
1093 new file mode 100644
1094 index 0000000..13e0c3a
1095 --- /dev/null
1096 +++ include/litmus/replicate_lib.h
1097 @@ -0,0 +1,19 @@
1098 +#ifndef LITMUS_REPLICATE_LIB_H
1099 +#define LITMUS_REPLICATE_LIB_H
1100 +
1101 +#include <linux/list.h>
1102 +#include <linux/mm_types.h>
1103 +#include <linux/mm_inline.h>
1104 +
1105 +/* Data structure for the "master" list */
1106 +struct shared_lib_page {
1107 + struct page *master_page;
1108 + struct page *r_page[NR_CPUS+1];
1109 + unsigned long int master_pfn;
1110 + unsigned long int r_pfn[NR_CPUS+1];
1111 + struct list_head list;
1112 +};
1113 +
1114 +extern struct list_head shared_lib_pages;
1115 +
1116 +#endif
1117 diff --git include/litmus/reservation.h include/litmus/reservation.h
1118 new file mode 100644
1119 index 0000000..7e022b3
1120 --- /dev/null
1121 +++ include/litmus/reservation.h
1122 @@ -0,0 +1,256 @@
1123 +#ifndef LITMUS_RESERVATION_H
1124 +#define LITMUS_RESERVATION_H
1125 +
1126 +#include <linux/list.h>
1127 +#include <linux/hrtimer.h>
1128 +
1129 +struct reservation_client;
1130 +struct reservation_environment;
1131 +struct reservation;
1132 +
1133 +typedef enum {
1134 + /* reservation has no clients, is not consuming budget */
1135 + RESERVATION_INACTIVE = 0,
1136 +
1137 + /* reservation has clients, consumes budget when scheduled */
1138 + RESERVATION_ACTIVE,
1139 +
1140 + /* reservation has no clients, but may be consuming budget */
1141 + RESERVATION_ACTIVE_IDLE,
1142 +
1143 + /* Reservation has no budget and waits for
1144 + * replenishment. May or may not have clients. */
1145 + RESERVATION_DEPLETED,
1146 +} reservation_state_t;
1147 +
1148 +
1149 +/* ************************************************************************** */
1150 +
1151 +/* Select which task to dispatch. If NULL is returned, it means there is nothing
1152 + * to schedule right now and background work can be scheduled. */
1153 +typedef struct task_struct * (*dispatch_t) (
1154 + struct reservation_client *client
1155 +);
1156 +
1157 +/* Something that can be managed in a reservation and that can yield
1158 + * a process for dispatching. Contains a pointer to the reservation
1159 + * to which it "belongs". */
1160 +struct reservation_client {
1161 + struct list_head list;
1162 + struct reservation* reservation;
1163 + dispatch_t dispatch;
1164 +};
1165 +
1166 +
1167 +/* ************************************************************************** */
1168 +
1169 +/* Called by reservations to request state change. */
1170 +typedef void (*reservation_change_state_t) (
1171 + struct reservation_environment* env,
1172 + struct reservation *res,
1173 + reservation_state_t new_state
1174 +);
1175 +
1176 +/* The framework within wich reservations operate. */
1177 +struct reservation_environment {
1178 + lt_t time_zero;
1179 + lt_t current_time;
1180 +
1181 + /* services invoked by reservations */
1182 + reservation_change_state_t change_state;
1183 +};
1184 +
1185 +
1186 +/* ************************************************************************** */
1187 +
1188 +/* A new client is added or an existing client resumes. */
1189 +typedef void (*client_arrives_t) (
1190 + struct reservation *reservation,
1191 + struct reservation_client *client
1192 +);
1193 +
1194 +/* A client suspends or terminates. */
1195 +typedef void (*client_departs_t) (
1196 + struct reservation *reservation,
1197 + struct reservation_client *client,
1198 + int did_signal_job_completion
1199 +);
1200 +
1201 +/* A previously requested replenishment has occurred. */
1202 +typedef void (*on_replenishment_timer_t) (
1203 + struct reservation *reservation
1204 +);
1205 +
1206 +/* Update the reservation's budget to reflect execution or idling. */
1207 +typedef void (*drain_budget_t) (
1208 + struct reservation *reservation,
1209 + lt_t how_much
1210 +);
1211 +
1212 +/* Select a ready task from one of the clients for scheduling. */
1213 +typedef struct task_struct* (*dispatch_client_t) (
1214 + struct reservation *reservation,
1215 + lt_t *time_slice /* May be used to force rescheduling after
1216 + some amount of time. 0 => no limit */
1217 +);
1218 +
1219 +
1220 +struct reservation_ops {
1221 + dispatch_client_t dispatch_client;
1222 +
1223 + client_arrives_t client_arrives;
1224 + client_departs_t client_departs;
1225 +
1226 + on_replenishment_timer_t replenish;
1227 + drain_budget_t drain_budget;
1228 +};
1229 +
1230 +struct reservation {
1231 + /* used to queue in environment */
1232 + struct list_head list;
1233 +
1234 + reservation_state_t state;
1235 + unsigned int id;
1236 +
1237 + /* exact meaning defined by impl. */
1238 + lt_t priority;
1239 + lt_t cur_budget;
1240 + lt_t next_replenishment;
1241 +
1242 + /* budget stats */
1243 + lt_t budget_consumed; /* how much budget consumed in this allocation cycle? */
1244 + lt_t budget_consumed_total;
1245 +
1246 + /* interaction with framework */
1247 + struct reservation_environment *env;
1248 + struct reservation_ops *ops;
1249 +
1250 + struct list_head clients;
1251 +
1252 + /* for global env. */
1253 + int scheduled_on;
1254 + int event_added;
1255 + /* for blocked by ghost. Do not charge budget when ACTIVE */
1256 + int blocked_by_ghost;
1257 + /* ghost_job. If it is clear, do not charge budget when ACTIVE_IDLE */
1258 + int is_ghost;
1259 +};
1260 +
1261 +void reservation_init(struct reservation *res);
1262 +
1263 +/* Default implementations */
1264 +
1265 +/* simply select the first client in the list, set *for_at_most to zero */
1266 +struct task_struct* default_dispatch_client(
1267 + struct reservation *res,
1268 + lt_t *for_at_most
1269 +);
1270 +
1271 +/* "connector" reservation client to hook up tasks with reservations */
1272 +struct task_client {
1273 + struct reservation_client client;
1274 + struct task_struct *task;
1275 +};
1276 +
1277 +void task_client_init(struct task_client *tc, struct task_struct *task,
1278 + struct reservation *reservation);
1279 +
1280 +#define SUP_RESCHEDULE_NOW (0)
1281 +#define SUP_NO_SCHEDULER_UPDATE (ULLONG_MAX)
1282 +
1283 +/* A simple uniprocessor (SUP) flat (i.e., non-hierarchical) reservation
1284 + * environment.
1285 + */
1286 +struct sup_reservation_environment {
1287 + struct reservation_environment env;
1288 +
1289 + /* ordered by priority */
1290 + struct list_head active_reservations;
1291 +
1292 + /* ordered by next_replenishment */
1293 + struct list_head depleted_reservations;
1294 +
1295 + /* unordered */
1296 + struct list_head inactive_reservations;
1297 +
1298 + /* - SUP_RESCHEDULE_NOW means call sup_dispatch() now
1299 + * - SUP_NO_SCHEDULER_UPDATE means nothing to do
1300 + * any other value means program a timer for the given time
1301 + */
1302 + lt_t next_scheduler_update;
1303 + /* set to true if a call to sup_dispatch() is imminent */
1304 + bool will_schedule;
1305 +};
1306 +
1307 +/* Contract:
1308 + * - before calling into sup_ code, or any reservation methods,
1309 + * update the time with sup_update_time(); and
1310 + * - after calling into sup_ code, or any reservation methods,
1311 + * check next_scheduler_update and program timer or trigger
1312 + * scheduler invocation accordingly.
1313 + */
1314 +
1315 +void sup_init(struct sup_reservation_environment* sup_env);
1316 +void sup_add_new_reservation(struct sup_reservation_environment* sup_env,
1317 + struct reservation* new_res);
1318 +void sup_scheduler_update_after(struct sup_reservation_environment* sup_env,
1319 + lt_t timeout);
1320 +void sup_update_time(struct sup_reservation_environment* sup_env, lt_t now);
1321 +struct task_struct* sup_dispatch(struct sup_reservation_environment* sup_env);
1322 +
1323 +struct reservation* sup_find_by_id(struct sup_reservation_environment* sup_env,
1324 + unsigned int id);
1325 +
1326 +/* A global multiprocessor reservation environment. */
1327 +
1328 +typedef enum {
1329 + EVENT_REPLENISH = 0,
1330 + EVENT_DRAIN,
1331 + EVENT_OTHERS,
1332 +} event_type_t;
1333 +
1334 +
1335 +struct next_timer_event {
1336 + lt_t next_update;
1337 + int timer_armed_on;
1338 + unsigned int id;
1339 + event_type_t type;
1340 + struct list_head list;
1341 +};
1342 +
1343 +struct gmp_reservation_environment {
1344 + raw_spinlock_t lock;
1345 + struct reservation_environment env;
1346 +
1347 + /* ordered by priority */
1348 + struct list_head active_reservations;
1349 +
1350 + /* ordered by next_replenishment */
1351 + struct list_head depleted_reservations;
1352 +
1353 + /* unordered */
1354 + struct list_head inactive_reservations;
1355 +
1356 + /* timer event ordered by next_update */
1357 + struct list_head next_events;
1358 +
1359 + /* (schedule_now == true) means call gmp_dispatch() now */
1360 + int schedule_now;
1361 + /* set to true if a call to gmp_dispatch() is imminent */
1362 + bool will_schedule;
1363 +};
1364 +
1365 +void gmp_init(struct gmp_reservation_environment* gmp_env);
1366 +void gmp_add_new_reservation(struct gmp_reservation_environment* gmp_env,
1367 + struct reservation* new_res);
1368 +void gmp_add_event_after(struct gmp_reservation_environment* gmp_env,
1369 + lt_t timeout, unsigned int id, event_type_t type);
1370 +void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now);
1371 +int gmp_update_time(struct gmp_reservation_environment* gmp_env, lt_t now);
1372 +struct task_struct* gmp_dispatch(struct gmp_reservation_environment* gmp_env);
1373 +struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env, unsigned int id);
1374 +struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env, lt_t when);
1375 +struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env,
1376 + unsigned int id);
1377 +
1378 +#endif
1379 diff --git include/litmus/rt_param.h include/litmus/rt_param.h
1380 index 7b9a909..56de045 100644
1381 --- include/litmus/rt_param.h
1382 +++ include/litmus/rt_param.h
1383 @@ -51,6 +51,16 @@ typedef enum {
1384 TASK_EARLY
1385 } release_policy_t;
1386
1387 +#ifdef CONFIG_PGMRT_SUPPORT
1388 +typedef enum {
1389 + PGM_NOT_A_NODE,
1390 + PGM_SRC,
1391 + PGM_SINK,
1392 + PGM_SRC_SINK,
1393 + PGM_INTERNAL
1394 +} pgm_node_type_t;
1395 +#endif
1396 +
1397 /* We use the common priority interpretation "lower index == higher priority",
1398 * which is commonly used in fixed-priority schedulability analysis papers.
1399 * So, a numerically lower priority value implies higher scheduling priority,
1400 @@ -62,6 +72,7 @@ typedef enum {
1401 #define LITMUS_MAX_PRIORITY 512
1402 #define LITMUS_HIGHEST_PRIORITY 1
1403 #define LITMUS_LOWEST_PRIORITY (LITMUS_MAX_PRIORITY - 1)
1404 +#define LITMUS_NO_PRIORITY UINT_MAX
1405
1406 /* Provide generic comparison macros for userspace,
1407 * in case that we change this later. */
1408 @@ -71,6 +82,46 @@ typedef enum {
1409 ((p) >= LITMUS_HIGHEST_PRIORITY && \
1410 (p) <= LITMUS_LOWEST_PRIORITY)
1411
1412 +/* reservation support */
1413 +
1414 +typedef enum {
1415 + PERIODIC_POLLING,
1416 + SPORADIC_POLLING,
1417 + TABLE_DRIVEN,
1418 +} reservation_type_t;
1419 +
1420 +struct lt_interval {
1421 + lt_t start;
1422 + lt_t end;
1423 +};
1424 +
1425 +#ifndef __KERNEL__
1426 +#define __user
1427 +#endif
1428 +
1429 +struct reservation_config {
1430 + unsigned int id;
1431 + lt_t priority;
1432 + int cpu;
1433 +
1434 + union {
1435 + struct {
1436 + lt_t period;
1437 + lt_t budget;
1438 + lt_t relative_deadline;
1439 + lt_t offset;
1440 + } polling_params;
1441 +
1442 + struct {
1443 + lt_t major_cycle_length;
1444 + unsigned int num_intervals;
1445 + struct lt_interval __user *intervals;
1446 + } table_driven_params;
1447 + };
1448 +};
1449 +
1450 +/* regular sporadic task support */
1451 +
1452 struct rt_task {
1453 lt_t exec_cost;
1454 lt_t period;
1455 @@ -81,6 +132,10 @@ struct rt_task {
1456 task_class_t cls;
1457 budget_policy_t budget_policy; /* ignored by pfair */
1458 release_policy_t release_policy;
1459 +#ifdef CONFIG_PGMRT_SUPPORT
1460 + pgm_node_type_t pgm_type;
1461 + lt_t pgm_expected_etoe;
1462 +#endif
1463 };
1464
1465 union np_flag {
1466 @@ -121,6 +176,13 @@ struct control_page {
1467 uint64_t irq_syscall_start; /* Snapshot of irq_count when the syscall
1468 * started. */
1469
1470 +#ifdef CONFIG_PGMRT_SUPPORT
1471 + /* Flags from userspace signifying PGM wait states. */
1472 + volatile uint32_t pgm_waiting; /* waiting for tokens */
1473 + volatile uint32_t pgm_sending; /* sending tokens */
1474 + volatile uint32_t pgm_satisfied; /* done waiting/sending */
1475 +#endif
1476 +
1477 /* to be extended */
1478 };
1479
1480 @@ -165,6 +227,7 @@ struct rt_job {
1481 };
1482
1483 struct pfair_param;
1484 +struct mc2_task;
1485
1486 /* RT task parameters for scheduling extensions
1487 * These parameters are inherited during clone and therefore must
1488 @@ -246,7 +309,10 @@ struct rt_param {
1489 volatile int linked_on;
1490
1491 /* PFAIR/PD^2 state. Allocated on demand. */
1492 - struct pfair_param* pfair;
1493 + union {
1494 + void *plugin_state;
1495 + struct pfair_param *pfair;
1496 + };
1497
1498 /* Fields saved before BE->RT transition.
1499 */
1500 @@ -275,6 +341,10 @@ struct rt_param {
1501
1502 /* Pointer to the page shared between userspace and kernel. */
1503 struct control_page * ctrl_page;
1504 +
1505 + /* Mixed-criticality specific data */
1506 + struct mc2_task* mc2_data;
1507 + unsigned long addr_ctrl_page;
1508 };
1509
1510 #endif
1511 diff --git include/litmus/sched_plugin.h include/litmus/sched_plugin.h
1512 index 0ccccd6..4c8aaa6 100644
1513 --- include/litmus/sched_plugin.h
1514 +++ include/litmus/sched_plugin.h
1515 @@ -77,6 +77,17 @@ typedef long (*wait_for_release_at_t)(lt_t release_time);
1516 /* Informs the plugin when a synchronous release takes place. */
1517 typedef void (*synchronous_release_at_t)(lt_t time_zero);
1518
1519 +/* How much budget has the current task consumed so far, and how much
1520 + * has it left? The default implementation ties into the per-task
1521 + * budget enforcement code. Plugins can override this to report
1522 + * reservation-specific values. */
1523 +typedef void (*current_budget_t)(lt_t *used_so_far, lt_t *remaining);
1524 +
1525 +/* Reservation creation/removal backends. Meaning of reservation_type and
1526 + * reservation_id are entirely plugin-specific. */
1527 +typedef long (*reservation_create_t)(int reservation_type, void* __user config);
1528 +typedef long (*reservation_destroy_t)(unsigned int reservation_id, int cpu);
1529 +
1530 /************************ misc routines ***********************/
1531
1532
1533 @@ -109,6 +120,12 @@ struct sched_plugin {
1534 task_exit_t task_exit;
1535 task_cleanup_t task_cleanup;
1536
1537 + current_budget_t current_budget;
1538 +
1539 + /* Reservation support */
1540 + reservation_create_t reservation_create;
1541 + reservation_destroy_t reservation_destroy;
1542 +
1543 #ifdef CONFIG_LITMUS_LOCKING
1544 /* locking protocols */
1545 allocate_lock_t allocate_lock;
1546 diff --git include/litmus/trace.h include/litmus/trace.h
1547 index 6017872..7d36a11 100644
1548 --- include/litmus/trace.h
1549 +++ include/litmus/trace.h
1550 @@ -118,6 +118,9 @@ feather_callback void save_cpu_task_latency(unsigned long event, unsigned long w
1551 #define TS_TICK_START(t) CPU_TTIMESTAMP(110, t)
1552 #define TS_TICK_END(t) CPU_TTIMESTAMP(111, t)
1553
1554 +#define TS_RELEASE_C_START CPU_DTIMESTAMP(108, TSK_RT)
1555 +#define TS_RELEASE_C_END CPU_DTIMESTAMP(109, TSK_RT)
1556 +
1557 #define TS_QUANTUM_BOUNDARY_START CPU_TIMESTAMP_CUR(112)
1558 #define TS_QUANTUM_BOUNDARY_END CPU_TIMESTAMP_CUR(113)
1559
1560 @@ -137,6 +140,27 @@ feather_callback void save_cpu_task_latency(unsigned long event, unsigned long w
1561 #define TS_SEND_RESCHED_START(c) MSG_TIMESTAMP_SENT(190, c)
1562 #define TS_SEND_RESCHED_END MSG_TIMESTAMP_RECEIVED(191)
1563
1564 -#define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when))
1565 +#define TS_ISR_START CPU_TIMESTAMP_CUR(192)
1566 +#define TS_ISR_END CPU_TIMESTAMP_CUR(193)
1567 +
1568 +/* For RTAS2018 */
1569 +#define TS_NET_RX_HARDIRQ_START CPU_TIMESTAMP_CUR(194)
1570 +#define TS_NET_RX_HARDIRQ_END CPU_TIMESTAMP_CUR(195)
1571 +
1572 +#define TS_NET_RX_SOFTIRQ_START CPU_TIMESTAMP_CUR(196)
1573 +#define TS_NET_RX_SOFTIRQ_END CPU_TIMESTAMP_CUR(197)
1574 +
1575 +#define TS_UVC_IRQ_START CPU_TIMESTAMP_CUR(198)
1576 +#define TS_UVC_IRQ_END CPU_TIMESTAMP_CUR(199)
1577 +
1578 +#define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when))
1579 +#define TS_RELEASE_LATENCY_A(when) CPU_LTIMESTAMP(209, &(when))
1580 +#define TS_RELEASE_LATENCY_B(when) CPU_LTIMESTAMP(210, &(when))
1581 +#define TS_RELEASE_LATENCY_C(when) CPU_LTIMESTAMP(211, &(when))
1582 +
1583 +#define TS_SCHED_A_START CPU_DTIMESTAMP(212, TSK_UNKNOWN)
1584 +#define TS_SCHED_A_END(t) CPU_TTIMESTAMP(213, t)
1585 +#define TS_SCHED_C_START CPU_DTIMESTAMP(214, TSK_UNKNOWN)
1586 +#define TS_SCHED_C_END(t) CPU_TTIMESTAMP(215, t)
1587
1588 #endif /* !_SYS_TRACE_H_ */
1589 diff --git include/litmus/unistd_32.h include/litmus/unistd_32.h
1590 index 94264c2..86bbbb8d 100644
1591 --- include/litmus/unistd_32.h
1592 +++ include/litmus/unistd_32.h
1593 @@ -17,5 +17,13 @@
1594 #define __NR_wait_for_ts_release __LSC(9)
1595 #define __NR_release_ts __LSC(10)
1596 #define __NR_null_call __LSC(11)
1597 +#define __NR_get_current_budget __LSC(12)
1598 +#define __NR_reservation_create __LSC(13)
1599 +#define __NR_reservation_destroy __LSC(14)
1600 +#define __NR_set_mc2_task_param __LSC(15)
1601 +#define __NR_set_page_color __LSC(16)
1602 +#define __NR_test_call __LSC(17)
1603 +#define __NR_run_test __LSC(18)
1604 +#define __NR_lock_buffer __LSC(19)
1605
1606 -#define NR_litmus_syscalls 12
1607 +#define NR_litmus_syscalls 20
1608 diff --git include/litmus/unistd_64.h include/litmus/unistd_64.h
1609 index d5ced0d..4b96e7c 100644
1610 --- include/litmus/unistd_64.h
1611 +++ include/litmus/unistd_64.h
1612 @@ -29,5 +29,22 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release)
1613 __SYSCALL(__NR_release_ts, sys_release_ts)
1614 #define __NR_null_call __LSC(11)
1615 __SYSCALL(__NR_null_call, sys_null_call)
1616 +#define __NR_get_current_budget __LSC(12)
1617 +__SYSCALL(__NR_get_current_budget, sys_get_current_budget)
1618 +#define __NR_reservation_create __LSC(13)
1619 +__SYSCALL(__NR_reservation_create, sys_reservation_create)
1620 +#define __NR_reservation_destroy __LSC(14)
1621 +__SYSCALL(__NR_reservation_destroy, sys_reservation_destroy)
1622 +#define __NR_set_mc2_task_param __LSC(15)
1623 +__SYSCALL(__NR_set_mc2_task_param, sys_set_mc2_task_param)
1624 +#define __NR_set_page_color __LSC(16)
1625 +__SYSCALL(__NR_set_page_color, sys_set_page_color)
1626 +#define __NR_test_call __LSC(17)
1627 +__SYSCALL(__NR_test_call, sys_test_call)
1628 +#define __NR_run_test __LSC(18)
1629 +__SYSCALL(__NR_run_test, sys_run_test)
1630 +#define __NR_lock_buffer __LSC(19)
1631 +__SYACALL(__NR_lock_buffer, sys_lock_buffer)
1632
1633 -#define NR_litmus_syscalls 12
1634 +
1635 +#define NR_litmus_syscalls 20
1636 diff --git kernel/irq/handle.c kernel/irq/handle.c
1637 index 6354802..81df1b2 100644
1638 --- kernel/irq/handle.c
1639 +++ kernel/irq/handle.c
1640 @@ -18,6 +18,9 @@
1641
1642 #include <trace/events/irq.h>
1643
1644 +#include <litmus/litmus.h>
1645 +#include <litmus/trace.h>
1646 +
1647 #include "internals.h"
1648
1649 /**
1650 @@ -138,11 +141,9 @@ handle_irq_event_percpu(struct irq_desc *desc, struct irqaction *action)
1651
1652 do {
1653 irqreturn_t res;
1654 -
1655 trace_irq_handler_entry(irq, action);
1656 res = action->handler(irq, action->dev_id);
1657 trace_irq_handler_exit(irq, action, res);
1658 -
1659 if (WARN_ONCE(!irqs_disabled(),"irq %u handler %pF enabled interrupts\n",
1660 irq, action->handler))
1661 local_irq_disable();
1662 diff --git kernel/sched/litmus.c kernel/sched/litmus.c
1663 index 9d58690..cd36358 100644
1664 --- kernel/sched/litmus.c
1665 +++ kernel/sched/litmus.c
1666 @@ -20,8 +20,9 @@ static void update_time_litmus(struct rq *rq, struct task_struct *p)
1667 /* task counter */
1668 p->se.sum_exec_runtime += delta;
1669 if (delta) {
1670 - TRACE_TASK(p, "charged %llu exec time (total:%llu, rem:%llu)\n",
1671 - delta, p->rt_param.job_params.exec_time, budget_remaining(p));
1672 + //TRACE_TASK(p, "charged %llu exec time (total:%llu, rem:%llu)\n",
1673 + //delta, p->rt_param.job_params.exec_time, budget_remaining(p));
1674 + ;
1675 }
1676 /* sched_clock() */
1677 p->se.exec_start = rq->clock;
1678 diff --git kernel/softirq.c kernel/softirq.c
1679 index 99fe8b8..161450e 100644
1680 --- kernel/softirq.c
1681 +++ kernel/softirq.c
1682 @@ -27,6 +27,9 @@
1683 #include <linux/tick.h>
1684 #include <linux/irq.h>
1685
1686 +/* for measuring NET_RX bottom half */
1687 +#include <litmus/trace.h>
1688 +
1689 #define CREATE_TRACE_POINTS
1690 #include <trace/events/irq.h>
1691
1692 diff --git litmus/Makefile litmus/Makefile
1693 index 7970cd5..ccd532d 100644
1694 --- litmus/Makefile
1695 +++ litmus/Makefile
1696 @@ -11,6 +11,7 @@ obj-y = sched_plugin.o litmus.o \
1697 sync.o \
1698 rt_domain.o \
1699 edf_common.o \
1700 + mc2_common.o \
1701 fp_common.o \
1702 fdso.o \
1703 locking.o \
1704 @@ -19,13 +20,21 @@ obj-y = sched_plugin.o litmus.o \
1705 binheap.o \
1706 ctrldev.o \
1707 uncachedev.o \
1708 + reservation.o \
1709 + polling_reservations.o \
1710 sched_gsn_edf.o \
1711 sched_psn_edf.o \
1712 - sched_pfp.o
1713 + sched_pfp.o \
1714 + sched_mc2.o \
1715 + bank_proc.o \
1716 + color_shm.o \
1717 + replicate_lib.o \
1718 + cache_proc.o \
1719 + page_dev.o \
1720 + fakedev0.o
1721
1722 obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o
1723 obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o
1724 -
1725 obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o
1726 obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o
1727 obj-$(CONFIG_SCHED_DEBUG_TRACE) += sched_trace.o
1728 diff --git litmus/bank_proc.c litmus/bank_proc.c
1729 new file mode 100644
1730 index 0000000..353d38d
1731 --- /dev/null
1732 +++ litmus/bank_proc.c
1733 @@ -0,0 +1,773 @@
1734 +/*
1735 + * bank_proc.c -- Implementation of the page coloring for cache and bank partition.
1736 + * The file will keep a pool of colored pages. Users can require pages with
1737 + * specific color or bank number.
1738 + * Part of the code is modified from Jonathan Herman's code
1739 + */
1740 +#include <linux/init.h>
1741 +#include <linux/types.h>
1742 +#include <linux/kernel.h>
1743 +#include <linux/module.h>
1744 +#include <linux/sysctl.h>
1745 +#include <linux/slab.h>
1746 +#include <linux/io.h>
1747 +#include <linux/mutex.h>
1748 +#include <linux/mm.h>
1749 +#include <linux/random.h>
1750 +
1751 +#include <litmus/litmus_proc.h>
1752 +#include <litmus/sched_trace.h>
1753 +#include <litmus/litmus.h>
1754 +
1755 +#define LITMUS_LOCKDEP_NAME_MAX_LEN 50
1756 +
1757 +// This Address Decoding is used in imx6-sabredsd platform
1758 +#define BANK_MASK 0x38000000
1759 +#define BANK_SHIFT 27
1760 +#define CACHE_MASK 0x0000f000
1761 +#define CACHE_SHIFT 12
1762 +
1763 +#define PAGES_PER_COLOR 2000
1764 +#define NUM_BANKS 8
1765 +#define NUM_COLORS 16
1766 +
1767 +unsigned int NUM_PAGE_LIST; //8*16
1768 +
1769 +unsigned int number_banks;
1770 +unsigned int number_cachecolors;
1771 +
1772 +unsigned int set_partition_max = 0x0000ffff;
1773 +unsigned int set_partition_min = 0;
1774 +unsigned int bank_partition_max = 0x000000ff;
1775 +unsigned int bank_partition_min = 0;
1776 +
1777 +int show_page_pool = 0;
1778 +int refill_page_pool = 0;
1779 +spinlock_t reclaim_lock;
1780 +
1781 +unsigned int set_partition[9] = {
1782 + 0x00000003, /* Core 0, and Level A*/
1783 + 0x00000003, /* Core 0, and Level B*/
1784 + 0x0000000C, /* Core 1, and Level A*/
1785 + 0x0000000C, /* Core 1, and Level B*/
1786 + 0x00000030, /* Core 2, and Level A*/
1787 + 0x00000030, /* Core 2, and Level B*/
1788 + 0x000000C0, /* Core 3, and Level A*/
1789 + 0x000000C0, /* Core 3, and Level B*/
1790 + 0x0000ff00, /* Level C */
1791 +};
1792 +
1793 +unsigned int bank_partition[9] = {
1794 + 0x00000010, /* Core 0, and Level A*/
1795 + 0x00000010, /* Core 0, and Level B*/
1796 + 0x00000020, /* Core 1, and Level A*/
1797 + 0x00000020, /* Core 1, and Level B*/
1798 + 0x00000040, /* Core 2, and Level A*/
1799 + 0x00000040, /* Core 2, and Level B*/
1800 + 0x00000080, /* Core 3, and Level A*/
1801 + 0x00000080, /* Core 3, and Level B*/
1802 + 0x0000000c, /* Level C */
1803 +};
1804 +
1805 +unsigned int set_index[9] = {
1806 + 0, 0, 0, 0, 0, 0, 0, 0, 0
1807 +};
1808 +
1809 +unsigned int bank_index[9] = {
1810 + 0, 0, 0, 0, 0, 0, 0, 0, 0
1811 +};
1812 +
1813 +int node_index[9] = {
1814 + -1, -1, -1, -1, -1, -1, -1, -1, -1
1815 +};
1816 +
1817 +struct mutex void_lockdown_proc;
1818 +
1819 +/*
1820 + * Every page list should contain a lock, a list, and a number recording how many pages it store
1821 + */
1822 +struct color_group {
1823 + spinlock_t lock;
1824 + char _lock_name[LITMUS_LOCKDEP_NAME_MAX_LEN];
1825 + struct list_head list;
1826 + atomic_t nr_pages;
1827 +};
1828 +
1829 +
1830 +static struct color_group *color_groups;
1831 +
1832 +/*
1833 + * Naive function to count the number of 1's
1834 + */
1835 +unsigned int counting_one_set(unsigned int v)
1836 +{
1837 + unsigned int c; // c accumulates the total bits set in v
1838 +
1839 + for (c = 0; v; v >>= 1)
1840 + {
1841 + c += v & 1;
1842 + }
1843 + return c;
1844 +}
1845 +
1846 +unsigned int two_exp(unsigned int e)
1847 +{
1848 + unsigned int v = 1;
1849 + for (; e>0; e-- )
1850 + {
1851 + v=v*2;
1852 + }
1853 + return v;
1854 +}
1855 +
1856 +/* helper functions to find the next colored pool index */
1857 +static inline unsigned int first_index(unsigned long node)
1858 +{
1859 + unsigned int bank_no = 0, color_no = 0;
1860 +
1861 + while(bank_no < NUM_BANKS) {
1862 + if ((bank_partition[node]>>bank_no) & 0x1)
1863 + break;
1864 + bank_no++;
1865 + }
1866 + while(color_no < NUM_COLORS) {
1867 + if ((set_partition[node]>>color_no) & 0x1)
1868 + break;
1869 + color_no++;
1870 + }
1871 + return NUM_COLORS*bank_no + color_no;
1872 +}
1873 +
1874 +static inline unsigned int last_index(unsigned long node)
1875 +{
1876 + unsigned int bank_no = NUM_BANKS-1, color_no = NUM_COLORS-1;
1877 +
1878 + while(bank_no >= 0) {
1879 + if ((bank_partition[node]>>bank_no) & 0x1)
1880 + break;
1881 + bank_no--;
1882 + }
1883 + while(color_no >= 0) {
1884 + if ((set_partition[node]>>color_no) & 0x1)
1885 + break;
1886 + color_no--;
1887 + }
1888 + return NUM_COLORS*bank_no + color_no;
1889 +}
1890 +
1891 +static inline unsigned int next_color(unsigned long node, unsigned int current_color)
1892 +{
1893 + int try = 0, ret = 0;
1894 + current_color++;
1895 + if (current_color == NUM_COLORS) {
1896 + current_color = 0;
1897 + ret = 1;
1898 + }
1899 +
1900 + while (try < NUM_COLORS) {
1901 + if ((set_partition[node]>>current_color)&0x1)
1902 + break;
1903 + current_color++;
1904 + if (current_color == NUM_COLORS) {
1905 + current_color = 0;
1906 + ret = 1;
1907 + }
1908 + try++;
1909 + }
1910 + if (!ret)
1911 + return current_color;
1912 + else
1913 + return current_color + NUM_COLORS;
1914 +}
1915 +
1916 +static inline unsigned int next_bank(unsigned long node, unsigned int current_bank)
1917 +{
1918 + int try = 0;
1919 + current_bank++;
1920 + if (current_bank == NUM_BANKS) {
1921 + current_bank = 0;
1922 + }
1923 +
1924 + while (try < NUM_BANKS) {
1925 + if ((bank_partition[node]>>current_bank)&0x1)
1926 + break;
1927 + current_bank++;
1928 + if (current_bank == NUM_BANKS) {
1929 + current_bank = 0;
1930 + }
1931 + try++;
1932 + }
1933 + return current_bank;
1934 +}
1935 +
1936 +static inline unsigned int get_next_index(unsigned long node, unsigned int current_index)
1937 +{
1938 + unsigned int bank_no, color_no, color_ret, bank_ret;
1939 + bank_no = current_index>>4; // 2^4 = 16 colors
1940 + color_no = current_index - bank_no*NUM_COLORS;
1941 + bank_ret = bank_no;
1942 + color_ret = next_color(node, color_no);
1943 + if (color_ret >= NUM_COLORS) {
1944 + // next bank
1945 + color_ret -= NUM_COLORS;
1946 + bank_ret = next_bank(node, bank_no);
1947 + }
1948 +
1949 + return bank_ret * NUM_COLORS + color_ret;
1950 +}
1951 +
1952 +/* Decoding page color, 0~15 */
1953 +static inline unsigned int page_color(struct page *page)
1954 +{
1955 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
1956 +}
1957 +
1958 +/* Decoding page bank number, 0~7 */
1959 +static inline unsigned int page_bank(struct page *page)
1960 +{
1961 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
1962 +}
1963 +
1964 +static inline unsigned int page_list_index(struct page *page)
1965 +{
1966 + unsigned int idx;
1967 + idx = (page_color(page) + page_bank(page)*(number_cachecolors));
1968 +
1969 + return idx;
1970 +}
1971 +
1972 +
1973 +
1974 +/*
1975 + * It is used to determine the smallest number of page lists.
1976 + */
1977 +static unsigned long smallest_nr_pages(void)
1978 +{
1979 + unsigned long i, min_pages;
1980 + struct color_group *cgroup;
1981 + cgroup = &color_groups[16*2];
1982 + min_pages =atomic_read(&cgroup->nr_pages);
1983 + for (i = 16*2; i < NUM_PAGE_LIST; ++i) {
1984 + cgroup = &color_groups[i];
1985 + if (atomic_read(&cgroup->nr_pages) < min_pages)
1986 + min_pages = atomic_read(&cgroup->nr_pages);
1987 + }
1988 + return min_pages;
1989 +}
1990 +
1991 +static void show_nr_pages(void)
1992 +{
1993 + unsigned long i;
1994 + struct color_group *cgroup;
1995 + printk("show nr pages***************************************\n");
1996 + for (i = 0; i < NUM_PAGE_LIST; ++i) {
1997 + cgroup = &color_groups[i];
1998 + printk("(%03ld) = %03d, ", i, atomic_read(&cgroup->nr_pages));
1999 + if((i % 8) ==7) {
2000 + printk("\n");
2001 + }
2002 + }
2003 +}
2004 +
2005 +/*
2006 + * Add a page to current pool.
2007 + */
2008 +void add_page_to_color_list(struct page *page)
2009 +{
2010 + const unsigned long color = page_list_index(page);
2011 + struct color_group *cgroup = &color_groups[color];
2012 + BUG_ON(in_list(&page->lru) || PageLRU(page));
2013 + BUG_ON(page_count(page) > 1);
2014 + spin_lock(&cgroup->lock);
2015 + list_add_tail(&page->lru, &cgroup->list);
2016 + atomic_inc(&cgroup->nr_pages);
2017 + SetPageLRU(page);
2018 + spin_unlock(&cgroup->lock);
2019 +}
2020 +
2021 +/*
2022 + * Replenish the page pool.
2023 + * If the newly allocate page is what we want, it will be pushed to the correct page list
2024 + * otherwise, it will be freed.
2025 + * A user needs to invoke this function until the page pool has enough pages.
2026 + */
2027 +static int do_add_pages(void)
2028 +{
2029 + struct page *page, *page_tmp;
2030 + LIST_HEAD(free_later);
2031 + unsigned long color;
2032 + int ret = 0;
2033 + int i = 0;
2034 + int free_counter = 0;
2035 + unsigned long counter[128]= {0};
2036 +
2037 + // until all the page lists contain enough pages
2038 + for (i=0; i< 1024*20;i++) {
2039 + page = alloc_page(GFP_HIGHUSER_MOVABLE);
2040 +
2041 + if (unlikely(!page)) {
2042 + printk(KERN_WARNING "Could not allocate pages.\n");
2043 + ret = -ENOMEM;
2044 + goto out;
2045 + }
2046 + color = page_list_index(page);
2047 + counter[color]++;
2048 + if (atomic_read(&color_groups[color].nr_pages) < PAGES_PER_COLOR && color>=0) {
2049 + add_page_to_color_list(page);
2050 + } else {
2051 + // Pages here will be freed later
2052 + list_add_tail(&page->lru, &free_later);
2053 + free_counter++;
2054 + }
2055 + }
2056 +
2057 + // Free the unwanted pages
2058 + list_for_each_entry_safe(page, page_tmp, &free_later, lru) {
2059 + list_del(&page->lru);
2060 + __free_page(page);
2061 + }
2062 +out:
2063 + return ret;
2064 +}
2065 +
2066 +/*
2067 + * Provide pages for replacement according cache color
2068 + * This should be the only implementation here
2069 + * This function should not be accessed by others directly.
2070 + *
2071 + */
2072 +static struct page *new_alloc_page_color( unsigned long color)
2073 +{
2074 +// printk("allocate new page color = %d\n", color);
2075 + struct color_group *cgroup;
2076 + struct page *rPage = NULL;
2077 +
2078 + if( (color <0) || (color)>(number_cachecolors*number_banks -1)) {
2079 + TRACE_CUR("Wrong color %lu\n", color);
2080 + goto out;
2081 + }
2082 +
2083 +
2084 + cgroup = &color_groups[color];
2085 + spin_lock(&cgroup->lock);
2086 + if (unlikely(!atomic_read(&cgroup->nr_pages))) {
2087 + TRACE_CUR("No free %lu colored pages.\n", color);
2088 + goto out_unlock;
2089 + }
2090 + rPage = list_first_entry(&cgroup->list, struct page, lru);
2091 + BUG_ON(page_count(rPage) > 1);
2092 + //get_page(rPage);
2093 + list_del(&rPage->lru);
2094 + atomic_dec(&cgroup->nr_pages);
2095 + ClearPageLRU(rPage);
2096 +out_unlock:
2097 + spin_unlock(&cgroup->lock);
2098 +out:
2099 + return rPage;
2100 +}
2101 +
2102 +struct page* get_colored_page(unsigned long color)
2103 +{
2104 + return new_alloc_page_color(color);
2105 +}
2106 +
2107 +/*
2108 + * provide pages for replacement according to
2109 + * node = 0 for Level A tasks in Cpu 0
2110 + * node = 1 for Level B tasks in Cpu 0
2111 + * node = 2 for Level A tasks in Cpu 1
2112 + * node = 3 for Level B tasks in Cpu 1
2113 + * node = 4 for Level A tasks in Cpu 2
2114 + * node = 5 for Level B tasks in Cpu 2
2115 + * node = 6 for Level A tasks in Cpu 3
2116 + * node = 7 for Level B tasks in Cpu 3
2117 + * node = 8 for Level C tasks
2118 + */
2119 +struct page *new_alloc_page(struct page *page, unsigned long node, int **x)
2120 +{
2121 + struct page *rPage = NULL;
2122 + int try = 0;
2123 + unsigned int idx;
2124 +
2125 + if (node_index[node] == -1)
2126 + idx = first_index(node);
2127 + else
2128 + idx = node_index[node];
2129 +
2130 + BUG_ON(idx<0 || idx>127);
2131 + rPage = new_alloc_page_color(idx);
2132 + if (node_index[node] == last_index(node))
2133 + node_index[node] = first_index(node);
2134 + else
2135 + node_index[node]++;
2136 +
2137 + while (!rPage) {
2138 + try++;
2139 + if (try>=256)
2140 + break;
2141 + idx = get_next_index(node, idx);
2142 + printk(KERN_ALERT "try = %d out of page! requesting node = %ld, idx = %d\n", try, node, idx);
2143 + BUG_ON(idx<0 || idx>127);
2144 + rPage = new_alloc_page_color(idx);
2145 + }
2146 + node_index[node] = idx;
2147 + return rPage;
2148 +}
2149 +
2150 +
2151 +/*
2152 + * Reclaim pages.
2153 + */
2154 +void reclaim_page(struct page *page)
2155 +{
2156 + const unsigned long color = page_list_index(page);
2157 + spin_lock(&reclaim_lock);
2158 + put_page(page);
2159 + add_page_to_color_list(page);
2160 +
2161 + spin_unlock(&reclaim_lock);
2162 + printk("Reclaimed page(%ld) = color %x, bank %x, [color] =%d \n", color, page_color(page), page_bank(page), atomic_read(&color_groups[color].nr_pages));
2163 +}
2164 +
2165 +
2166 +/*
2167 + * Initialize the numbers of banks and cache colors
2168 + */
2169 +static void __init init_variables(void)
2170 +{
2171 + number_banks = counting_one_set(BANK_MASK);
2172 + number_banks = two_exp(number_banks);
2173 +
2174 + number_cachecolors = counting_one_set(CACHE_MASK);
2175 + number_cachecolors = two_exp(number_cachecolors);
2176 + NUM_PAGE_LIST = number_banks * number_cachecolors;
2177 + printk(KERN_WARNING "number of banks = %d, number of cachecolors=%d\n", number_banks, number_cachecolors);
2178 + mutex_init(&void_lockdown_proc);
2179 + spin_lock_init(&reclaim_lock);
2180 +
2181 +}
2182 +
2183 +
2184 +/*
2185 + * Initialize the page pool
2186 + */
2187 +static int __init init_color_groups(void)
2188 +{
2189 + struct color_group *cgroup;
2190 + unsigned long i;
2191 + int err = 0;
2192 +
2193 + printk("NUM_PAGE_LIST = %d\n", NUM_PAGE_LIST);
2194 + color_groups = kmalloc(NUM_PAGE_LIST *sizeof(struct color_group), GFP_KERNEL);
2195 +
2196 + if (!color_groups) {
2197 + printk(KERN_WARNING "Could not allocate color groups.\n");
2198 + err = -ENOMEM;
2199 + }else{
2200 +
2201 + for (i = 0; i < NUM_PAGE_LIST; ++i) {
2202 + cgroup = &color_groups[i];
2203 + atomic_set(&cgroup->nr_pages, 0);
2204 + INIT_LIST_HEAD(&cgroup->list);
2205 + spin_lock_init(&cgroup->lock);
2206 + }
2207 + }
2208 + return err;
2209 +}
2210 +
2211 +int set_partition_handler(struct ctl_table *table, int write, void __user *buffer,
2212 + size_t *lenp, loff_t *ppos)
2213 +{
2214 + int ret = 0, i = 0;
2215 + mutex_lock(&void_lockdown_proc);
2216 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2217 + if (ret)
2218 + goto out;
2219 + if (write) {
2220 + printk("New set Partition : \n");
2221 + for(i =0;i <9;i++)
2222 + {
2223 + set_index[i] = 0;
2224 + printk("set[%d] = %x \n", i, set_partition[i]);
2225 + }
2226 + }
2227 +out:
2228 + mutex_unlock(&void_lockdown_proc);
2229 + return ret;
2230 +}
2231 +
2232 +int bank_partition_handler(struct ctl_table *table, int write, void __user *buffer,
2233 + size_t *lenp, loff_t *ppos)
2234 +{
2235 + int ret = 0, i = 0;
2236 + mutex_lock(&void_lockdown_proc);
2237 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2238 + if (ret)
2239 + goto out;
2240 + if (write) {
2241 + for(i =0;i <9;i++)
2242 + {
2243 + bank_index[i] = 0;
2244 + }
2245 + }
2246 +out:
2247 + mutex_unlock(&void_lockdown_proc);
2248 + return ret;
2249 +}
2250 +
2251 +int show_page_pool_handler(struct ctl_table *table, int write, void __user *buffer,
2252 + size_t *lenp, loff_t *ppos)
2253 +{
2254 + int ret = 0;
2255 + mutex_lock(&void_lockdown_proc);
2256 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2257 + if (ret)
2258 + goto out;
2259 + if (write) {
2260 + show_nr_pages();
2261 + }
2262 +out:
2263 + mutex_unlock(&void_lockdown_proc);
2264 + return ret;
2265 +}
2266 +
2267 +int refill_page_pool_handler(struct ctl_table *table, int write, void __user *buffer,
2268 + size_t *lenp, loff_t *ppos)
2269 +{
2270 + int ret = 0;
2271 + mutex_lock(&void_lockdown_proc);
2272 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2273 + if (ret)
2274 + goto out;
2275 + if (write) {
2276 + do_add_pages();
2277 + show_nr_pages();
2278 + }
2279 +out:
2280 + mutex_unlock(&void_lockdown_proc);
2281 + return ret;
2282 +}
2283 +
2284 +static struct ctl_table cache_table[] =
2285 +{
2286 +
2287 + {
2288 + .procname = "C0_LA_set",
2289 + .mode = 0666,
2290 + .proc_handler = set_partition_handler,
2291 + .data = &set_partition[0],
2292 + .maxlen = sizeof(set_partition[0]),
2293 + .extra1 = &set_partition_min,
2294 + .extra2 = &set_partition_max,
2295 + },
2296 + {
2297 + .procname = "C0_LB_set",
2298 + .mode = 0666,
2299 + .proc_handler = set_partition_handler,
2300 + .data = &set_partition[1],
2301 + .maxlen = sizeof(set_partition[1]),
2302 + .extra1 = &set_partition_min,
2303 + .extra2 = &set_partition_max,
2304 + },
2305 + {
2306 + .procname = "C1_LA_set",
2307 + .mode = 0666,
2308 + .proc_handler = set_partition_handler,
2309 + .data = &set_partition[2],
2310 + .maxlen = sizeof(set_partition[2]),
2311 + .extra1 = &set_partition_min,
2312 + .extra2 = &set_partition_max,
2313 + },
2314 + {
2315 + .procname = "C1_LB_set",
2316 + .mode = 0666,
2317 + .proc_handler = set_partition_handler,
2318 + .data = &set_partition[3],
2319 + .maxlen = sizeof(set_partition[3]),
2320 + .extra1 = &set_partition_min,
2321 + .extra2 = &set_partition_max,
2322 + },
2323 + {
2324 + .procname = "C2_LA_set",
2325 + .mode = 0666,
2326 + .proc_handler = set_partition_handler,
2327 + .data = &set_partition[4],
2328 + .maxlen = sizeof(set_partition[4]),
2329 + .extra1 = &set_partition_min,
2330 + .extra2 = &set_partition_max,
2331 + },
2332 + {
2333 + .procname = "C2_LB_set",
2334 + .mode = 0666,
2335 + .proc_handler = set_partition_handler,
2336 + .data = &set_partition[5],
2337 + .maxlen = sizeof(set_partition[5]),
2338 + .extra1 = &set_partition_min,
2339 + .extra2 = &set_partition_max,
2340 + },
2341 + {
2342 + .procname = "C3_LA_set",
2343 + .mode = 0666,
2344 + .proc_handler = set_partition_handler,
2345 + .data = &set_partition[6],
2346 + .maxlen = sizeof(set_partition[6]),
2347 + .extra1 = &set_partition_min,
2348 + .extra2 = &set_partition_max,
2349 + },
2350 + {
2351 + .procname = "C3_LB_set",
2352 + .mode = 0666,
2353 + .proc_handler = set_partition_handler,
2354 + .data = &set_partition[7],
2355 + .maxlen = sizeof(set_partition[7]),
2356 + .extra1 = &set_partition_min,
2357 + .extra2 = &set_partition_max,
2358 + },
2359 + {
2360 + .procname = "Call_LC_set",
2361 + .mode = 0666,
2362 + .proc_handler = set_partition_handler,
2363 + .data = &set_partition[8],
2364 + .maxlen = sizeof(set_partition[8]),
2365 + .extra1 = &set_partition_min,
2366 + .extra2 = &set_partition_max,
2367 + },
2368 + {
2369 + .procname = "C0_LA_bank",
2370 + .mode = 0666,
2371 + .proc_handler = bank_partition_handler,
2372 + .data = &bank_partition[0],
2373 + .maxlen = sizeof(set_partition[0]),
2374 + .extra1 = &bank_partition_min,
2375 + .extra2 = &bank_partition_max,
2376 + },
2377 + {
2378 + .procname = "C0_LB_bank",
2379 + .mode = 0666,
2380 + .proc_handler = bank_partition_handler,
2381 + .data = &bank_partition[1],
2382 + .maxlen = sizeof(set_partition[1]),
2383 + .extra1 = &bank_partition_min,
2384 + .extra2 = &bank_partition_max,
2385 + },
2386 + {
2387 + .procname = "C1_LA_bank",
2388 + .mode = 0666,
2389 + .proc_handler = bank_partition_handler,
2390 + .data = &bank_partition[2],
2391 + .maxlen = sizeof(set_partition[2]),
2392 + .extra1 = &bank_partition_min,
2393 + .extra2 = &bank_partition_max,
2394 + },
2395 + {
2396 + .procname = "C1_LB_bank",
2397 + .mode = 0666,
2398 + .proc_handler = bank_partition_handler,
2399 + .data = &bank_partition[3],
2400 + .maxlen = sizeof(set_partition[3]),
2401 + .extra1 = &bank_partition_min,
2402 + .extra2 = &bank_partition_max,
2403 + },
2404 + {
2405 + .procname = "C2_LA_bank",
2406 + .mode = 0666,
2407 + .proc_handler = bank_partition_handler,
2408 + .data = &bank_partition[4],
2409 + .maxlen = sizeof(set_partition[4]),
2410 + .extra1 = &bank_partition_min,
2411 + .extra2 = &bank_partition_max,
2412 + },
2413 + {
2414 + .procname = "C2_LB_bank",
2415 + .mode = 0666,
2416 + .proc_handler = bank_partition_handler,
2417 + .data = &bank_partition[5],
2418 + .maxlen = sizeof(set_partition[5]),
2419 + .extra1 = &bank_partition_min,
2420 + .extra2 = &bank_partition_max,
2421 + },
2422 + {
2423 + .procname = "C3_LA_bank",
2424 + .mode = 0666,
2425 + .proc_handler = bank_partition_handler,
2426 + .data = &bank_partition[6],
2427 + .maxlen = sizeof(set_partition[6]),
2428 + .extra1 = &bank_partition_min,
2429 + .extra2 = &bank_partition_max,
2430 + },
2431 + {
2432 + .procname = "C3_LB_bank",
2433 + .mode = 0666,
2434 + .proc_handler = bank_partition_handler,
2435 + .data = &bank_partition[7],
2436 + .maxlen = sizeof(set_partition[7]),
2437 + .extra1 = &bank_partition_min,
2438 + .extra2 = &bank_partition_max,
2439 + },
2440 + {
2441 + .procname = "Call_LC_bank",
2442 + .mode = 0666,
2443 + .proc_handler = bank_partition_handler,
2444 + .data = &bank_partition[8],
2445 + .maxlen = sizeof(set_partition[8]),
2446 + .extra1 = &bank_partition_min,
2447 + .extra2 = &bank_partition_max,
2448 + },
2449 + {
2450 + .procname = "show_page_pool",
2451 + .mode = 0666,
2452 + .proc_handler = show_page_pool_handler,
2453 + .data = &show_page_pool,
2454 + .maxlen = sizeof(show_page_pool),
2455 + }, {
2456 + .procname = "refill_page_pool",
2457 + .mode = 0666,
2458 + .proc_handler = refill_page_pool_handler,
2459 + .data = &refill_page_pool,
2460 + .maxlen = sizeof(refill_page_pool),
2461 + },
2462 + { }
2463 +};
2464 +
2465 +static struct ctl_table litmus_dir_table[] = {
2466 + {
2467 + .procname = "litmus",
2468 + .mode = 0555,
2469 + .child = cache_table,
2470 + },
2471 + { }
2472 +};
2473 +
2474 +
2475 +static struct ctl_table_header *litmus_sysctls;
2476 +
2477 +
2478 +/*
2479 + * Initialzie this proc
2480 + */
2481 +static int __init litmus_color_init(void)
2482 +{
2483 + int err=0;
2484 + printk("Init bankproc.c\n");
2485 +
2486 + init_variables();
2487 +
2488 + printk(KERN_INFO "Registering LITMUS^RT proc color sysctl.\n");
2489 +
2490 + litmus_sysctls = register_sysctl_table(litmus_dir_table);
2491 + if (!litmus_sysctls) {
2492 + printk(KERN_WARNING "Could not register LITMUS^RT color sysctl.\n");
2493 + err = -EFAULT;
2494 + goto out;
2495 + }
2496 +
2497 + init_color_groups();
2498 + do_add_pages();
2499 +
2500 + printk(KERN_INFO "Registering LITMUS^RT color and bank proc.\n");
2501 +out:
2502 + return err;
2503 +}
2504 +
2505 +module_init(litmus_color_init);
2506 +
2507 diff --git litmus/budget.c litmus/budget.c
2508 index 47bf78a..d67f4b3 100644
2509 --- litmus/budget.c
2510 +++ litmus/budget.c
2511 @@ -1,9 +1,11 @@
2512 #include <linux/sched.h>
2513 #include <linux/percpu.h>
2514 #include <linux/hrtimer.h>
2515 +#include <linux/uaccess.h>
2516
2517 #include <litmus/litmus.h>
2518 #include <litmus/preempt.h>
2519 +#include <litmus/sched_plugin.h>
2520
2521 #include <litmus/budget.h>
2522
2523 @@ -113,4 +115,54 @@ static int __init init_budget_enforcement(void)
2524 return 0;
2525 }
2526
2527 +void litmus_current_budget(lt_t *used_so_far, lt_t *remaining)
2528 +{
2529 + struct task_struct *t = current;
2530 + unsigned long flags;
2531 + s64 delta;
2532 +
2533 + local_irq_save(flags);
2534 +
2535 + delta = sched_clock_cpu(smp_processor_id()) - t->se.exec_start;
2536 + if (delta < 0)
2537 + delta = 0;
2538 +
2539 + TRACE_CUR("current_budget: sc:%llu start:%llu lt_t:%llu delta:%lld exec-time:%llu rem:%llu\n",
2540 + sched_clock_cpu(smp_processor_id()), t->se.exec_start,
2541 + litmus_clock(), delta,
2542 + tsk_rt(t)->job_params.exec_time,
2543 + budget_remaining(t));
2544 +
2545 + if (used_so_far)
2546 + *used_so_far = tsk_rt(t)->job_params.exec_time + delta;
2547 +
2548 + if (remaining) {
2549 + *remaining = budget_remaining(t);
2550 + if (*remaining > delta)
2551 + *remaining -= delta;
2552 + else
2553 + *remaining = 0;
2554 + }
2555 +
2556 + local_irq_restore(flags);
2557 +}
2558 +
2559 +asmlinkage long sys_get_current_budget(
2560 + lt_t __user * _expended,
2561 + lt_t __user *_remaining)
2562 +{
2563 + lt_t expended = 0, remaining = 0;
2564 +
2565 + if (is_realtime(current))
2566 + litmus->current_budget(&expended, &remaining);
2567 +
2568 + if (_expended && put_user(expended, _expended))
2569 + return -EFAULT;
2570 +
2571 + if (_remaining && put_user(remaining, _remaining))
2572 + return -EFAULT;
2573 +
2574 + return 0;
2575 +}
2576 +
2577 module_init(init_budget_enforcement);
2578 diff --git litmus/cache_proc.c litmus/cache_proc.c
2579 new file mode 100644
2580 index 0000000..90fb17d
2581 --- /dev/null
2582 +++ litmus/cache_proc.c
2583 @@ -0,0 +1,1338 @@
2584 +#include <asm/uaccess.h>
2585 +#include <linux/uaccess.h>
2586 +#include <linux/init.h>
2587 +#include <linux/types.h>
2588 +#include <linux/kernel.h>
2589 +#include <linux/module.h>
2590 +#include <linux/sysctl.h>
2591 +#include <linux/slab.h>
2592 +#include <linux/io.h>
2593 +#include <linux/mutex.h>
2594 +#include <linux/time.h>
2595 +#include <linux/random.h>
2596 +#include <linux/sched.h>
2597 +
2598 +#include <litmus/rt_param.h>
2599 +#include <litmus/litmus.h>
2600 +#include <litmus/litmus_proc.h>
2601 +#include <litmus/sched_trace.h>
2602 +#include <litmus/cache_proc.h>
2603 +#include <litmus/mc2_common.h>
2604 +
2605 +#include <asm/hardware/cache-l2x0.h>
2606 +#include <asm/cacheflush.h>
2607 +
2608 +#define UNLOCK_ALL 0x00000000 /* allocation in any way */
2609 +#define LOCK_ALL (~UNLOCK_ALL)
2610 +#define MAX_NR_WAYS 16
2611 +#define MAX_NR_COLORS 16
2612 +#define CACHELINE_SIZE 32
2613 +#define INTS_IN_CACHELINE (CACHELINE_SIZE/sizeof(int))
2614 +#define CACHELINES_IN_1KB (1024 / sizeof(cacheline_t))
2615 +
2616 +typedef struct cacheline
2617 +{
2618 + int line[INTS_IN_CACHELINE];
2619 +} __attribute__((aligned(CACHELINE_SIZE))) cacheline_t;
2620 +
2621 +void mem_lock(u32 lock_val, int cpu);
2622 +
2623 +/*
2624 + * unlocked_way[i] : allocation can occur in way i
2625 + *
2626 + * 0 = allocation can occur in the corresponding way
2627 + * 1 = allocation cannot occur in the corresponding way
2628 + */
2629 +u32 unlocked_way[MAX_NR_WAYS] = {
2630 + 0xFFFFFFFE, /* way 0 unlocked */
2631 + 0xFFFFFFFD,
2632 + 0xFFFFFFFB,
2633 + 0xFFFFFFF7,
2634 + 0xFFFFFFEF, /* way 4 unlocked */
2635 + 0xFFFFFFDF,
2636 + 0xFFFFFFBF,
2637 + 0xFFFFFF7F,
2638 + 0xFFFFFEFF, /* way 8 unlocked */
2639 + 0xFFFFFDFF,
2640 + 0xFFFFFBFF,
2641 + 0xFFFFF7FF,
2642 + 0xFFFFEFFF, /* way 12 unlocked */
2643 + 0xFFFFDFFF,
2644 + 0xFFFFBFFF,
2645 + 0xFFFF7FFF,
2646 +};
2647 +
2648 +u32 nr_unlocked_way[MAX_NR_WAYS+1] = {
2649 + 0x0000FFFF, /* all ways are locked. usable = 0*/
2650 + 0x0000FFFE, /* way ~0 unlocked. usable = 1 */
2651 + 0x0000FFFC,
2652 + 0x0000FFF8,
2653 + 0x0000FFF0,
2654 + 0x0000FFE0,
2655 + 0x0000FFC0,
2656 + 0x0000FF80,
2657 + 0x0000FF00,
2658 + 0x0000FE00,
2659 + 0x0000FC00,
2660 + 0x0000F800,
2661 + 0x0000F000,
2662 + 0x0000E000,
2663 + 0x0000C000,
2664 + 0x00008000,
2665 + 0x00000000, /* way ~15 unlocked. usable = 16 */
2666 +};
2667 +
2668 +u32 way_partition[4] = {
2669 + 0xfffffff0, /* cpu0 */
2670 + 0xffffff0f, /* cpu1 */
2671 + 0xfffff0ff, /* cpu2 */
2672 + 0xffff0fff, /* cpu3 */
2673 +};
2674 +
2675 +u32 way_partitions[9] = {
2676 + 0xffff00ff, /* cpu0 A */
2677 + 0xffff00ff, /* cpu0 B */
2678 + 0xffff00ff, /* cpu1 A */
2679 + 0xffff00ff, /* cpu1 B */
2680 + 0xffff00ff, /* cpu2 A */
2681 + 0xffff00ff, /* cpu2 B */
2682 + 0xffff00ff, /* cpu3 A */
2683 + 0xffff00ff, /* cpu3 B */
2684 + 0xffffff00, /* lv C */
2685 +};
2686 +
2687 +u32 prev_lockdown_d_reg[5] = {
2688 + 0x0000FF00,
2689 + 0x0000FF00,
2690 + 0x0000FF00,
2691 + 0x0000FF00,
2692 + 0x000000FF, /* share with level-C */
2693 +};
2694 +
2695 +u32 prev_lockdown_i_reg[5] = {
2696 + 0x0000FF00,
2697 + 0x0000FF00,
2698 + 0x0000FF00,
2699 + 0x0000FF00,
2700 + 0x000000FF, /* share with level-C */
2701 +};
2702 +
2703 +u32 prev_lbm_i_reg[8] = {
2704 + 0x00000000,
2705 + 0x00000000,
2706 + 0x00000000,
2707 + 0x00000000,
2708 + 0x00000000,
2709 + 0x00000000,
2710 + 0x00000000,
2711 + 0x00000000,
2712 +};
2713 +
2714 +u32 prev_lbm_d_reg[8] = {
2715 + 0x00000000,
2716 + 0x00000000,
2717 + 0x00000000,
2718 + 0x00000000,
2719 + 0x00000000,
2720 + 0x00000000,
2721 + 0x00000000,
2722 + 0x00000000,
2723 +};
2724 +
2725 +static void __iomem *cache_base;
2726 +static void __iomem *lockreg_d;
2727 +static void __iomem *lockreg_i;
2728 +
2729 +static u32 cache_id;
2730 +
2731 +struct mutex actlr_mutex;
2732 +struct mutex l2x0_prefetch_mutex;
2733 +struct mutex lockdown_proc;
2734 +static u32 way_partition_min;
2735 +static u32 way_partition_max;
2736 +
2737 +static int zero = 0;
2738 +static int one = 1;
2739 +
2740 +static int l1_prefetch_proc;
2741 +static int l2_prefetch_hint_proc;
2742 +static int l2_double_linefill_proc;
2743 +static int l2_data_prefetch_proc;
2744 +static int os_isolation;
2745 +static int use_part;
2746 +
2747 +u32 lockdown_reg[9] = {
2748 + 0x00000000,
2749 + 0x00000000,
2750 + 0x00000000,
2751 + 0x00000000,
2752 + 0x00000000,
2753 + 0x00000000,
2754 + 0x00000000,
2755 + 0x00000000,
2756 +};
2757 +
2758 +
2759 +#define ld_d_reg(cpu) ({ int __cpu = cpu; \
2760 + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_D_BASE + \
2761 + __cpu * L2X0_LOCKDOWN_STRIDE; __v; })
2762 +#define ld_i_reg(cpu) ({ int __cpu = cpu; \
2763 + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_I_BASE + \
2764 + __cpu * L2X0_LOCKDOWN_STRIDE; __v; })
2765 +
2766 +int lock_all;
2767 +int nr_lockregs;
2768 +static raw_spinlock_t cache_lock;
2769 +static raw_spinlock_t prefetch_lock;
2770 +static void ***flusher_pages = NULL;
2771 +
2772 +extern void l2c310_flush_all(void);
2773 +
2774 +static inline void cache_wait_way(void __iomem *reg, unsigned long mask)
2775 +{
2776 + /* wait for cache operation by line or way to complete */
2777 + while (readl_relaxed(reg) & mask)
2778 + cpu_relax();
2779 +}
2780 +
2781 +#ifdef CONFIG_CACHE_L2X0
2782 +static inline void cache_wait(void __iomem *reg, unsigned long mask)
2783 +{
2784 + /* cache operations by line are atomic on PL310 */
2785 +}
2786 +#else
2787 +#define cache_wait cache_wait_way
2788 +#endif
2789 +
2790 +static inline void cache_sync(void)
2791 +{
2792 + void __iomem *base = cache_base;
2793 +
2794 + writel_relaxed(0, base + L2X0_CACHE_SYNC);
2795 + cache_wait(base + L2X0_CACHE_SYNC, 1);
2796 +}
2797 +
2798 +static void print_lockdown_registers(int cpu)
2799 +{
2800 + int i;
2801 + //for (i = 0; i < nr_lockregs; i++) {
2802 + for (i = 0; i < 4; i++) {
2803 + printk("P%d Lockdown Data CPU %2d: 0x%04x\n", cpu,
2804 + i, readl_relaxed(ld_d_reg(i)));
2805 + printk("P%d Lockdown Inst CPU %2d: 0x%04x\n", cpu,
2806 + i, readl_relaxed(ld_i_reg(i)));
2807 + }
2808 +}
2809 +
2810 +static void test_lockdown(void *ignore)
2811 +{
2812 + int i, cpu;
2813 +
2814 + cpu = smp_processor_id();
2815 + printk("Start lockdown test on CPU %d.\n", cpu);
2816 +
2817 + for (i = 0; i < nr_lockregs; i++) {
2818 + printk("CPU %2d data reg: 0x%8p\n", i, ld_d_reg(i));
2819 + printk("CPU %2d inst reg: 0x%8p\n", i, ld_i_reg(i));
2820 + }
2821 +
2822 + printk("Lockdown initial state:\n");
2823 + print_lockdown_registers(cpu);
2824 + printk("---\n");
2825 +
2826 + for (i = 0; i < nr_lockregs; i++) {
2827 + writel_relaxed(1, ld_d_reg(i));
2828 + writel_relaxed(2, ld_i_reg(i));
2829 + }
2830 + printk("Lockdown all data=1 instr=2:\n");
2831 + print_lockdown_registers(cpu);
2832 + printk("---\n");
2833 +
2834 + for (i = 0; i < nr_lockregs; i++) {
2835 + writel_relaxed((1 << i), ld_d_reg(i));
2836 + writel_relaxed(((1 << 8) >> i), ld_i_reg(i));
2837 + }
2838 + printk("Lockdown varies:\n");
2839 + print_lockdown_registers(cpu);
2840 + printk("---\n");
2841 +
2842 + for (i = 0; i < nr_lockregs; i++) {
2843 + writel_relaxed(UNLOCK_ALL, ld_d_reg(i));
2844 + writel_relaxed(UNLOCK_ALL, ld_i_reg(i));
2845 + }
2846 + printk("Lockdown all zero:\n");
2847 + print_lockdown_registers(cpu);
2848 +
2849 + printk("End lockdown test.\n");
2850 +}
2851 +
2852 +void litmus_setup_lockdown(void __iomem *base, u32 id)
2853 +{
2854 + cache_base = base;
2855 + cache_id = id;
2856 + lockreg_d = cache_base + L2X0_LOCKDOWN_WAY_D_BASE;
2857 + lockreg_i = cache_base + L2X0_LOCKDOWN_WAY_I_BASE;
2858 +
2859 + if (L2X0_CACHE_ID_PART_L310 == (cache_id & L2X0_CACHE_ID_PART_MASK)) {
2860 + nr_lockregs = 8;
2861 + } else {
2862 + printk("Unknown cache ID!\n");
2863 + nr_lockregs = 1;
2864 + }
2865 +
2866 + mutex_init(&actlr_mutex);
2867 + mutex_init(&l2x0_prefetch_mutex);
2868 + mutex_init(&lockdown_proc);
2869 + raw_spin_lock_init(&cache_lock);
2870 + raw_spin_lock_init(&prefetch_lock);
2871 +
2872 + test_lockdown(NULL);
2873 +}
2874 +
2875 +int way_partition_handler(struct ctl_table *table, int write, void __user *buffer,
2876 + size_t *lenp, loff_t *ppos)
2877 +{
2878 + int ret = 0, i;
2879 + unsigned long flags;
2880 +
2881 + mutex_lock(&lockdown_proc);
2882 +
2883 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2884 + if (ret)
2885 + goto out;
2886 +
2887 + if (write) {
2888 + printk("Way-partition settings:\n");
2889 + for (i = 0; i < 9; i++) {
2890 + printk("0x%08X\n", way_partitions[i]);
2891 + }
2892 + for (i = 0; i < 4; i++) {
2893 + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
2894 + i * L2X0_LOCKDOWN_STRIDE);
2895 + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
2896 + i * L2X0_LOCKDOWN_STRIDE);
2897 + }
2898 + }
2899 +
2900 + local_irq_save(flags);
2901 + print_lockdown_registers(smp_processor_id());
2902 + l2c310_flush_all();
2903 + local_irq_restore(flags);
2904 +out:
2905 + mutex_unlock(&lockdown_proc);
2906 + return ret;
2907 +}
2908 +
2909 +int lock_all_handler(struct ctl_table *table, int write, void __user *buffer,
2910 + size_t *lenp, loff_t *ppos)
2911 +{
2912 + int ret = 0, i;
2913 + unsigned long flags;
2914 +
2915 + mutex_lock(&lockdown_proc);
2916 +
2917 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2918 + if (ret)
2919 + goto out;
2920 +
2921 + if (write && lock_all == 1) {
2922 + for (i = 0; i < nr_lockregs; i++) {
2923 + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
2924 + i * L2X0_LOCKDOWN_STRIDE);
2925 + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
2926 + i * L2X0_LOCKDOWN_STRIDE);
2927 + }
2928 +/*
2929 + for (i = 0; i < nr_lockregs; i++) {
2930 + barrier();
2931 + mem_lock(LOCK_ALL, i);
2932 + barrier();
2933 + //writel_relaxed(nr_unlocked_way[0], ld_d_reg(i));
2934 + //writel_relaxed(nr_unlocked_way[0], ld_i_reg(i));
2935 + }
2936 +*/
2937 + }
2938 + if (write && lock_all == 0) {
2939 + for (i = 0; i < nr_lockregs; i++) {
2940 + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
2941 + i * L2X0_LOCKDOWN_STRIDE);
2942 + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
2943 + i * L2X0_LOCKDOWN_STRIDE);
2944 + }
2945 +
2946 + }
2947 +
2948 + local_irq_save(flags);
2949 + print_lockdown_registers(smp_processor_id());
2950 + l2c310_flush_all();
2951 + local_irq_restore(flags);
2952 +out:
2953 + mutex_unlock(&lockdown_proc);
2954 + return ret;
2955 +}
2956 +
2957 +void cache_lockdown(u32 lock_val, int cpu)
2958 +{
2959 + __asm__ __volatile__ (
2960 +" str %[lockval], [%[dcachereg]]\n"
2961 +" str %[lockval], [%[icachereg]]\n"
2962 + :
2963 + : [dcachereg] "r" (ld_d_reg(cpu)),
2964 + [icachereg] "r" (ld_i_reg(cpu)),
2965 + [lockval] "r" (lock_val)
2966 + : "cc");
2967 +}
2968 +
2969 +void do_partition(enum crit_level lv, int cpu)
2970 +{
2971 + u32 regs;
2972 + unsigned long flags;
2973 +
2974 + if (lock_all || !use_part)
2975 + return;
2976 + raw_spin_lock_irqsave(&cache_lock, flags);
2977 + switch(lv) {
2978 + case CRIT_LEVEL_A:
2979 + regs = ~way_partitions[cpu*2];
2980 + regs &= 0x0000ffff;
2981 + break;
2982 + case CRIT_LEVEL_B:
2983 + regs = ~way_partitions[cpu*2+1];
2984 + regs &= 0x0000ffff;
2985 + break;
2986 + case CRIT_LEVEL_C:
2987 + case NUM_CRIT_LEVELS:
2988 + regs = ~way_partitions[8];
2989 + regs &= 0x0000ffff;
2990 + break;
2991 + default:
2992 + BUG();
2993 +
2994 + }
2995 + barrier();
2996 +
2997 + writel_relaxed(regs, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + cpu * L2X0_LOCKDOWN_STRIDE);
2998 + writel_relaxed(regs, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + cpu * L2X0_LOCKDOWN_STRIDE);
2999 + barrier();
3000 +
3001 + raw_spin_unlock_irqrestore(&cache_lock, flags);
3002 +}
3003 +
3004 +void lock_cache(int cpu, u32 val)
3005 +{
3006 + unsigned long flags;
3007 +
3008 + local_irq_save(flags);
3009 + if (val != 0xffffffff) {
3010 + writel_relaxed(val, cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
3011 + cpu * L2X0_LOCKDOWN_STRIDE);
3012 + writel_relaxed(val, cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
3013 + cpu * L2X0_LOCKDOWN_STRIDE);
3014 + }
3015 + else {
3016 + int i;
3017 + for (i = 0; i < 4; i++)
3018 + do_partition(CRIT_LEVEL_A, i);
3019 + }
3020 + local_irq_restore(flags);
3021 +}
3022 +
3023 +int use_part_proc_handler(struct ctl_table *table, int write, void __user *buffer,
3024 + size_t *lenp, loff_t *ppos)
3025 +{
3026 + int ret = 0;
3027 +
3028 + mutex_lock(&lockdown_proc);
3029 +
3030 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3031 + if (ret)
3032 + goto out;
3033 +
3034 +
3035 + printk("USE_PART HANDLER = %d\n", use_part);
3036 +
3037 +out:
3038 + mutex_unlock(&lockdown_proc);
3039 + return ret;
3040 +}
3041 +
3042 +int os_isolation_proc_handler(struct ctl_table *table, int write, void __user *buffer,
3043 + size_t *lenp, loff_t *ppos)
3044 +{
3045 + int ret = 0;
3046 +
3047 + mutex_lock(&lockdown_proc);
3048 +
3049 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3050 + if (ret)
3051 + goto out;
3052 +
3053 +
3054 + printk("OS_ISOLATION HANDLER = %d\n", os_isolation);
3055 +
3056 +out:
3057 + mutex_unlock(&lockdown_proc);
3058 + return ret;
3059 +}
3060 +
3061 +int lockdown_reg_handler(struct ctl_table *table, int write, void __user *buffer,
3062 + size_t *lenp, loff_t *ppos)
3063 +{
3064 + int ret = 0, i;
3065 +
3066 + mutex_lock(&lockdown_proc);
3067 +
3068 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3069 + if (ret)
3070 + goto out;
3071 +
3072 + if (write) {
3073 + for (i = 0; i < nr_lockregs; i++) {
3074 + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
3075 + i * L2X0_LOCKDOWN_STRIDE);
3076 + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
3077 + i * L2X0_LOCKDOWN_STRIDE);
3078 + }
3079 + }
3080 +
3081 +out:
3082 + mutex_unlock(&lockdown_proc);
3083 + return ret;
3084 +}
3085 +
3086 +int lockdown_global_handler(struct ctl_table *table, int write, void __user *buffer,
3087 + size_t *lenp, loff_t *ppos)
3088 +{
3089 + int ret = 0, i;
3090 +
3091 + mutex_lock(&lockdown_proc);
3092 +
3093 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
3094 + if (ret)
3095 + goto out;
3096 +
3097 + if (write) {
3098 + for (i = 0; i < nr_lockregs; i++) {
3099 + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_D_BASE +
3100 + i * L2X0_LOCKDOWN_STRIDE);
3101 + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_I_BASE +
3102 + i * L2X0_LOCKDOWN_STRIDE);
3103 + }
3104 + }
3105 +
3106 +out:
3107 + mutex_unlock(&lockdown_proc);
3108 + return ret;
3109 +}
3110 +
3111 +void inline enter_irq_mode(void)
3112 +{
3113 + int cpu = smp_processor_id();
3114 +
3115 + if (os_isolation == 0)
3116 + return;
3117 + prev_lockdown_i_reg[cpu] = readl_relaxed(ld_i_reg(cpu));
3118 + prev_lockdown_d_reg[cpu] = readl_relaxed(ld_d_reg(cpu));
3119 + writel_relaxed(way_partitions[8], ld_i_reg(cpu));
3120 + writel_relaxed(way_partitions[8], ld_d_reg(cpu));
3121 +}
3122 +
3123 +void inline exit_irq_mode(void)
3124 +{
3125 + int cpu = smp_processor_id();
3126 +
3127 + if (os_isolation == 0)
3128 + return;
3129 +
3130 + writel_relaxed(prev_lockdown_i_reg[cpu], ld_i_reg(cpu));
3131 + writel_relaxed(prev_lockdown_d_reg[cpu], ld_d_reg(cpu));
3132 +}
3133 +
3134 +/* Operate on the Cortex-A9's ACTLR register */
3135 +#define ACTLR_L2_PREFETCH_HINT (1 << 1)
3136 +#define ACTLR_L1_PREFETCH (1 << 2)
3137 +
3138 +/*
3139 + * Change the ACTLR.
3140 + * @mode - If 1 (0), set (clear) the bit given in @mask in the ACTLR.
3141 + * @mask - A mask in which one bit is set to operate on the ACTLR.
3142 + */
3143 +static void actlr_change(int mode, int mask)
3144 +{
3145 + u32 orig_value, new_value, reread_value;
3146 +
3147 + if (0 != mode && 1 != mode) {
3148 + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n",
3149 + __FUNCTION__);
3150 + return;
3151 + }
3152 +
3153 + /* get the original value */
3154 + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (orig_value));
3155 +
3156 + if (0 == mode)
3157 + new_value = orig_value & ~(mask);
3158 + else
3159 + new_value = orig_value | mask;
3160 +
3161 + asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (new_value));
3162 + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (reread_value));
3163 +
3164 + printk("ACTLR: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n",
3165 + orig_value, new_value, reread_value);
3166 +}
3167 +
3168 +int litmus_l1_prefetch_proc_handler(struct ctl_table *table, int write,
3169 + void __user *buffer, size_t *lenp, loff_t *ppos)
3170 +{
3171 + int ret, mode;
3172 +
3173 + mutex_lock(&actlr_mutex);
3174 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
3175 +
3176 + if (!ret && write) {
3177 + mode = *((int*)table->data);
3178 + actlr_change(mode, ACTLR_L1_PREFETCH);
3179 + }
3180 + mutex_unlock(&actlr_mutex);
3181 +
3182 + return ret;
3183 +}
3184 +
3185 +int litmus_l2_prefetch_hint_proc_handler(struct ctl_table *table, int write,
3186 + void __user *buffer, size_t *lenp, loff_t *ppos)
3187 +{
3188 + int ret, mode;
3189 +
3190 + mutex_lock(&actlr_mutex);
3191 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
3192 + if (!ret && write) {
3193 + mode = *((int*)table->data);
3194 + actlr_change(mode, ACTLR_L2_PREFETCH_HINT);
3195 + }
3196 + mutex_unlock(&actlr_mutex);
3197 +
3198 + return ret;
3199 +}
3200 +
3201 +
3202 +/* Operate on the PL-310's Prefetch Control Register, L310_PREFETCH_CTRL */
3203 +#define L2X0_PREFETCH_DOUBLE_LINEFILL (1 << 30)
3204 +#define L2X0_PREFETCH_INST_PREFETCH (1 << 29)
3205 +#define L2X0_PREFETCH_DATA_PREFETCH (1 << 28)
3206 +static void l2x0_prefetch_change(int mode, int mask)
3207 +{
3208 + u32 orig_value, new_value, reread_value;
3209 +
3210 + if (0 != mode && 1 != mode) {
3211 + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n",
3212 + __FUNCTION__);
3213 + return;
3214 + }
3215 +
3216 + orig_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL);
3217 +
3218 + if (0 == mode)
3219 + new_value = orig_value & ~(mask);
3220 + else
3221 + new_value = orig_value | mask;
3222 +
3223 + writel_relaxed(new_value, cache_base + L310_PREFETCH_CTRL);
3224 + reread_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL);
3225 +
3226 + printk("l2x0 prefetch: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n",
3227 + orig_value, new_value, reread_value);
3228 +}
3229 +
3230 +int litmus_l2_double_linefill_proc_handler(struct ctl_table *table, int write,
3231 + void __user *buffer, size_t *lenp, loff_t *ppos)
3232 +{
3233 + int ret, mode;
3234 +
3235 + mutex_lock(&l2x0_prefetch_mutex);
3236 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
3237 + if (!ret && write) {
3238 + mode = *((int*)table->data);
3239 + l2x0_prefetch_change(mode, L2X0_PREFETCH_DOUBLE_LINEFILL);
3240 + }
3241 + mutex_unlock(&l2x0_prefetch_mutex);
3242 +
3243 + return ret;
3244 +}
3245 +
3246 +int litmus_l2_data_prefetch_proc_handler(struct ctl_table *table, int write,
3247 + void __user *buffer, size_t *lenp, loff_t *ppos)
3248 +{
3249 + int ret, mode;
3250 +
3251 + mutex_lock(&l2x0_prefetch_mutex);
3252 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
3253 + if (!ret && write) {
3254 + mode = *((int*)table->data);
3255 + l2x0_prefetch_change(mode, L2X0_PREFETCH_DATA_PREFETCH|L2X0_PREFETCH_INST_PREFETCH);
3256 + }
3257 + mutex_unlock(&l2x0_prefetch_mutex);
3258 +
3259 + return ret;
3260 +}
3261 +
3262 +int do_perf_test_proc_handler(struct ctl_table *table, int write,
3263 + void __user *buffer, size_t *lenp, loff_t *ppos);
3264 +
3265 +int setup_flusher_proc_handler(struct ctl_table *table, int write,
3266 + void __user *buffer, size_t *lenp, loff_t *ppos);
3267 +
3268 +static struct ctl_table cache_table[] =
3269 +{
3270 + {
3271 + .procname = "C0_LA_way",
3272 + .mode = 0666,
3273 + .proc_handler = way_partition_handler,
3274 + .data = &way_partitions[0],
3275 + .maxlen = sizeof(way_partitions[0]),
3276 + .extra1 = &way_partition_min,
3277 + .extra2 = &way_partition_max,
3278 + },
3279 + {
3280 + .procname = "C0_LB_way",
3281 + .mode = 0666,
3282 + .proc_handler = way_partition_handler,
3283 + .data = &way_partitions[1],
3284 + .maxlen = sizeof(way_partitions[1]),
3285 + .extra1 = &way_partition_min,
3286 + .extra2 = &way_partition_max,
3287 + },
3288 + {
3289 + .procname = "C1_LA_way",
3290 + .mode = 0666,
3291 + .proc_handler = way_partition_handler,
3292 + .data = &way_partitions[2],
3293 + .maxlen = sizeof(way_partitions[2]),
3294 + .extra1 = &way_partition_min,
3295 + .extra2 = &way_partition_max,
3296 + },
3297 + {
3298 + .procname = "C1_LB_way",
3299 + .mode = 0666,
3300 + .proc_handler = way_partition_handler,
3301 + .data = &way_partitions[3],
3302 + .maxlen = sizeof(way_partitions[3]),
3303 + .extra1 = &way_partition_min,
3304 + .extra2 = &way_partition_max,
3305 + },
3306 + {
3307 + .procname = "C2_LA_way",
3308 + .mode = 0666,
3309 + .proc_handler = way_partition_handler,
3310 + .data = &way_partitions[4],
3311 + .maxlen = sizeof(way_partitions[4]),
3312 + .extra1 = &way_partition_min,
3313 + .extra2 = &way_partition_max,
3314 + },
3315 + {
3316 + .procname = "C2_LB_way",
3317 + .mode = 0666,
3318 + .proc_handler = way_partition_handler,
3319 + .data = &way_partitions[5],
3320 + .maxlen = sizeof(way_partitions[5]),
3321 + .extra1 = &way_partition_min,
3322 + .extra2 = &way_partition_max,
3323 + },
3324 + {
3325 + .procname = "C3_LA_way",
3326 + .mode = 0666,
3327 + .proc_handler = way_partition_handler,
3328 + .data = &way_partitions[6],
3329 + .maxlen = sizeof(way_partitions[6]),
3330 + .extra1 = &way_partition_min,
3331 + .extra2 = &way_partition_max,
3332 + },
3333 + {
3334 + .procname = "C3_LB_way",
3335 + .mode = 0666,
3336 + .proc_handler = way_partition_handler,
3337 + .data = &way_partitions[7],
3338 + .maxlen = sizeof(way_partitions[7]),
3339 + .extra1 = &way_partition_min,
3340 + .extra2 = &way_partition_max,
3341 + },
3342 + {
3343 + .procname = "Call_LC_way",
3344 + .mode = 0666,
3345 + .proc_handler = way_partition_handler,
3346 + .data = &way_partitions[8],
3347 + .maxlen = sizeof(way_partitions[8]),
3348 + .extra1 = &way_partition_min,
3349 + .extra2 = &way_partition_max,
3350 + },
3351 + {
3352 + .procname = "lock_all",
3353 + .mode = 0666,
3354 + .proc_handler = lock_all_handler,
3355 + .data = &lock_all,
3356 + .maxlen = sizeof(lock_all),
3357 + .extra1 = &zero,
3358 + .extra2 = &one,
3359 + },
3360 + {
3361 + .procname = "l1_prefetch",
3362 + .mode = 0644,
3363 + .proc_handler = litmus_l1_prefetch_proc_handler,
3364 + .data = &l1_prefetch_proc,
3365 + .maxlen = sizeof(l1_prefetch_proc),
3366 + },
3367 + {
3368 + .procname = "l2_prefetch_hint",
3369 + .mode = 0644,
3370 + .proc_handler = litmus_l2_prefetch_hint_proc_handler,
3371 + .data = &l2_prefetch_hint_proc,
3372 + .maxlen = sizeof(l2_prefetch_hint_proc),
3373 + },
3374 + {
3375 + .procname = "l2_double_linefill",
3376 + .mode = 0644,
3377 + .proc_handler = litmus_l2_double_linefill_proc_handler,
3378 + .data = &l2_double_linefill_proc,
3379 + .maxlen = sizeof(l2_double_linefill_proc),
3380 + },
3381 + {
3382 + .procname = "l2_data_prefetch",
3383 + .mode = 0644,
3384 + .proc_handler = litmus_l2_data_prefetch_proc_handler,
3385 + .data = &l2_data_prefetch_proc,
3386 + .maxlen = sizeof(l2_data_prefetch_proc),
3387 + },
3388 + {
3389 + .procname = "os_isolation",
3390 + .mode = 0644,
3391 + .proc_handler = os_isolation_proc_handler,
3392 + .data = &os_isolation,
3393 + .maxlen = sizeof(os_isolation),
3394 + },
3395 + {
3396 + .procname = "use_part",
3397 + .mode = 0644,
3398 + .proc_handler = use_part_proc_handler,
3399 + .data = &use_part,
3400 + .maxlen = sizeof(use_part),
3401 + },
3402 + {
3403 + .procname = "do_perf_test",
3404 + .mode = 0644,
3405 + .proc_handler = do_perf_test_proc_handler,
3406 + },
3407 + {
3408 + .procname = "setup_flusher",
3409 + .mode = 0644,
3410 + .proc_handler = setup_flusher_proc_handler,
3411 + },
3412 + {
3413 + .procname = "lockdown_reg_0",
3414 + .mode = 0644,
3415 + .proc_handler = lockdown_reg_handler,
3416 + .data = &lockdown_reg[0],
3417 + .maxlen = sizeof(lockdown_reg[0]),
3418 + .extra1 = &way_partition_min,
3419 + .extra2 = &way_partition_max,
3420 + },
3421 + {
3422 + .procname = "lockdown_reg_1",
3423 + .mode = 0644,
3424 + .proc_handler = lockdown_reg_handler,
3425 + .data = &lockdown_reg[1],
3426 + .maxlen = sizeof(lockdown_reg[1]),
3427 + .extra1 = &way_partition_min,
3428 + .extra2 = &way_partition_max,
3429 + },
3430 + {
3431 + .procname = "lockdown_reg_2",
3432 + .mode = 0644,
3433 + .proc_handler = lockdown_reg_handler,
3434 + .data = &lockdown_reg[2],
3435 + .maxlen = sizeof(lockdown_reg[2]),
3436 + .extra1 = &way_partition_min,
3437 + .extra2 = &way_partition_max,
3438 + },
3439 + {
3440 + .procname = "lockdown_reg_3",
3441 + .mode = 0644,
3442 + .proc_handler = lockdown_reg_handler,
3443 + .data = &lockdown_reg[3],
3444 + .maxlen = sizeof(lockdown_reg[3]),
3445 + .extra1 = &way_partition_min,
3446 + .extra2 = &way_partition_max,
3447 + },
3448 + {
3449 + .procname = "lockdown_regs",
3450 + .mode = 0644,
3451 + .proc_handler = lockdown_global_handler,
3452 + .data = &lockdown_reg[8],
3453 + .maxlen = sizeof(lockdown_reg[8]),
3454 + .extra1 = &way_partition_min,
3455 + .extra2 = &way_partition_max,
3456 + },
3457 + { }
3458 +};
3459 +
3460 +static struct ctl_table litmus_dir_table[] = {
3461 + {
3462 + .procname = "litmus",
3463 + .mode = 0555,
3464 + .child = cache_table,
3465 + },
3466 + { }
3467 +};
3468 +
3469 +u32 color_read_in_mem(u32 lock_val, u32 unlock_val, void *start, void *end)
3470 +{
3471 + u32 v = 0;
3472 +
3473 + __asm__ __volatile__ (
3474 +" .align 5\n"
3475 +" str %[lockval], [%[cachereg]]\n"
3476 +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n"
3477 +" cmp %[end], %[addr] @ subtracts addr from end\n"
3478 +" bgt 1b\n @ read more, if necessary\n"
3479 + : [addr] "+r" (start),
3480 + [val] "+r" (v)
3481 + : [end] "r" (end),
3482 +#ifdef CONFIG_CACHE_L2X0
3483 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())),
3484 +#else
3485 + [cachereg] "r" (lockreg_d),
3486 +#endif
3487 + [lockval] "r" (lock_val)
3488 + : "cc");
3489 +
3490 + return v;
3491 +}
3492 +
3493 +
3494 +/*
3495 + * Prefetch by reading the first word of each cache line in a page.
3496 + *
3497 + * @lockdown_reg: address of the lockdown register to write
3498 + * @lock_val: value to be written to @lockdown_reg
3499 + * @unlock_val: will unlock the cache to this value
3500 + * @addr: start address to be prefetched
3501 + * @end_addr: end address to prefetch (exclusive)
3502 + *
3503 + * Assumes: addr < end_addr AND addr != end_addr
3504 + */
3505 +u32 color_read_in_mem_lock(u32 lock_val, u32 unlock_val, void *start, void *end)
3506 +{
3507 +#ifndef CONFIG_CACHE_L2X0
3508 + unsigned long flags;
3509 +#endif
3510 + u32 v = 0;
3511 +
3512 +#ifndef CONFIG_CACHE_L2X0
3513 + raw_spin_lock_irqsave(&prefetch_lock, flags);
3514 +#endif
3515 +
3516 + __asm__ __volatile__ (
3517 +" .align 5\n"
3518 +" str %[lockval], [%[cachereg]]\n"
3519 +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n"
3520 +" cmp %[end], %[addr] @ subtracts addr from end\n"
3521 +" bgt 1b\n @ read more, if necessary\n"
3522 +" str %[unlockval], [%[cachereg]]\n"
3523 + : [addr] "+r" (start),
3524 + [val] "+r" (v)
3525 + : [end] "r" (end),
3526 +#ifdef CONFIG_CACHE_L2X0
3527 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())),
3528 +#else
3529 + [cachereg] "r" (lockreg_d),
3530 +#endif
3531 + [lockval] "r" (lock_val),
3532 + [unlockval] "r" (unlock_val)
3533 + : "cc");
3534 +
3535 +#ifndef CONFIG_CACHE_L2X0
3536 + raw_spin_unlock_irqrestore(&prefetch_lock, flags);
3537 +#endif
3538 +
3539 + return v;
3540 +}
3541 +
3542 +static long update_timeval(struct timespec lhs, struct timespec rhs)
3543 +{
3544 + long val;
3545 + struct timespec ts;
3546 +
3547 + ts = timespec_sub(rhs, lhs);
3548 + val = ts.tv_sec*NSEC_PER_SEC + ts.tv_nsec;
3549 +
3550 + return val;
3551 +}
3552 +
3553 +extern void v7_flush_kern_dcache_area(void *, size_t);
3554 +extern void v7_flush_kern_cache_all(void);
3555 +/*
3556 + * Ensure that this page is not in the L1 or L2 cache.
3557 + * Since the L1 cache is VIPT and the L2 cache is PIPT, we can use either the
3558 + * kernel or user vaddr.
3559 + */
3560 +void color_flush_page(void *vaddr, size_t size)
3561 +{
3562 + v7_flush_kern_dcache_area(vaddr, size);
3563 + //v7_flush_kern_cache_all();
3564 +}
3565 +
3566 +extern struct page* get_colored_page(unsigned long color);
3567 +
3568 +int setup_flusher_array(void)
3569 +{
3570 + int color, way, ret = 0;
3571 + struct page *page;
3572 +
3573 + if (flusher_pages != NULL)
3574 + goto out;
3575 +
3576 + flusher_pages = (void***) kmalloc(MAX_NR_WAYS
3577 + * sizeof(*flusher_pages), GFP_KERNEL);
3578 + if (!flusher_pages) {
3579 + printk(KERN_WARNING "No memory for flusher array!\n");
3580 + ret = -EINVAL;
3581 + goto out;
3582 + }
3583 + for (way = 0; way < MAX_NR_WAYS; way++) {
3584 + void **flusher_color_arr;
3585 + flusher_color_arr = (void**) kmalloc(sizeof(**flusher_pages)
3586 + * MAX_NR_COLORS, GFP_KERNEL);
3587 + if (!flusher_color_arr) {
3588 + printk(KERN_WARNING "No memory for flusher array!\n");
3589 + ret = -ENOMEM;
3590 + goto out_free;
3591 + }
3592 +
3593 + flusher_pages[way] = flusher_color_arr;
3594 + for (color = 0; color < MAX_NR_COLORS; color++) {
3595 + int node;
3596 + node = color + 112; // populate from bank 7
3597 + page = get_colored_page(node);
3598 + if (!page) {
3599 + printk(KERN_WARNING "no more colored pages\n");
3600 + ret = -EINVAL;
3601 + goto out_free;
3602 + }
3603 + flusher_pages[way][color] = page_address(page);
3604 + if (!flusher_pages[way][color]) {
3605 + printk(KERN_WARNING "bad page address\n");
3606 + ret = -EINVAL;
3607 + goto out_free;
3608 + }
3609 + }
3610 + }
3611 +
3612 +out:
3613 + return ret;
3614 +out_free:
3615 + for (way = 0; way < MAX_NR_WAYS; way++) {
3616 + for (color = 0; color < MAX_NR_COLORS; color++) {
3617 + /* not bothering to try and give back colored pages */
3618 + }
3619 + kfree(flusher_pages[way]);
3620 + }
3621 + kfree(flusher_pages);
3622 + flusher_pages = NULL;
3623 + return ret;
3624 +}
3625 +
3626 +void flush_cache(int all)
3627 +{
3628 + int way, color, cpu;
3629 + unsigned long flags;
3630 +
3631 + raw_spin_lock_irqsave(&cache_lock, flags);
3632 + cpu = raw_smp_processor_id();
3633 +
3634 + prev_lbm_i_reg[cpu] = readl_relaxed(ld_i_reg(cpu));
3635 + prev_lbm_d_reg[cpu] = readl_relaxed(ld_d_reg(cpu));
3636 + for (way=0;way<MAX_NR_WAYS;way++) {
3637 + if (( (0x00000001 << way) & (prev_lbm_d_reg[cpu]) ) &&
3638 + !all)
3639 + continue;
3640 + for (color=0;color<MAX_NR_COLORS;color++) {
3641 + void *vaddr = flusher_pages[way][color];
3642 + u32 lvalue = unlocked_way[way];
3643 + color_read_in_mem_lock(lvalue, LOCK_ALL,
3644 + vaddr, vaddr + PAGE_SIZE);
3645 + }
3646 +
3647 + }
3648 +
3649 + writel_relaxed(prev_lbm_i_reg[cpu], ld_i_reg(cpu));
3650 + writel_relaxed(prev_lbm_d_reg[cpu], ld_d_reg(cpu));
3651 + raw_spin_unlock_irqrestore(&cache_lock, flags);
3652 +}
3653 +
3654 +/* src = shared, dst = local */
3655 +#if 1 // random
3656 +asmlinkage long sys_run_test(int type, int size, cacheline_t *src, cacheline_t *dst, lt_t __user *ts)
3657 +{
3658 + /* size is in KB */
3659 + long ret = 0;
3660 + lt_t t1, t2;
3661 + int numlines = size * CACHELINES_IN_1KB;
3662 + int next, sum = 0, ran;
3663 + unsigned long flags;
3664 +
3665 + get_random_bytes(&ran, sizeof(int));
3666 + next = ran % ((size*1024)/sizeof(cacheline_t));
3667 +
3668 + //preempt_disable();
3669 + if (type == 1) {
3670 + int i, j;
3671 + color_read_in_mem_lock(0x0000FFF0, 0x0000000f, (void*)src, (void*)src + size*1024);
3672 + color_read_in_mem_lock(0x0000FF0F, 0x0000000f, (void*)dst, (void*)dst + size*1024);
3673 +
3674 + local_irq_save(flags);
3675 + t1 = litmus_clock();
3676 + for (i = 0; i < numlines; i++) {
3677 + next = src[next].line[0];
3678 + for (j = 1; j < INTS_IN_CACHELINE; j++) {
3679 + //dst[next].line[j] = src[next].line[j]; // read
3680 + src[next].line[j] = dst[next].line[j]; // write
3681 + }
3682 + }
3683 + t2 = litmus_clock();
3684 + local_irq_restore(flags);
3685 + sum = next + (int)t2;
3686 + t2 -= t1;
3687 + ret = put_user(t2, ts);
3688 + }
3689 + else {
3690 + int i, j;
3691 + color_read_in_mem_lock(0x0000FF0F, 0x0000000f, (void*)dst, (void*)dst + size*1024);
3692 + local_irq_save(flags);
3693 + t1 = litmus_clock();
3694 + for (i = 0; i < numlines; i++) {
3695 + next = src[next].line[0];
3696 + for (j = 1; j < INTS_IN_CACHELINE; j++) {
3697 + //dst[next].line[j] = src[next].line[j]; //read
3698 + src[next].line[j] = dst[next].line[j]; //write
3699 + }
3700 + }
3701 + t2 = litmus_clock();
3702 + local_irq_restore(flags);
3703 + sum = next + (int)t2;
3704 + t2 -= t1;
3705 + ret = put_user(t2, ts);
3706 + v7_flush_kern_dcache_area(src, size*1024);
3707 + }
3708 + //preempt_enable();
3709 + flush_cache(1);
3710 +
3711 + return ret;
3712 +}
3713 +#else
3714 +// sequential
3715 +asmlinkage long sys_run_test(int type, int size, cacheline_t *src, cacheline_t *dst, lt_t __user *ts)
3716 +{
3717 + /* size is in KB */
3718 + long ret = 0;
3719 + lt_t t1, t2;
3720 + int numlines = size * CACHELINES_IN_1KB;
3721 + int sum = 0;
3722 + unsigned long flags;
3723 +
3724 + //preempt_disable();
3725 + if (type == 1) {
3726 + int i, j;
3727 + color_read_in_mem_lock(0x0000FFF0, 0x0000000f, (void*)src, (void*)src + size*1024);
3728 + color_read_in_mem_lock(0x0000FF0F, 0x0000000f, (void*)dst, (void*)dst + size*1024);
3729 +
3730 + local_irq_save(flags);
3731 + t1 = litmus_clock();
3732 + for (i = 0; i < numlines; i++) {
3733 + for (j = 0; j < INTS_IN_CACHELINE; j++) {
3734 + //dst[i].line[j] = src[i].line[j]; // read
3735 + src[i].line[j] = dst[i].line[j]; // write
3736 + }
3737 + }
3738 + t2 = litmus_clock();
3739 + local_irq_restore(flags);
3740 + sum = (int)(t1 + t2);
3741 + t2 -= t1;
3742 + ret = put_user(t2, ts);
3743 + }
3744 + else {
3745 + int i, j;
3746 + color_read_in_mem_lock(0x0000FF0F, 0x0000000f, (void*)dst, (void*)dst + size*1024);
3747 + local_irq_save(flags);
3748 + t1 = litmus_clock();
3749 + for (i = 0; i < numlines; i++) {
3750 + for (j = 0; j < INTS_IN_CACHELINE; j++) {
3751 + //dst[i].line[j] = src[i].line[j]; //read
3752 + src[i].line[j] = dst[i].line[j]; //write
3753 + }
3754 + }
3755 + t2 = litmus_clock();
3756 + local_irq_restore(flags);
3757 + sum = (int)(t1 + t2);
3758 + t2 -= t1;
3759 + ret = put_user(t2, ts);
3760 + v7_flush_kern_dcache_area(src, size*1024);
3761 + }
3762 + //preempt_enable();
3763 + flush_cache(1);
3764 +
3765 + return ret;
3766 +}
3767 +#endif
3768 +
3769 +asmlinkage long sys_lock_buffer(void* vaddr, size_t size, u32 lock_way, u32 unlock_way)
3770 +{
3771 + /* size is in bytes */
3772 + long ret = 0;
3773 + int i;
3774 + u32 lock_val, unlock_val;
3775 +
3776 + lock_val = ~lock_way & 0x0000ffff;
3777 + unlock_val = ~unlock_way & 0x0000ffff;
3778 + color_read_in_mem_lock(lock_val, unlock_val, (void*)vaddr, (void*)vaddr + size);
3779 +
3780 + return ret;
3781 +}
3782 +
3783 +#define TRIALS 1000
3784 +
3785 +static int perf_test(void) {
3786 + struct timespec before, after;
3787 + struct page *page;
3788 + void *vaddr;
3789 + u32 *data;
3790 + long time, flush_time;
3791 + int i, num_pages = 1;
3792 + unsigned int order = 4;
3793 +
3794 + for (i = 0; i < order; i++) {
3795 + num_pages = num_pages*2;
3796 + }
3797 +
3798 + printk("Number of pages: %d\n", num_pages);
3799 + //page = alloc_page(__GFP_MOVABLE);
3800 + page = alloc_pages(__GFP_MOVABLE, order);
3801 + if (!page) {
3802 + printk(KERN_WARNING "No memory\n");
3803 + return -ENOMEM;
3804 + }
3805 +
3806 + vaddr = page_address(page);
3807 + if (!vaddr)
3808 + printk(KERN_WARNING "%s: vaddr is null\n", __FUNCTION__);
3809 + data = (u32*) vaddr;
3810 +
3811 + getnstimeofday(&before);
3812 + barrier();
3813 + for (i = 0; i < TRIALS; i++) {
3814 + color_flush_page(vaddr, PAGE_SIZE*num_pages);
3815 + }
3816 + barrier();
3817 + getnstimeofday(&after);
3818 + time = update_timeval(before, after);
3819 + printk("Average for flushes without re-reading: %ld\n", time / TRIALS);
3820 + flush_time = time / TRIALS;
3821 +
3822 + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3823 +
3824 + barrier();
3825 + getnstimeofday(&before);
3826 + barrier();
3827 + for (i = 0; i < TRIALS; i++) {
3828 + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3829 + }
3830 + barrier();
3831 + getnstimeofday(&after);
3832 + time = update_timeval(before, after);
3833 + printk("Average for read from cache: %ld\n", time / TRIALS);
3834 +
3835 + getnstimeofday(&before);
3836 + barrier();
3837 + for (i = 0; i < TRIALS; i++) {
3838 + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3839 + color_flush_page(vaddr, PAGE_SIZE*num_pages);
3840 + }
3841 + barrier();
3842 + getnstimeofday(&after);
3843 + time = update_timeval(before, after);
3844 + printk("Average for read from mem: %ld (%ld)\n", time / TRIALS - flush_time, time / TRIALS);
3845 +
3846 + // write in locked way
3847 + color_read_in_mem_lock(nr_unlocked_way[2], LOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3848 + for (i = 0; i < PAGE_SIZE*num_pages/sizeof(u32); i++) {
3849 + data[i] = i%63353;
3850 + }
3851 + // read
3852 + barrier();
3853 + getnstimeofday(&before);
3854 + barrier();
3855 + for (i = 0; i < TRIALS; i++) {
3856 + color_read_in_mem(unlocked_way[0], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages);
3857 + }
3858 + barrier();
3859 + getnstimeofday(&after);
3860 + time = update_timeval(before, after);
3861 + printk("Average for read in after write: %ld\n", time / TRIALS);
3862 +
3863 +
3864 + //free_page((unsigned long)vaddr);
3865 + free_pages((unsigned long)vaddr, order);
3866 +
3867 + return 0;
3868 +}
3869 +
3870 +int do_perf_test_proc_handler(struct ctl_table *table, int write,
3871 + void __user *buffer, size_t *lenp, loff_t *ppos)
3872 +{
3873 + int ret = 0;
3874 +
3875 + if (write) {
3876 + ret = perf_test();
3877 + }
3878 +
3879 + return ret;
3880 +}
3881 +
3882 +int setup_flusher_proc_handler(struct ctl_table *table, int write,
3883 + void __user *buffer, size_t *lenp, loff_t *ppos)
3884 +{
3885 + int ret = -EINVAL;
3886 +
3887 + if (write && flusher_pages == NULL) {
3888 + ret = setup_flusher_array();
3889 + printk(KERN_INFO "setup flusher return: %d\n", ret);
3890 +
3891 + }
3892 + else if (flusher_pages) {
3893 + printk(KERN_INFO "flusher_pages is already set!\n");
3894 + ret = 0;
3895 + }
3896 +
3897 + return ret;
3898 +}
3899 +
3900 +static struct ctl_table_header *litmus_sysctls;
3901 +
3902 +static int __init litmus_sysctl_init(void)
3903 +{
3904 + int ret = 0;
3905 +
3906 + printk(KERN_INFO "Registering LITMUS^RT proc sysctl.\n");
3907 + litmus_sysctls = register_sysctl_table(litmus_dir_table);
3908 + if (!litmus_sysctls) {
3909 + printk(KERN_WARNING "Could not register LITMUS^RT sysctl.\n");
3910 + ret = -EFAULT;
3911 + goto out;
3912 + }
3913 +
3914 + way_partition_min = 0x00000000;
3915 + way_partition_max = 0x0000FFFF;
3916 +
3917 +out:
3918 + return ret;
3919 +}
3920 +
3921 +module_init(litmus_sysctl_init);
3922 diff --git litmus/color_shm.c litmus/color_shm.c
3923 new file mode 100644
3924 index 0000000..7d340f32
3925 --- /dev/null
3926 +++ litmus/color_shm.c
3927 @@ -0,0 +1,402 @@
3928 +#include <linux/sched.h>
3929 +#include <linux/mm.h>
3930 +#include <linux/fs.h>
3931 +#include <linux/miscdevice.h>
3932 +#include <linux/spinlock.h>
3933 +#include <linux/module.h>
3934 +#include <linux/highmem.h>
3935 +#include <linux/slab.h>
3936 +#include <linux/mutex.h>
3937 +#include <asm/uaccess.h>
3938 +
3939 +#include <litmus/litmus.h>
3940 +
3941 +#define DEV_NAME "litmus/color_shm"
3942 +
3943 +/* Major number assigned to our device.
3944 + * Refer Documentation/devices.txt */
3945 +#define SHM_MAJOR 240
3946 +#define MAX_COLORED_PAGE 256
3947 +#define NUM_BANKS 8
3948 +#define NUM_COLORS 16
3949 +
3950 +static struct mutex dev_lock;
3951 +static int bypass_cache;
3952 +
3953 +struct color_ioctl_cmd {
3954 + unsigned int color;
3955 + unsigned int bank;
3956 +};
3957 +
3958 +struct color_ioctl_offset {
3959 + unsigned long offset;
3960 + int lock;
3961 +};
3962 +
3963 +#define SET_COLOR_SHM_CMD _IOW(SHM_MAJOR, 0x1, struct color_ioctl_cmd)
3964 +#define SET_COLOR_SHM_OFFSET _IOW(SHM_MAJOR, 0x2, struct color_ioctl_offset)
3965 +
3966 +struct color_ioctl_cmd color_param;
3967 +struct color_ioctl_offset color_offset;
3968 +
3969 +static int mmap_common_checks(struct vm_area_struct *vma)
3970 +{
3971 + /* you can only map the "first" page */
3972 + if (vma->vm_pgoff != 0)
3973 + return -EINVAL;
3974 +
3975 + return 0;
3976 +}
3977 +
3978 +static void mmap_common_vma_flags(struct vm_area_struct *vma)
3979 +{
3980 + /* This mapping should not be kept across forks,
3981 + * cannot be expanded, and is not a "normal" page. */
3982 + //vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_IO | VM_SHARED | VM_MAYSHARE;
3983 + vma->vm_flags |= VM_SHARED | VM_MAYSHARE | VM_LOCKED;
3984 +
3985 + /* We don't want the first write access to trigger a "minor" page fault
3986 + * to mark the page as dirty. This is transient, private memory, we
3987 + * don't care if it was touched or not. __S011 means RW access, but not
3988 + * execute, and avoids copy-on-write behavior.
3989 + * See protection_map in mmap.c. */
3990 + vma->vm_page_prot = PAGE_SHARED;
3991 +}
3992 +
3993 +#define vma_nr_pages(vma) \
3994 + ({unsigned long v = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); v;})
3995 +
3996 +extern struct page* get_colored_page(unsigned long color);
3997 +
3998 +static int do_map_colored_page(struct vm_area_struct *vma,
3999 + const unsigned long addr,
4000 + const unsigned long color_no)
4001 +{
4002 + int err = 0;
4003 + unsigned long offset = 2048;
4004 +
4005 + struct page *page = get_colored_page(color_no);
4006 +
4007 + if (!page) {
4008 + printk(KERN_INFO "Could not get page with color %lu.\n",
4009 + color_no);
4010 + err = -ENOMEM;
4011 + goto out;
4012 + }
4013 +
4014 + printk(KERN_INFO "vma: %p addr: 0x%lx color_no: %lu\n",
4015 + vma, addr, color_no);
4016 +
4017 + printk(KERN_INFO "vm_start: %lu vm_end: %lu\n",
4018 + vma->vm_start, vma->vm_end);
4019 +
4020 + printk(KERN_INFO "inserting page (pa: 0x%lx) at vaddr: 0x%lx "
4021 + "flags: 0x%lx prot: 0x%lx\n",
4022 + page_to_phys(page), addr,
4023 + vma->vm_flags, pgprot_val(vma->vm_page_prot));
4024 +
4025 +
4026 + err = vm_insert_page(vma, addr, page);
4027 + if (err) {
4028 + printk(KERN_INFO "vm_insert_page() failed (%d)\n", err);
4029 + err = -EINVAL;
4030 + goto out;
4031 + }
4032 +out:
4033 + return err;
4034 +}
4035 +
4036 +static int do_map_colored_pages(struct vm_area_struct *vma)
4037 +{
4038 + const unsigned long nr_pages = vma_nr_pages(vma);
4039 + unsigned long nr_mapped;
4040 + int i, start_bank = -1, start_color = -1;
4041 + int cur_bank = -1, cur_color = -1, err = 0;
4042 + int colors[16] = {0}, banks[8] = {0};
4043 +
4044 + if (bypass_cache == 1)
4045 + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
4046 +
4047 + for (i = 0; i < NUM_BANKS; i++) {
4048 + if (((color_param.bank >> i)&0x1) == 1)
4049 + banks[i] = 1;
4050 + }
4051 +
4052 + for (i = 0; i < NUM_COLORS; i++) {
4053 + if (((color_param.color >> i)&0x1) == 1)
4054 + colors[i] = 1;
4055 + }
4056 +
4057 + for (i = 0; i < NUM_BANKS; i++) {
4058 + if (banks[i] == 1) {
4059 + start_bank = i;
4060 + break;
4061 + }
4062 + }
4063 + for (i = 0; i < NUM_COLORS; i++) {
4064 + if (colors[i] == 1) {
4065 + start_color = i;
4066 + break;
4067 + }
4068 + }
4069 +
4070 + cur_bank = start_bank;
4071 + cur_color = start_color;
4072 +
4073 + for (i = 0; i < NUM_BANKS; i++) {
4074 + printk(KERN_INFO "BANK[%d] = %d\n", i, banks[i]);
4075 + }
4076 + printk(KERN_INFO "cur_bank = %d\n", cur_bank);
4077 + for (i = 0; i < NUM_COLORS; i++) {
4078 + printk(KERN_INFO "COLOR[%d] = %d\n", i, colors[i]);
4079 + }
4080 + printk(KERN_INFO "cur_color = %d\n", cur_color);
4081 +
4082 +
4083 + TRACE_CUR("allocating %lu pages (flags:%lx prot:%lx)\n",
4084 + nr_pages, vma->vm_flags, pgprot_val(vma->vm_page_prot));
4085 +
4086 + for (nr_mapped = 0; nr_mapped < nr_pages; nr_mapped++) {
4087 + const unsigned long addr = vma->vm_start + (nr_mapped << PAGE_SHIFT);
4088 + const unsigned long color_no = cur_bank*NUM_COLORS + cur_color;
4089 +
4090 + err = do_map_colored_page(vma, addr, color_no);
4091 + printk(KERN_INFO "mapped bank[%d], color[%d], color_no = %lu at 0x%lx\n",
4092 + cur_bank, cur_color, color_no, addr);
4093 + if (err) {
4094 + TRACE_CUR("Could not map colored page set.\n");
4095 + err = -EINVAL;
4096 + goto out;
4097 + }
4098 + do {
4099 + cur_color++;
4100 + } while(colors[cur_color] == 0);
4101 +
4102 + if (cur_color >= NUM_COLORS) {
4103 + do {
4104 + cur_bank++;
4105 + } while(banks[cur_bank] == 0);
4106 + cur_color = start_color;
4107 + }
4108 +
4109 + if (cur_bank >= NUM_BANKS) {
4110 + cur_bank = start_bank;
4111 + }
4112 + }
4113 + TRACE_CUR("Successfully mapped %lu pages.\n", nr_mapped);
4114 + out:
4115 + return err;
4116 +}
4117 +
4118 +static int map_colored_pages(struct vm_area_struct *vma)
4119 +{
4120 + int err = 0;
4121 +
4122 + printk(KERN_INFO "User requests %lu pages.\n", vma_nr_pages(vma));
4123 + if (MAX_COLORED_PAGE < vma_nr_pages(vma)) {
4124 + TRACE_CUR("Max page request %lu but want %lu.\n",
4125 + MAX_COLORED_PAGE, vma_nr_pages(vma));
4126 + err = -EINVAL;
4127 + goto out;
4128 + }
4129 + err = do_map_colored_pages(vma);
4130 +out:
4131 + return err;
4132 +}
4133 +
4134 +static void litmus_color_shm_vm_close(struct vm_area_struct *vma)
4135 +{
4136 +
4137 + TRACE_CUR("flags=0x%lx prot=0x%lx\n",
4138 + vma->vm_flags, pgprot_val(vma->vm_page_prot));
4139 +
4140 + TRACE_CUR("%p:%p vma:%p vma->vm_private_data:%p closed.\n",
4141 + (void*) vma->vm_start, (void*) vma->vm_end, vma,
4142 + vma->vm_private_data);
4143 +
4144 +}
4145 +
4146 +static int litmus_color_shm_vm_fault(struct vm_area_struct *vma,
4147 + struct vm_fault *vmf)
4148 +{
4149 + /* This function should never be called, since
4150 + * all pages should have been mapped by mmap()
4151 + * already. */
4152 + TRACE_CUR("flags=0x%lx (off:%ld)\n", vma->vm_flags, vmf->pgoff);
4153 + printk(KERN_INFO "flags=0x%lx (off:%ld)\n", vma->vm_flags, vmf->pgoff);
4154 +
4155 + printk(KERN_INFO "Page fault in color ctrl page! prot=0x%lx\n", pgprot_val(vma->vm_page_prot));
4156 +
4157 + return VM_FAULT_SIGBUS;
4158 +}
4159 +
4160 +static struct vm_operations_struct litmus_color_shm_vm_ops = {
4161 + .close = litmus_color_shm_vm_close,
4162 + .fault = litmus_color_shm_vm_fault,
4163 +};
4164 +
4165 +static int litmus_color_shm_mmap(struct file *filp, struct vm_area_struct *vma)
4166 +{
4167 + int err = 0;
4168 +
4169 + printk(KERN_INFO "mmap called\n");
4170 +
4171 + if (color_param.color == 0x00000000 || color_param.bank == 0x00000000) {
4172 + printk(KERN_INFO "color_info not set.\n");
4173 + return -EINVAL;
4174 + }
4175 + if (color_offset.offset == 0xffffffff || color_offset.lock == -1) {
4176 + printk(KERN_INFO "color_offset not set.\n");
4177 + return -EINVAL;
4178 + }
4179 +
4180 + err = mmap_common_checks(vma);
4181 + if (err) {
4182 + TRACE_CUR("failed mmap common checks\n");
4183 + goto out;
4184 + }
4185 +
4186 + vma->vm_ops = &litmus_color_shm_vm_ops;
4187 + mmap_common_vma_flags(vma);
4188 +
4189 + err = map_colored_pages(vma);
4190 +
4191 + TRACE_CUR("flags=0x%lx prot=0x%lx\n", vma->vm_flags,
4192 + pgprot_val(vma->vm_page_prot));
4193 +out:
4194 + color_param.color = 0x00000000;
4195 + color_param.bank = 0x00000000;
4196 + color_offset.offset = 0xffffffff;
4197 + color_offset.lock = -1;
4198 +
4199 + return err;
4200 +
4201 +}
4202 +
4203 +static long litmus_color_shm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
4204 +{
4205 + long err = -ENOIOCTLCMD;
4206 + struct color_ioctl_cmd color_info;
4207 + struct color_ioctl_offset color_off;
4208 +
4209 + printk(KERN_INFO "color_shm ioctl\n");
4210 +
4211 + if (_IOC_TYPE(cmd) != SHM_MAJOR)
4212 + return -ENOTTY;
4213 +
4214 +
4215 + switch (cmd) {
4216 + case SET_COLOR_SHM_CMD:
4217 +
4218 + err = copy_from_user(&color_info, (void*)arg, sizeof(struct color_ioctl_cmd));
4219 +
4220 + color_param.color = color_info.color;
4221 + color_param.bank = color_info.bank;
4222 + printk(KERN_INFO "COLOR = %x\n", color_param.color);
4223 + printk(KERN_INFO "BANK = %x\n", color_param.bank);
4224 + err = 0;
4225 + break;
4226 + case SET_COLOR_SHM_OFFSET:
4227 + err = copy_from_user(&color_off, (void*)arg, sizeof(struct color_ioctl_offset));
4228 +
4229 + color_offset.offset = color_off.offset;
4230 + color_offset.lock = color_off.lock;
4231 + printk(KERN_INFO "OFFSET = %x\n", color_offset.offset);
4232 + printk(KERN_INFO "LOCK = %d\n", color_offset.lock);
4233 + err = 0;
4234 + break;
4235 +
4236 + default:
4237 + printk(KERN_INFO "Invalid IOCTL CMD\n");
4238 + err = -EINVAL;
4239 + }
4240 +
4241 + return err;
4242 +}
4243 +
4244 +static struct file_operations litmus_color_shm_fops = {
4245 + .owner = THIS_MODULE,
4246 + .mmap = litmus_color_shm_mmap,
4247 + .unlocked_ioctl = litmus_color_shm_ioctl,
4248 +};
4249 +
4250 +static struct miscdevice litmus_color_shm_dev = {
4251 + .name = DEV_NAME,
4252 + .minor = MISC_DYNAMIC_MINOR,
4253 + .fops = &litmus_color_shm_fops,
4254 +};
4255 +
4256 +struct mutex bypass_mutex;
4257 +
4258 +int bypass_proc_handler(struct ctl_table *table, int write,
4259 + void __user *buffer, size_t *lenp, loff_t *ppos)
4260 +{
4261 + int ret, mode;
4262 +
4263 + mutex_lock(&bypass_mutex);
4264 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
4265 + printk(KERN_INFO "shm_bypass = %d\n", bypass_cache);
4266 + mutex_unlock(&bypass_mutex);
4267 +
4268 + return ret;
4269 +}
4270 +
4271 +static int zero = 0;
4272 +static int one = 1;
4273 +
4274 +static struct ctl_table cache_table[] =
4275 +{
4276 + {
4277 + .procname = "shm_bypass",
4278 + .mode = 0666,
4279 + .proc_handler = bypass_proc_handler,
4280 + .data = &bypass_cache,
4281 + .maxlen = sizeof(bypass_cache),
4282 + .extra1 = &zero,
4283 + .extra2 = &one,
4284 + },
4285 + { }
4286 +};
4287 +
4288 +static struct ctl_table litmus_dir_table[] = {
4289 + {
4290 + .procname = "litmus",
4291 + .mode = 0555,
4292 + .child = cache_table,
4293 + },
4294 + { }
4295 +};
4296 +
4297 +static struct ctl_table_header *litmus_sysctls;
4298 +
4299 +static int __init init_color_shm_devices(void)
4300 +{
4301 + int err;
4302 +
4303 + printk(KERN_INFO "Registering LITMUS^RT color_shm devices.\n");
4304 + litmus_sysctls = register_sysctl_table(litmus_dir_table);
4305 + if (!litmus_sysctls) {
4306 + printk(KERN_WARNING "Could not register LITMUS^RT color_shm sysctl.\n");
4307 + err = -EFAULT;
4308 + }
4309 +
4310 + mutex_init(&dev_lock);
4311 + mutex_init(&bypass_mutex);
4312 + color_param.color = 0x00000000;
4313 + color_param.bank = 0x00000000;
4314 + color_offset.offset = 0xffffffff;
4315 + color_offset.lock = -1;
4316 + bypass_cache = 0;
4317 + err = misc_register(&litmus_color_shm_dev);
4318 +
4319 + return err;
4320 +}
4321 +
4322 +static void __exit exit_color_shm_devices(void)
4323 +{
4324 + misc_deregister(&litmus_color_shm_dev);
4325 + printk(KERN_INFO "Unregistering %s device.\n", DEV_NAME);
4326 +}
4327 +
4328 +module_init(init_color_shm_devices);
4329 +module_exit(exit_color_shm_devices);
4330 \ No newline at end of file
4331 diff --git litmus/fakedev0.c litmus/fakedev0.c
4332 new file mode 100644
4333 index 0000000..9a448bf
4334 --- /dev/null
4335 +++ litmus/fakedev0.c
4336 @@ -0,0 +1,123 @@
4337 +#include <linux/sched.h>
4338 +#include <linux/kernel.h>
4339 +#include <linux/mm.h>
4340 +#include <linux/fs.h>
4341 +#include <linux/errno.h>
4342 +#include <linux/highmem.h>
4343 +#include <asm/page.h>
4344 +#include <linux/miscdevice.h>
4345 +#include <linux/module.h>
4346 +
4347 +#include <litmus/litmus.h>
4348 +
4349 +/* device for allocating pages not cached by the CPU */
4350 +
4351 +#define FAKEDEV0_NAME "litmus/fakedev0"
4352 +
4353 +#define NUM_BANKS 8
4354 +#define BANK_MASK 0x38000000
4355 +#define BANK_SHIFT 27
4356 +
4357 +#define NUM_COLORS 16
4358 +#define CACHE_MASK 0x0000f000
4359 +#define CACHE_SHIFT 12
4360 +
4361 +/* Decoding page color, 0~15 */
4362 +static inline unsigned int page_color(struct page *page)
4363 +{
4364 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
4365 +}
4366 +
4367 +/* Decoding page bank number, 0~7 */
4368 +static inline unsigned int page_bank(struct page *page)
4369 +{
4370 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
4371 +}
4372 +
4373 +void litmus_fakedev0_vm_open(struct vm_area_struct *vma)
4374 +{
4375 +}
4376 +
4377 +void litmus_fakedev0_vm_close(struct vm_area_struct *vma)
4378 +{
4379 +}
4380 +
4381 +int litmus_fakedev0_vm_fault(struct vm_area_struct* vma,
4382 + struct vm_fault* vmf)
4383 +{
4384 + /* modeled after SG DMA video4linux, but without DMA. */
4385 + /* (see drivers/media/video/videobuf-dma-sg.c) */
4386 + struct page *page;
4387 +
4388 + page = alloc_page(GFP_USER|GFP_COLOR);
4389 + if (!page)
4390 + return VM_FAULT_OOM;
4391 +
4392 + clear_user_highpage(page, (unsigned long)vmf->virtual_address);
4393 + vmf->page = page;
4394 +
4395 + return 0;
4396 +}
4397 +
4398 +static struct vm_operations_struct litmus_fakedev0_vm_ops = {
4399 + .open = litmus_fakedev0_vm_open,
4400 + .close = litmus_fakedev0_vm_close,
4401 + .fault = litmus_fakedev0_vm_fault,
4402 +};
4403 +
4404 +static int litmus_fakedev0_mmap(struct file* filp, struct vm_area_struct* vma)
4405 +{
4406 + /* first make sure mapper knows what he's doing */
4407 +
4408 + /* you can only map the "first" page */
4409 + if (vma->vm_pgoff != 0)
4410 + return -EINVAL;
4411 +
4412 + /* you can't share it with anyone */
4413 + if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED))
4414 + return -EINVAL;
4415 +
4416 + /* cannot be expanded, and is not a "normal" page. */
4417 + vma->vm_flags |= (VM_DONTEXPAND);
4418 +
4419 + /* noncached pages are not explicitly locked in memory (for now). */
4420 + //vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
4421 +
4422 + vma->vm_ops = &litmus_fakedev0_vm_ops;
4423 +
4424 + return 0;
4425 +}
4426 +
4427 +static struct file_operations litmus_fakedev0_fops = {
4428 + .owner = THIS_MODULE,
4429 + .mmap = litmus_fakedev0_mmap,
4430 +};
4431 +
4432 +static struct miscdevice litmus_fakedev0_dev = {
4433 + .name = FAKEDEV0_NAME,
4434 + .minor = MISC_DYNAMIC_MINOR,
4435 + .fops = &litmus_fakedev0_fops,
4436 + /* pages are not locked, so there is no reason why
4437 + anyone cannot allocate an fakedev0 pages */
4438 + .mode = (S_IRUGO | S_IWUGO),
4439 +};
4440 +
4441 +static int __init init_litmus_fakedev0_dev(void)
4442 +{
4443 + int err;
4444 +
4445 + printk("Initializing LITMUS^RT fakedev0 device.\n");
4446 + err = misc_register(&litmus_fakedev0_dev);
4447 + if (err)
4448 + printk("Could not allocate %s device (%d).\n", FAKEDEV0_NAME, err);
4449 + return err;
4450 +}
4451 +
4452 +static void __exit exit_litmus_fakedev0_dev(void)
4453 +{
4454 + misc_deregister(&litmus_fakedev0_dev);
4455 +}
4456 +
4457 +module_init(init_litmus_fakedev0_dev);
4458 +module_exit(exit_litmus_fakedev0_dev);
4459 +
4460 diff --git litmus/litmus.c litmus/litmus.c
4461 index db5ce0e9..6000826 100644
4462 --- litmus/litmus.c
4463 +++ litmus/litmus.c
4464 @@ -14,6 +14,10 @@
4465 #include <linux/sched/rt.h>
4466 #include <linux/rwsem.h>
4467 #include <linux/interrupt.h>
4468 +#include <linux/migrate.h>
4469 +#include <linux/mm.h>
4470 +#include <linux/memcontrol.h>
4471 +#include <linux/mm_inline.h>
4472
4473 #include <litmus/litmus.h>
4474 #include <litmus/bheap.h>
4475 @@ -21,6 +25,10 @@
4476 #include <litmus/rt_domain.h>
4477 #include <litmus/litmus_proc.h>
4478 #include <litmus/sched_trace.h>
4479 +#include <litmus/cache_proc.h>
4480 +#include <litmus/mc2_common.h>
4481 +#include <litmus/replicate_lib.h>
4482 +#include <litmus/page_dev.h>
4483
4484 #ifdef CONFIG_SCHED_CPU_AFFINITY
4485 #include <litmus/affinity.h>
4486 @@ -31,6 +39,8 @@
4487 #include <trace/events/litmus.h>
4488 #endif
4489
4490 +extern void l2c310_flush_all(void);
4491 +
4492 /* Number of RT tasks that exist in the system */
4493 atomic_t rt_task_count = ATOMIC_INIT(0);
4494
4495 @@ -160,6 +170,14 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param)
4496 pid, tp.budget_policy);
4497 goto out_unlock;
4498 }
4499 +#ifdef CONFIG_PGMRT_SUPPORT
4500 + if (tp.pgm_type < PGM_NOT_A_NODE || tp.pgm_type > PGM_INTERNAL) {
4501 + printk(KERN_INFO "litmus: real-time task %d rejected "
4502 + "because of unknown PGM node type specified (%d)\n",
4503 + pid, tp.pgm_type);
4504 + goto out_unlock;
4505 + }
4506 +#endif
4507
4508 target->rt_param.task_params = tp;
4509
4510 @@ -314,6 +332,275 @@ asmlinkage long sys_null_call(cycles_t __user *ts)
4511 return ret;
4512 }
4513
4514 +asmlinkage long sys_reservation_create(int type, void __user *config)
4515 +{
4516 + return litmus->reservation_create(type, config);
4517 +}
4518 +
4519 +asmlinkage long sys_reservation_destroy(unsigned int reservation_id, int cpu)
4520 +{
4521 + return litmus->reservation_destroy(reservation_id, cpu);
4522 +}
4523 +
4524 +static unsigned long color_mask;
4525 +
4526 +extern int isolate_lru_page(struct page *page);
4527 +extern void putback_movable_page(struct page *page);
4528 +extern struct page *new_alloc_page(struct page *page, unsigned long node, int **x);
4529 +
4530 +static struct page *alloc_colored_page(struct page *page, unsigned long node, int **result)
4531 +{
4532 + struct page *newpage;
4533 + gfp_t gfp_mask;
4534 +
4535 + gfp_mask = GFP_HIGHUSER_MOVABLE;
4536 + if (node != 8)
4537 + gfp_mask |= GFP_COLOR;
4538 +
4539 + newpage = alloc_pages(gfp_mask, 0);
4540 +
4541 + return newpage;
4542 +}
4543 +
4544 +#define INVALID_PFN (0xffffffff)
4545 +LIST_HEAD(shared_lib_pages);
4546 +
4547 +EXPORT_SYMBOL(shared_lib_pages);
4548 +
4549 +/* Reallocate pages of a task
4550 + * Private pages - Migrate to a new page.
4551 + * Shared pages - Use a replica. Make a replica if necessary.
4552 + * @cpu : CPU id of the calling task
4553 + * returns the number of pages that is not moved.
4554 + */
4555 +asmlinkage long sys_set_page_color(int cpu)
4556 +{
4557 + long ret = 0;
4558 + struct vm_area_struct *vma_itr = NULL;
4559 + int nr_pages = 0, nr_shared_pages = 0, nr_failed = 0, nr_not_migrated = 0;
4560 + unsigned long node;
4561 + enum crit_level lv;
4562 + struct mm_struct *mm;
4563 +
4564 + LIST_HEAD(pagelist);
4565 + LIST_HEAD(task_shared_pagelist);
4566 +
4567 + migrate_prep();
4568 +
4569 + /* Find the current mm_struct */
4570 + rcu_read_lock();
4571 + get_task_struct(current);
4572 + rcu_read_unlock();
4573 + mm = get_task_mm(current);
4574 + put_task_struct(current);
4575 +
4576 + down_read(&mm->mmap_sem);
4577 +
4578 + vma_itr = mm->mmap;
4579 + /* Iterate all vm_area_struct */
4580 + while (vma_itr != NULL) {
4581 + unsigned int num_pages = 0, i;
4582 + struct page *old_page = NULL;
4583 + int pages_in_vma = 0;
4584 +
4585 + num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE;
4586 + /* Traverse all pages in vm_area_struct */
4587 + for (i = 0; i < num_pages; i++) {
4588 + old_page = follow_page(vma_itr, vma_itr->vm_start + PAGE_SIZE*i, FOLL_GET|FOLL_SPLIT);
4589 +
4590 + if (IS_ERR(old_page))
4591 + continue;
4592 + if (!old_page)
4593 + continue;
4594 +
4595 + if (PageReserved(old_page)) {
4596 + TRACE("Reserved Page!\n");
4597 + put_page(old_page);
4598 + continue;
4599 + }
4600 +
4601 + TRACE_TASK(current, "addr: %08x, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-");
4602 + pages_in_vma++;
4603 +
4604 + /* Conditions for replicable pages */
4605 + if (page_count(old_page) > 2 && vma_itr->vm_file != NULL && !(vma_itr->vm_flags&VM_WRITE)) {
4606 + struct shared_lib_page *lib_page;
4607 + int is_exist = 0;
4608 +
4609 + /* Update PSL (Per-core shared library (master)) list */
4610 + /* Check if this page is in the PSL list */
4611 + rcu_read_lock();
4612 + list_for_each_entry(lib_page, &shared_lib_pages, list)
4613 + {
4614 + if (page_to_pfn(old_page) == lib_page->master_pfn) {
4615 + is_exist = 1;
4616 + break;
4617 + }
4618 + }
4619 + rcu_read_unlock();
4620 +
4621 + if (is_exist == 0) {
4622 + int cpu_i;
4623 + lib_page = kmalloc(sizeof(struct shared_lib_page), GFP_KERNEL);
4624 + lib_page->master_page = old_page;
4625 + lib_page->master_pfn = page_to_pfn(old_page);
4626 + for (cpu_i = 0; cpu_i < NR_CPUS+1; cpu_i++) {
4627 + lib_page->r_page[cpu_i] = NULL;
4628 + lib_page->r_pfn[cpu_i] = INVALID_PFN;
4629 + }
4630 + list_add_tail(&lib_page->list, &shared_lib_pages);
4631 + }
4632 +
4633 + /* add to task_shared_pagelist */
4634 + ret = isolate_lru_page(old_page);
4635 + if (!ret) {
4636 + list_add_tail(&old_page->lru, &task_shared_pagelist);
4637 + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page));
4638 + nr_shared_pages++;
4639 + } else {
4640 + TRACE_TASK(current, "isolate_lru_page for a shared page failed\n");
4641 + nr_failed++;
4642 + }
4643 + put_page(old_page);
4644 + }
4645 + else {
4646 + ret = isolate_lru_page(old_page);
4647 + if (!ret) {
4648 + list_add_tail(&old_page->lru, &pagelist);
4649 + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page));
4650 + nr_pages++;
4651 + } else if (!is_in_correct_bank(old_page, cpu)) {
4652 + TRACE_TASK(current, "isolate_lru_page for a private page failed\n");
4653 + nr_failed++;
4654 + } else {
4655 + TRACE_TASK(current, "page is already in the correct bank\n");
4656 + }
4657 + put_page(old_page);
4658 + }
4659 + }
4660 + TRACE_TASK(current, "PAGES_IN_VMA = %d size = %d KB\n", pages_in_vma, pages_in_vma*4);
4661 + vma_itr = vma_itr->vm_next;
4662 + }
4663 +
4664 + ret = 0;
4665 + lv = tsk_rt(current)->mc2_data->crit;
4666 + if (cpu == -1)
4667 + node = 8;
4668 + else
4669 + node = cpu*2 + lv;
4670 +
4671 + /* Migrate private pages */
4672 + if (!list_empty(&pagelist)) {
4673 + ret = migrate_pages(&pagelist, alloc_colored_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL);
4674 + TRACE_TASK(current, "%ld pages not migrated.\n", ret);
4675 + nr_not_migrated = ret;
4676 + if (ret) {
4677 + putback_movable_pages(&pagelist);
4678 + }
4679 + }
4680 +
4681 + /* Replicate shared pages */
4682 + if (!list_empty(&task_shared_pagelist)) {
4683 + ret = replicate_pages(&task_shared_pagelist, alloc_colored_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL);
4684 + TRACE_TASK(current, "%ld shared pages not migrated.\n", ret);
4685 + nr_not_migrated += ret;
4686 + if (ret) {
4687 + putback_movable_pages(&task_shared_pagelist);
4688 + }
4689 + }
4690 +
4691 + up_read(&mm->mmap_sem);
4692 +
4693 + TRACE_TASK(current, "nr_pages = %d nr_failed = %d nr_not_migrated = %d\n", nr_pages, nr_failed, nr_not_migrated);
4694 + printk(KERN_INFO "node = %ld, nr_private_pages = %d, nr_shared_pages = %d, nr_failed_to_isolate_lru = %d, nr_not_migrated = %d\n", node, nr_pages, nr_shared_pages, nr_failed, nr_not_migrated);
4695 +
4696 + return nr_not_migrated;
4697 +}
4698 +
4699 +#define BANK_MASK 0x38000000
4700 +#define BANK_SHIFT 27
4701 +#define CACHE_MASK 0x0000f000
4702 +#define CACHE_SHIFT 12
4703 +
4704 +/* Decoding page color, 0~15 */
4705 +static inline unsigned int page_color(struct page *page)
4706 +{
4707 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
4708 +}
4709 +
4710 +/* Decoding page bank number, 0~7 */
4711 +static inline unsigned int page_bank(struct page *page)
4712 +{
4713 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
4714 +}
4715 +
4716 +/* sys_test_call() is a test system call for debugging */
4717 +asmlinkage long sys_test_call(unsigned int param)
4718 +{
4719 + long ret = 0;
4720 + struct vm_area_struct *vma_itr = NULL;
4721 +
4722 + TRACE_CUR("test_call param = %d\n", param);
4723 +
4724 + /* if param == 0,
4725 + * show vm regions and the page frame numbers
4726 + * associated with the vm region.
4727 + * if param == 1,
4728 + * print the master list.
4729 + */
4730 + if (param == 0) {
4731 + down_read(¤t->mm->mmap_sem);
4732 + vma_itr = current->mm->mmap;
4733 + while (vma_itr != NULL) {
4734 + int i, num_pages;
4735 + struct page* old_page;
4736 + TRACE_TASK(current, "------------------------------------------------------\n");
4737 + TRACE_TASK(current, "vm_start : %lx\n", vma_itr->vm_start);
4738 + TRACE_TASK(current, "vm_end : %lx\n", vma_itr->vm_end);
4739 + TRACE_TASK(current, "vm_flags : %lx\n", vma_itr->vm_flags);
4740 + TRACE_TASK(current, "vm_prot : %x\n", pgprot_val(vma_itr->vm_page_prot));
4741 + TRACE_TASK(current, "VM_SHARED? %ld\n", vma_itr->vm_flags & VM_SHARED);
4742 +
4743 + num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE;
4744 + for (i = 0; i < num_pages; i++) {
4745 + old_page = follow_page(vma_itr, vma_itr->vm_start + PAGE_SIZE*i, FOLL_GET|FOLL_SPLIT);
4746 +
4747 + if (IS_ERR(old_page))
4748 + continue;
4749 + if (!old_page)
4750 + continue;
4751 +
4752 + if (PageReserved(old_page)) {
4753 + TRACE("Reserved Page!\n");
4754 + put_page(old_page);
4755 + continue;
4756 + }
4757 +
4758 + TRACE_TASK(current, "addr: %08x, phy: %08x, color: %d, bank: %d, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s mapping: %p\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_phys(old_page), page_color(old_page), page_bank(old_page), page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-", &(old_page->mapping));
4759 + printk(KERN_INFO "addr: %08x, phy: %08x, color: %d, bank: %d, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s mapping: %p\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_phys(old_page), page_color(old_page), page_bank(old_page), page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-", &(old_page->mapping));
4760 + put_page(old_page);
4761 + }
4762 + vma_itr = vma_itr->vm_next;
4763 + }
4764 + TRACE_TASK(current, "------------------------------------------------------\n");
4765 + up_read(¤t->mm->mmap_sem);
4766 + } else if (param == 1) {
4767 + TRACE_TASK(current, "Shared pages and replicas.\n");
4768 + {
4769 + struct shared_lib_page *lpage;
4770 +
4771 + rcu_read_lock();
4772 + list_for_each_entry(lpage, &shared_lib_pages, list)
4773 + {
4774 + TRACE_TASK(current, "master_PFN = %05lx r_PFN = %05lx, %05lx, %05lx, %05lx, %05lx\n", lpage->master_pfn, lpage->r_pfn[0], lpage->r_pfn[1], lpage->r_pfn[2], lpage->r_pfn[3], lpage->r_pfn[4]);
4775 + }
4776 + rcu_read_unlock();
4777 + }
4778 + }
4779 +
4780 + return ret;
4781 +}
4782 +
4783 /* p is a real-time task. Re-init its state as a best-effort task. */
4784 static void reinit_litmus_state(struct task_struct* p, int restore)
4785 {
4786 @@ -651,6 +938,12 @@ static int __init _init_litmus(void)
4787 * mode change lock is used to enforce single mode change
4788 * operation.
4789 */
4790 +#if defined(CONFIG_CPU_V7)
4791 + unsigned int line_size_log = 5; // 2^5 = 32 byte
4792 + unsigned int cache_info_sets = 2048; // 64KB (way_size) / 32B (line_size) = 2048
4793 + printk("LITMIS^RT-ARM kernel\n");
4794 +#endif
4795 +
4796 printk("Starting LITMUS^RT kernel\n");
4797
4798 register_sched_plugin(&linux_sched_plugin);
4799 @@ -665,11 +958,15 @@ static int __init _init_litmus(void)
4800 else
4801 printk("Could not register kill rt tasks magic sysrq.\n");
4802 #endif
4803 -
4804 init_litmus_proc();
4805
4806 register_reboot_notifier(&shutdown_notifier);
4807
4808 +#if defined(CONFIG_CPU_V7)
4809 + color_mask = ((cache_info_sets << line_size_log) - 1) ^ (PAGE_SIZE - 1);
4810 + printk("Page color mask %lx\n", color_mask);
4811 +#endif
4812 +
4813 return 0;
4814 }
4815
4816 diff --git litmus/mc2_common.c litmus/mc2_common.c
4817 new file mode 100644
4818 index 0000000..a8ea5d9
4819 --- /dev/null
4820 +++ litmus/mc2_common.c
4821 @@ -0,0 +1,78 @@
4822 +/*
4823 + * litmus/mc2_common.c
4824 + *
4825 + * Common functions for MC2 plugin.
4826 + */
4827 +
4828 +#include <linux/percpu.h>
4829 +#include <linux/sched.h>
4830 +#include <linux/list.h>
4831 +#include <linux/slab.h>
4832 +#include <asm/uaccess.h>
4833 +
4834 +#include <litmus/litmus.h>
4835 +#include <litmus/sched_plugin.h>
4836 +#include <litmus/sched_trace.h>
4837 +
4838 +#include <litmus/mc2_common.h>
4839 +
4840 +long mc2_task_client_init(struct task_client *tc, struct mc2_task *mc2_param, struct task_struct *tsk, struct reservation *res)
4841 +{
4842 + task_client_init(tc, tsk, res);
4843 + if ((mc2_param->crit < CRIT_LEVEL_A) ||
4844 + (mc2_param->crit > CRIT_LEVEL_C))
4845 + return -EINVAL;
4846 +
4847 + TRACE_TASK(tsk, "mc2_task_client_init: crit_level = %d\n", mc2_param->crit);
4848 +
4849 + return 0;
4850 +}
4851 +
4852 +asmlinkage long sys_set_mc2_task_param(pid_t pid, struct mc2_task __user * param)
4853 +{
4854 + struct task_struct *target;
4855 + int retval = -EINVAL;
4856 + struct mc2_task *mp = kzalloc(sizeof(*mp), GFP_KERNEL);
4857 +
4858 + if (!mp)
4859 + return -ENOMEM;
4860 +
4861 + printk("Setting up mc^2 task parameters for process %d.\n", pid);
4862 +
4863 + if (pid < 0 || param == 0) {
4864 + goto out;
4865 + }
4866 + if (copy_from_user(mp, param, sizeof(*mp))) {
4867 + retval = -EFAULT;
4868 + goto out;
4869 + }
4870 +
4871 + /* Task search and manipulation must be protected */
4872 + read_lock_irq(&tasklist_lock);
4873 + if (!(target = find_task_by_vpid(pid))) {
4874 + retval = -ESRCH;
4875 + goto out_unlock;
4876 + }
4877 +
4878 + if (is_realtime(target)) {
4879 + /* The task is already a real-time task.
4880 + * We cannot not allow parameter changes at this point.
4881 + */
4882 + retval = -EBUSY;
4883 + goto out_unlock;
4884 + }
4885 + if (mp->crit < CRIT_LEVEL_A || mp->crit >= NUM_CRIT_LEVELS) {
4886 + printk(KERN_INFO "litmus: real-time task %d rejected "
4887 + "because of invalid criticality level\n", pid);
4888 + goto out_unlock;
4889 + }
4890 +
4891 + //target->rt_param.plugin_state = mp;
4892 + target->rt_param.mc2_data = mp;
4893 +
4894 + retval = 0;
4895 +out_unlock:
4896 + read_unlock_irq(&tasklist_lock);
4897 +out:
4898 + return retval;
4899 +}
4900 \ No newline at end of file
4901 diff --git litmus/page_dev.c litmus/page_dev.c
4902 new file mode 100644
4903 index 0000000..e9baaaf
4904 --- /dev/null
4905 +++ litmus/page_dev.c
4906 @@ -0,0 +1,421 @@
4907 +/*
4908 + * page_dev.c - Implementation of the page coloring for cache and bank partition.
4909 + * The file will keep a pool of colored pages. MMU can allocate pages with
4910 + * specific color or bank number.
4911 + * Author: Namhoon Kim (namhoonk@cs.unc.edu)
4912 + */
4913 +
4914 +#include <litmus/page_dev.h>
4915 +#include <litmus/debug_trace.h>
4916 +
4917 +// This Address Decoding is used in imx6-sabredsd platform
4918 +#define NUM_BANKS 8
4919 +#define BANK_MASK 0x38000000
4920 +#define BANK_SHIFT 27
4921 +
4922 +#define NUM_COLORS 16
4923 +#define CACHE_MASK 0x0000f000
4924 +#define CACHE_SHIFT 12
4925 +
4926 +#define NR_LLC_PARTITIONS 9
4927 +#define NR_DRAM_PARTITIONS 5
4928 +
4929 +struct mutex dev_mutex;
4930 +
4931 +/* Initial partitions for LLC and DRAM bank */
4932 +/* 4 color for each core, all colors for Level C */
4933 +unsigned int llc_partition[NR_LLC_PARTITIONS] = {
4934 + 0x0000000f, /* Core 0, and Level A*/
4935 + 0x0000000f, /* Core 0, and Level B*/
4936 + 0x000000f0, /* Core 1, and Level A*/
4937 + 0x000000f0, /* Core 1, and Level B*/
4938 + 0x00000f00, /* Core 2, and Level A*/
4939 + 0x00000f00, /* Core 2, and Level B*/
4940 + 0x0000f000, /* Core 3, and Level A*/
4941 + 0x0000f000, /* Core 3, and Level B*/
4942 + 0x0000ffff, /* Level C */
4943 +};
4944 +
4945 +/* 1 bank for each core, 2 banks for Level C */
4946 +unsigned int dram_partition[NR_DRAM_PARTITIONS] = {
4947 + 0x00000010,
4948 + 0x00000020,
4949 + 0x00000040,
4950 + 0x00000080,
4951 + 0x0000000f,
4952 +};
4953 +
4954 +/* Decoding page color, 0~15 */
4955 +static inline unsigned int page_color(struct page *page)
4956 +{
4957 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
4958 +}
4959 +
4960 +/* Decoding page bank number, 0~7 */
4961 +static inline unsigned int page_bank(struct page *page)
4962 +{
4963 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
4964 +}
4965 +
4966 +int bank_to_partition(unsigned int bank)
4967 +{
4968 + int i;
4969 + unsigned int bank_bit = 0x1<<bank;
4970 +
4971 + for (i = 0; i<NR_DRAM_PARTITIONS; i++) {
4972 + if (dram_partition[i] & bank_bit)
4973 + return i;
4974 + }
4975 +
4976 + return -EINVAL;
4977 +}
4978 +
4979 +int get_area_index(int cpu)
4980 +{
4981 + int index = 0x10, area_index = 0;
4982 +
4983 + while (index < 0x100) {
4984 + if (dram_partition[cpu]&index)
4985 + break;
4986 + index = index << 1;
4987 + area_index++;
4988 + }
4989 +
4990 + return area_index;
4991 +}
4992 +
4993 +/* use this function ONLY for Lv.A/B pages */
4994 +int is_in_correct_bank(struct page* page, int cpu)
4995 +{
4996 + int bank;
4997 + unsigned int page_bank_bit;
4998 +
4999 + bank = page_bank(page);
5000 + page_bank_bit = 1 << bank;
5001 +
5002 + if (cpu == -1 || cpu == NR_CPUS)
5003 + return (page_bank_bit & dram_partition[NR_CPUS]);
5004 + else
5005 + return (page_bank_bit & dram_partition[cpu]);
5006 +}
5007 +
5008 +int is_in_llc_partition(struct page* page, int cpu)
5009 +{
5010 + int color;
5011 + unsigned int page_color_bit;
5012 +
5013 + color = page_color(page);
5014 + page_color_bit = 1 << color;
5015 +
5016 + if (cpu == -1 || cpu == NR_CPUS)
5017 + return (page_color_bit & llc_partition[8]);
5018 + else
5019 + return (page_color_bit & (llc_partition[cpu*2] | llc_partition[cpu*2+1]));
5020 +}
5021 +
5022 +/* Bounds for values */
5023 +unsigned int llc_partition_max = 0x0000ffff;
5024 +unsigned int llc_partition_min = 0;
5025 +unsigned int dram_partition_max = 0x000000ff;
5026 +unsigned int dram_partition_min = 0;
5027 +
5028 +/* slabtest module */
5029 +int buf_size = 0;
5030 +int buf_num = 1;
5031 +
5032 +int slabtest_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
5033 +{
5034 + int ret = 0, i;
5035 + int** testbuffer;
5036 + mutex_lock(&dev_mutex);
5037 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
5038 +
5039 + if (ret)
5040 + goto out;
5041 +
5042 + if (write) {
5043 + int idx;
5044 + int n_data = buf_size/sizeof(int);
5045 +
5046 + printk(KERN_INFO "-------SLABTEST on CPU%d with %d buffer size\n", raw_smp_processor_id(), buf_size);
5047 +
5048 + testbuffer = kmalloc(sizeof(int*)*buf_num, GFP_KERNEL|GFP_COLOR);
5049 +
5050 + for (idx=0; idx<buf_num; idx++)
5051 + {
5052 + printk(KERN_INFO "kmalloc size %d, n_data %d\n", buf_size, n_data);
5053 + testbuffer[idx] = kmalloc(buf_size, GFP_KERNEL|GFP_COLOR);
5054 +
5055 + if (!testbuffer[idx]) {
5056 + printk(KERN_ERR "kmalloc failed size = %d\n", buf_size);
5057 + goto out;
5058 + }
5059 + }
5060 +
5061 +
5062 + /* do test */
5063 + for (idx=0; idx<buf_num; idx++)
5064 + {
5065 + int t = 0;
5066 + printk(KERN_INFO "kmalloc size = %d n_data = %d\n", buf_size, n_data);
5067 + printk(KERN_INFO "write data to buffer\n");
5068 + for (i = 0; i < n_data; i++) {
5069 + testbuffer[idx][i] = i%27;
5070 + }
5071 + printk(KERN_INFO "read data from buffer\n");
5072 + for (i = 0; i < n_data; i++) {
5073 + t += testbuffer[idx][i];
5074 + //printk(KERN_INFO "[%d] = %d\n", i, testbuffer[idx][i]);
5075 + }
5076 + }
5077 +
5078 + for (idx=0; idx<buf_num; idx++)
5079 + kfree(testbuffer[idx]);
5080 +
5081 + kfree(testbuffer);
5082 + printk(KERN_INFO "-------SLABTEST FINISHED on CPU%d\n", raw_smp_processor_id());
5083 + }
5084 +out:
5085 + mutex_unlock(&dev_mutex);
5086 + return ret;
5087 +}
5088 +
5089 +int num_buffer_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
5090 +{
5091 + int ret = 0;
5092 + mutex_lock(&dev_mutex);
5093 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
5094 +
5095 + if (ret)
5096 + goto out;
5097 +
5098 + if (write) {
5099 + printk(KERN_INFO "buf_num = %d\n", buf_num);
5100 + }
5101 +out:
5102 + mutex_unlock(&dev_mutex);
5103 + return ret;
5104 +}
5105 +
5106 +static struct ctl_table partition_table[] =
5107 +{
5108 +
5109 + {
5110 + .procname = "C0_LA_color",
5111 + .mode = 0666,
5112 + .proc_handler = llc_partition_handler,
5113 + .data = &llc_partition[0],
5114 + .maxlen = sizeof(llc_partition[0]),
5115 + .extra1 = &llc_partition_min,
5116 + .extra2 = &llc_partition_max,
5117 + },
5118 + {
5119 + .procname = "C0_LB_color",
5120 + .mode = 0666,
5121 + .proc_handler = llc_partition_handler,
5122 + .data = &llc_partition[1],
5123 + .maxlen = sizeof(llc_partition[1]),
5124 + .extra1 = &llc_partition_min,
5125 + .extra2 = &llc_partition_max,
5126 + },
5127 + {
5128 + .procname = "C1_LA_color",
5129 + .mode = 0666,
5130 + .proc_handler = llc_partition_handler,
5131 + .data = &llc_partition[2],
5132 + .maxlen = sizeof(llc_partition[2]),
5133 + .extra1 = &llc_partition_min,
5134 + .extra2 = &llc_partition_max,
5135 + },
5136 + {
5137 + .procname = "C1_LB_color",
5138 + .mode = 0666,
5139 + .proc_handler = llc_partition_handler,
5140 + .data = &llc_partition[3],
5141 + .maxlen = sizeof(llc_partition[3]),
5142 + .extra1 = &llc_partition_min,
5143 + .extra2 = &llc_partition_max,
5144 + },
5145 + {
5146 + .procname = "C2_LA_color",
5147 + .mode = 0666,
5148 + .proc_handler = llc_partition_handler,
5149 + .data = &llc_partition[4],
5150 + .maxlen = sizeof(llc_partition[4]),
5151 + .extra1 = &llc_partition_min,
5152 + .extra2 = &llc_partition_max,
5153 + },
5154 + {
5155 + .procname = "C2_LB_color",
5156 + .mode = 0666,
5157 + .proc_handler = llc_partition_handler,
5158 + .data = &llc_partition[5],
5159 + .maxlen = sizeof(llc_partition[5]),
5160 + .extra1 = &llc_partition_min,
5161 + .extra2 = &llc_partition_max,
5162 + },
5163 + {
5164 + .procname = "C3_LA_color",
5165 + .mode = 0666,
5166 + .proc_handler = llc_partition_handler,
5167 + .data = &llc_partition[6],
5168 + .maxlen = sizeof(llc_partition[6]),
5169 + .extra1 = &llc_partition_min,
5170 + .extra2 = &llc_partition_max,
5171 + },
5172 + {
5173 + .procname = "C3_LB_color",
5174 + .mode = 0666,
5175 + .proc_handler = llc_partition_handler,
5176 + .data = &llc_partition[7],
5177 + .maxlen = sizeof(llc_partition[7]),
5178 + .extra1 = &llc_partition_min,
5179 + .extra2 = &llc_partition_max,
5180 + },
5181 + {
5182 + .procname = "Call_LC_color",
5183 + .mode = 0666,
5184 + .proc_handler = llc_partition_handler,
5185 + .data = &llc_partition[8],
5186 + .maxlen = sizeof(llc_partition[8]),
5187 + .extra1 = &llc_partition_min,
5188 + .extra2 = &llc_partition_max,
5189 + },
5190 + {
5191 + .procname = "C0_dram",
5192 + .mode = 0666,
5193 + .proc_handler = dram_partition_handler,
5194 + .data = &dram_partition[0],
5195 + .maxlen = sizeof(llc_partition[0]),
5196 + .extra1 = &dram_partition_min,
5197 + .extra2 = &dram_partition_max,
5198 + },
5199 + {
5200 + .procname = "C1_dram",
5201 + .mode = 0666,
5202 + .proc_handler = dram_partition_handler,
5203 + .data = &dram_partition[1],
5204 + .maxlen = sizeof(llc_partition[1]),
5205 + .extra1 = &dram_partition_min,
5206 + .extra2 = &dram_partition_max,
5207 + },
5208 + {
5209 + .procname = "C2_dram",
5210 + .mode = 0666,
5211 + .proc_handler = dram_partition_handler,
5212 + .data = &dram_partition[2],
5213 + .maxlen = sizeof(llc_partition[2]),
5214 + .extra1 = &dram_partition_min,
5215 + .extra2 = &dram_partition_max,
5216 + },
5217 + {
5218 + .procname = "C3_dram",
5219 + .mode = 0666,
5220 + .proc_handler = dram_partition_handler,
5221 + .data = &dram_partition[3],
5222 + .maxlen = sizeof(llc_partition[3]),
5223 + .extra1 = &dram_partition_min,
5224 + .extra2 = &dram_partition_max,
5225 + },
5226 + {
5227 + .procname = "CS_dram",
5228 + .mode = 0666,
5229 + .proc_handler = dram_partition_handler,
5230 + .data = &dram_partition[4],
5231 + .maxlen = sizeof(llc_partition[4]),
5232 + .extra1 = &dram_partition_min,
5233 + .extra2 = &dram_partition_max,
5234 + },
5235 + {
5236 + .procname = "slabtest",
5237 + .mode = 0666,
5238 + .proc_handler = slabtest_handler,
5239 + .data = &buf_size,
5240 + .maxlen = sizeof(buf_size),
5241 + },
5242 + {
5243 + .procname = "num_buffer",
5244 + .mode = 0666,
5245 + .proc_handler = num_buffer_handler,
5246 + .data = &buf_num,
5247 + .maxlen = sizeof(buf_num),
5248 + },
5249 + { }
5250 +};
5251 +
5252 +static struct ctl_table litmus_dir_table[] = {
5253 + {
5254 + .procname = "litmus",
5255 + .mode = 0555,
5256 + .child = partition_table,
5257 + },
5258 + { }
5259 +};
5260 +
5261 +static struct ctl_table_header *litmus_sysctls;
5262 +
5263 +int llc_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
5264 +{
5265 + int ret = 0, i;
5266 + mutex_lock(&dev_mutex);
5267 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
5268 + if (ret)
5269 + goto out;
5270 + if (write) {
5271 + printk("New LLC Partition : \n");
5272 + for(i = 0; i < NR_LLC_PARTITIONS; i++) {
5273 + printk("llc_partition[%d] = 0x%04x\n", i, llc_partition[i]);
5274 + }
5275 + }
5276 +out:
5277 + mutex_unlock(&dev_mutex);
5278 + return ret;
5279 +}
5280 +
5281 +int dram_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos)
5282 +{
5283 + int ret = 0, i;
5284 + mutex_lock(&dev_mutex);
5285 + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
5286 + if (ret)
5287 + goto out;
5288 + if (write) {
5289 + for(i = 0; i < NR_DRAM_PARTITIONS; i++) {
5290 + printk("dram_partition[%d] = 0x%04x\n", i, dram_partition[i]);
5291 + }
5292 + }
5293 +out:
5294 + mutex_unlock(&dev_mutex);
5295 + return ret;
5296 +}
5297 +
5298 +/*
5299 + * Initialize this page_dev proc.
5300 + */
5301 +static int __init init_litmus_page_dev(void)
5302 +{
5303 + int err = 0;
5304 +
5305 + printk("Initialize page_dev.c\n");
5306 +
5307 + mutex_init(&dev_mutex);
5308 +
5309 + litmus_sysctls = register_sysctl_table(litmus_dir_table);
5310 + if (!litmus_sysctls) {
5311 + printk(KERN_WARNING "Could not register LITMUS^RT page_dev sysctl.\n");
5312 + err = -EFAULT;
5313 + goto out;
5314 + }
5315 +
5316 + printk(KERN_INFO "Registering LITMUS^RT page_dev proc.\n");
5317 +out:
5318 + return err;
5319 +}
5320 +
5321 +static void __exit exit_litmus_page_dev(void)
5322 +{
5323 + mutex_destroy(&dev_mutex);
5324 +}
5325 +
5326 +module_init(init_litmus_page_dev);
5327 +module_exit(exit_litmus_page_dev);
5328 \ No newline at end of file
5329 diff --git litmus/polling_reservations.c litmus/polling_reservations.c
5330 new file mode 100644
5331 index 0000000..d44a403
5332 --- /dev/null
5333 +++ litmus/polling_reservations.c
5334 @@ -0,0 +1,563 @@
5335 +#include <linux/sched.h>
5336 +
5337 +#include <litmus/litmus.h>
5338 +#include <litmus/reservation.h>
5339 +#include <litmus/polling_reservations.h>
5340 +
5341 +static void periodic_polling_client_arrives(
5342 + struct reservation* res,
5343 + struct reservation_client *client
5344 +)
5345 +{
5346 + struct polling_reservation *pres =
5347 + container_of(res, struct polling_reservation, res);
5348 + lt_t instances, tmp;
5349 +
5350 + list_add_tail(&client->list, &res->clients);
5351 +
5352 + switch (res->state) {
5353 + case RESERVATION_INACTIVE:
5354 + /* Figure out next replenishment time. */
5355 + if (res->env->time_zero == 0) {
5356 + tmp = res->env->current_time - res->env->time_zero;
5357 + instances = div64_u64(tmp, pres->period);
5358 + res->next_replenishment =
5359 + (instances + 1) * pres->period + pres->offset;
5360 + }
5361 + else {
5362 + tmp = res->env->current_time - res->env->time_zero;
5363 + instances = div64_u64(tmp, pres->period);
5364 + res->next_replenishment = res->env->time_zero + instances * pres->period;
5365 + }
5366 +
5367 + TRACE("ENV_TIME_ZERO %llu\n", res->env->time_zero);
5368 + TRACE("pol-res: R%d activate tmp=%llu instances=%llu period=%llu nextrp=%llu cur=%llu\n",
5369 + res->id, tmp, instances, pres->period, res->next_replenishment,
5370 + res->env->current_time);
5371 +
5372 + res->env->change_state(res->env, res,
5373 + RESERVATION_DEPLETED);
5374 + break;
5375 +
5376 + case RESERVATION_ACTIVE:
5377 + case RESERVATION_DEPLETED:
5378 + /* do nothing */
5379 + break;
5380 +
5381 + case RESERVATION_ACTIVE_IDLE:
5382 + res->blocked_by_ghost = 0;
5383 + res->env->change_state(res->env, res,
5384 + RESERVATION_ACTIVE);
5385 + break;
5386 + }
5387 +}
5388 +
5389 +
5390 +static void periodic_polling_client_departs(
5391 + struct reservation *res,
5392 + struct reservation_client *client,
5393 + int did_signal_job_completion
5394 +)
5395 +{
5396 + list_del(&client->list);
5397 +
5398 + switch (res->state) {
5399 + case RESERVATION_INACTIVE:
5400 + case RESERVATION_ACTIVE_IDLE:
5401 + BUG(); /* INACTIVE or IDLE <=> no client */
5402 + break;
5403 +
5404 + case RESERVATION_ACTIVE:
5405 + if (list_empty(&res->clients)) {
5406 + res->env->change_state(res->env, res,
5407 +// RESERVATION_ACTIVE_IDLE);
5408 + res->cur_budget ?
5409 + RESERVATION_ACTIVE_IDLE :
5410 + RESERVATION_DEPLETED);
5411 +// did_signal_job_completion ?
5412 +// RESERVATION_DEPLETED :
5413 +// RESERVATION_ACTIVE_IDLE);
5414 + } /* else: nothing to do, more clients ready */
5415 + break;
5416 +
5417 + case RESERVATION_DEPLETED:
5418 + /* do nothing */
5419 + break;
5420 + }
5421 +}
5422 +
5423 +static void periodic_polling_on_replenishment(
5424 + struct reservation *res
5425 +)
5426 +{
5427 + struct polling_reservation *pres =
5428 + container_of(res, struct polling_reservation, res);
5429 +
5430 + /* replenish budget */
5431 + res->cur_budget = pres->max_budget;
5432 + res->next_replenishment += pres->period;
5433 + res->budget_consumed = 0;
5434 +
5435 + TRACE("polling_replenish(%u): next_replenishment=%llu\n", res->id, res->next_replenishment);
5436 + switch (res->state) {
5437 + case RESERVATION_DEPLETED:
5438 + case RESERVATION_INACTIVE:
5439 + case RESERVATION_ACTIVE_IDLE:
5440 + if (list_empty(&res->clients))
5441 + /* no clients => poll again later */
5442 + res->env->change_state(res->env, res,
5443 + RESERVATION_INACTIVE);
5444 + else
5445 + /* we have clients & budget => ACTIVE */
5446 + res->env->change_state(res->env, res,
5447 + RESERVATION_ACTIVE);
5448 + break;
5449 +
5450 + case RESERVATION_ACTIVE:
5451 + /* Replenished while active => tardy? In any case,
5452 + * go ahead and stay active. */
5453 + break;
5454 + }
5455 +}
5456 +
5457 +static void periodic_polling_on_replenishment_edf(
5458 + struct reservation *res
5459 +)
5460 +{
5461 + struct polling_reservation *pres =
5462 + container_of(res, struct polling_reservation, res);
5463 +
5464 + /* update current priority */
5465 + res->priority = res->next_replenishment + pres->deadline;
5466 +
5467 + /* do common updates */
5468 + periodic_polling_on_replenishment(res);
5469 +}
5470 +
5471 +static void common_drain_budget(
5472 + struct reservation *res,
5473 + lt_t how_much)
5474 +{
5475 + if (how_much >= res->cur_budget)
5476 + res->cur_budget = 0;
5477 + else
5478 + res->cur_budget -= how_much;
5479 +
5480 + res->budget_consumed += how_much;
5481 + res->budget_consumed_total += how_much;
5482 +
5483 + switch (res->state) {
5484 + case RESERVATION_DEPLETED:
5485 + case RESERVATION_INACTIVE:
5486 + //BUG();
5487 + TRACE("!!!!!!!!!!!!!!!STATE ERROR R%d STATE(%d)\n", res->id, res->state);
5488 + break;
5489 +
5490 + case RESERVATION_ACTIVE_IDLE:
5491 + case RESERVATION_ACTIVE:
5492 + if (!res->cur_budget) {
5493 + res->env->change_state(res->env, res,
5494 + RESERVATION_DEPLETED);
5495 + } /* else: stay in current state */
5496 + break;
5497 + }
5498 +}
5499 +
5500 +static struct reservation_ops periodic_polling_ops_fp = {
5501 + .dispatch_client = default_dispatch_client,
5502 + .client_arrives = periodic_polling_client_arrives,
5503 + .client_departs = periodic_polling_client_departs,
5504 + .replenish = periodic_polling_on_replenishment,
5505 + .drain_budget = common_drain_budget,
5506 +};
5507 +
5508 +static struct reservation_ops periodic_polling_ops_edf = {
5509 + .dispatch_client = default_dispatch_client,
5510 + .client_arrives = periodic_polling_client_arrives,
5511 + .client_departs = periodic_polling_client_departs,
5512 + .replenish = periodic_polling_on_replenishment_edf,
5513 + .drain_budget = common_drain_budget,
5514 +};
5515 +
5516 +
5517 +
5518 +
5519 +static void sporadic_polling_client_arrives_fp(
5520 + struct reservation* res,
5521 + struct reservation_client *client
5522 +)
5523 +{
5524 + struct polling_reservation *pres =
5525 + container_of(res, struct polling_reservation, res);
5526 +
5527 + list_add_tail(&client->list, &res->clients);
5528 +
5529 + switch (res->state) {
5530 + case RESERVATION_INACTIVE:
5531 + /* Replenish now. */
5532 + res->cur_budget = pres->max_budget;
5533 + res->next_replenishment =
5534 + res->env->current_time + pres->period;
5535 +
5536 + res->env->change_state(res->env, res,
5537 + RESERVATION_ACTIVE);
5538 + break;
5539 +
5540 + case RESERVATION_ACTIVE:
5541 + case RESERVATION_DEPLETED:
5542 + /* do nothing */
5543 + break;
5544 +
5545 + case RESERVATION_ACTIVE_IDLE:
5546 + res->env->change_state(res->env, res,
5547 + RESERVATION_ACTIVE);
5548 + break;
5549 + }
5550 +}
5551 +
5552 +static void sporadic_polling_client_arrives_edf(
5553 + struct reservation* res,
5554 + struct reservation_client *client
5555 +)
5556 +{
5557 + struct polling_reservation *pres =
5558 + container_of(res, struct polling_reservation, res);
5559 +
5560 + list_add_tail(&client->list, &res->clients);
5561 +
5562 + switch (res->state) {
5563 + case RESERVATION_INACTIVE:
5564 + /* Replenish now. */
5565 + res->cur_budget = pres->max_budget;
5566 + res->next_replenishment =
5567 + res->env->current_time + pres->period;
5568 + res->priority =
5569 + res->env->current_time + pres->deadline;
5570 +
5571 + res->env->change_state(res->env, res,
5572 + RESERVATION_ACTIVE);
5573 + break;
5574 +
5575 + case RESERVATION_ACTIVE:
5576 + case RESERVATION_DEPLETED:
5577 + /* do nothing */
5578 + break;
5579 +
5580 + case RESERVATION_ACTIVE_IDLE:
5581 + res->env->change_state(res->env, res,
5582 + RESERVATION_ACTIVE);
5583 + break;
5584 + }
5585 +}
5586 +
5587 +static struct reservation_ops sporadic_polling_ops_fp = {
5588 + .dispatch_client = default_dispatch_client,
5589 + .client_arrives = sporadic_polling_client_arrives_fp,
5590 + .client_departs = periodic_polling_client_departs,
5591 + .replenish = periodic_polling_on_replenishment,
5592 + .drain_budget = common_drain_budget,
5593 +};
5594 +
5595 +static struct reservation_ops sporadic_polling_ops_edf = {
5596 + .dispatch_client = default_dispatch_client,
5597 + .client_arrives = sporadic_polling_client_arrives_edf,
5598 + .client_departs = periodic_polling_client_departs,
5599 + .replenish = periodic_polling_on_replenishment_edf,
5600 + .drain_budget = common_drain_budget,
5601 +};
5602 +
5603 +void polling_reservation_init(
5604 + struct polling_reservation *pres,
5605 + int use_edf_prio,
5606 + int use_periodic_polling,
5607 + lt_t budget, lt_t period, lt_t deadline, lt_t offset
5608 +)
5609 +{
5610 + if (!deadline)
5611 + deadline = period;
5612 + BUG_ON(budget > period);
5613 + BUG_ON(budget > deadline);
5614 + BUG_ON(offset >= period);
5615 +
5616 + reservation_init(&pres->res);
5617 + pres->max_budget = budget;
5618 + pres->period = period;
5619 + pres->deadline = deadline;
5620 + pres->offset = offset;
5621 + TRACE_TASK(current, "polling_reservation_init: periodic %d, use_edf %d\n", use_periodic_polling, use_edf_prio);
5622 + if (use_periodic_polling) {
5623 + if (use_edf_prio)
5624 + pres->res.ops = &periodic_polling_ops_edf;
5625 + else
5626 + pres->res.ops = &periodic_polling_ops_fp;
5627 + } else {
5628 + if (use_edf_prio)
5629 + pres->res.ops = &sporadic_polling_ops_edf;
5630 + else
5631 + pres->res.ops = &sporadic_polling_ops_fp;
5632 + }
5633 +}
5634 +
5635 +
5636 +static lt_t td_cur_major_cycle_start(struct table_driven_reservation *tdres)
5637 +{
5638 + lt_t x, tmp;
5639 +
5640 + tmp = tdres->res.env->current_time - tdres->res.env->time_zero;
5641 + x = div64_u64(tmp, tdres->major_cycle);
5642 + x *= tdres->major_cycle;
5643 + return x;
5644 +}
5645 +
5646 +
5647 +static lt_t td_next_major_cycle_start(struct table_driven_reservation *tdres)
5648 +{
5649 + lt_t x, tmp;
5650 +
5651 + tmp = tdres->res.env->current_time - tdres->res.env->time_zero;
5652 + x = div64_u64(tmp, tdres->major_cycle) + 1;
5653 + x *= tdres->major_cycle;
5654 + return x;
5655 +}
5656 +
5657 +static void td_client_arrives(
5658 + struct reservation* res,
5659 + struct reservation_client *client
5660 +)
5661 +{
5662 + struct table_driven_reservation *tdres =
5663 + container_of(res, struct table_driven_reservation, res);
5664 +
5665 + list_add_tail(&client->list, &res->clients);
5666 +
5667 + switch (res->state) {
5668 + case RESERVATION_INACTIVE:
5669 + /* Figure out first replenishment time. */
5670 + tdres->major_cycle_start = td_next_major_cycle_start(tdres);
5671 + res->next_replenishment = tdres->major_cycle_start;
5672 + res->next_replenishment += tdres->intervals[0].start;
5673 + tdres->next_interval = 0;
5674 +
5675 + res->env->change_state(res->env, res,
5676 + RESERVATION_DEPLETED);
5677 + break;
5678 +
5679 + case RESERVATION_ACTIVE:
5680 + case RESERVATION_DEPLETED:
5681 + /* do nothing */
5682 + break;
5683 +
5684 + case RESERVATION_ACTIVE_IDLE:
5685 + res->env->change_state(res->env, res,
5686 + RESERVATION_ACTIVE);
5687 + break;
5688 + }
5689 +}
5690 +
5691 +static void td_client_departs(
5692 + struct reservation *res,
5693 + struct reservation_client *client,
5694 + int did_signal_job_completion
5695 +)
5696 +{
5697 + list_del(&client->list);
5698 +
5699 + switch (res->state) {
5700 + case RESERVATION_INACTIVE:
5701 + case RESERVATION_ACTIVE_IDLE:
5702 + //BUG(); /* INACTIVE or IDLE <=> no client */
5703 + break;
5704 +
5705 + case RESERVATION_ACTIVE:
5706 + if (list_empty(&res->clients)) {
5707 + res->env->change_state(res->env, res,
5708 + RESERVATION_ACTIVE_IDLE);
5709 + } /* else: nothing to do, more clients ready */
5710 + break;
5711 +
5712 + case RESERVATION_DEPLETED:
5713 + /* do nothing */
5714 + break;
5715 + }
5716 +}
5717 +
5718 +static lt_t td_time_remaining_until_end(struct table_driven_reservation *tdres)
5719 +{
5720 + lt_t now = tdres->res.env->current_time;
5721 + lt_t end = tdres->cur_interval.end;
5722 + //TRACE("td_remaining(%u): start=%llu now=%llu end=%llu state=%d\n", tdres->res.id, tdres->cur_interval.start, now, end, tdres->res.state);
5723 + if (now >= end)
5724 + return 0;
5725 + else
5726 + return end - now;
5727 +}
5728 +
5729 +static void td_replenish(
5730 + struct reservation *res)
5731 +{
5732 + struct table_driven_reservation *tdres =
5733 + container_of(res, struct table_driven_reservation, res);
5734 +
5735 + //TRACE("td_replenish(%u): expected_replenishment=%llu\n", res->id, res->next_replenishment);
5736 +
5737 + /* figure out current interval */
5738 + tdres->cur_interval.start = tdres->major_cycle_start +
5739 + tdres->intervals[tdres->next_interval].start;
5740 + tdres->cur_interval.end = tdres->major_cycle_start +
5741 + tdres->intervals[tdres->next_interval].end;
5742 +/* TRACE("major_cycle_start=%llu => [%llu, %llu]\n",
5743 + tdres->major_cycle_start,
5744 + tdres->cur_interval.start,
5745 + tdres->cur_interval.end);
5746 +*/
5747 + /* reset budget */
5748 + res->cur_budget = td_time_remaining_until_end(tdres);
5749 + res->budget_consumed = 0;
5750 + //TRACE("td_replenish(%u): %s budget=%llu\n", res->id, res->cur_budget ? "" : "WARNING", res->cur_budget);
5751 +
5752 + /* prepare next slot */
5753 + tdres->next_interval = (tdres->next_interval + 1) % tdres->num_intervals;
5754 + if (!tdres->next_interval)
5755 + /* wrap to next major cycle */
5756 + tdres->major_cycle_start += tdres->major_cycle;
5757 +
5758 + /* determine next time this reservation becomes eligible to execute */
5759 + res->next_replenishment = tdres->major_cycle_start;
5760 + res->next_replenishment += tdres->intervals[tdres->next_interval].start;
5761 + //TRACE("td_replenish(%u): next_replenishment=%llu\n", res->id, res->next_replenishment);
5762 +
5763 +
5764 + switch (res->state) {
5765 + case RESERVATION_DEPLETED:
5766 + case RESERVATION_ACTIVE:
5767 + case RESERVATION_ACTIVE_IDLE:
5768 + if (list_empty(&res->clients))
5769 + res->env->change_state(res->env, res,
5770 + RESERVATION_ACTIVE_IDLE);
5771 + else
5772 + /* we have clients & budget => ACTIVE */
5773 + res->env->change_state(res->env, res,
5774 + RESERVATION_ACTIVE);
5775 + break;
5776 +
5777 + case RESERVATION_INACTIVE:
5778 + BUG();
5779 + break;
5780 + }
5781 +}
5782 +
5783 +static void td_drain_budget(
5784 + struct reservation *res,
5785 + lt_t how_much)
5786 +{
5787 + struct table_driven_reservation *tdres =
5788 + container_of(res, struct table_driven_reservation, res);
5789 +
5790 + res->budget_consumed += how_much;
5791 + res->budget_consumed_total += how_much;
5792 +
5793 + /* Table-driven scheduling: instead of tracking the budget, we compute
5794 + * how much time is left in this allocation interval. */
5795 +
5796 + /* sanity check: we should never try to drain from future slots */
5797 + //TRACE("TD_DRAIN STATE(%d) [%llu,%llu] %llu ?\n", res->state, tdres->cur_interval.start, tdres->cur_interval.end, res->env->current_time);
5798 + //BUG_ON(tdres->cur_interval.start > res->env->current_time);
5799 + if (tdres->cur_interval.start > res->env->current_time)
5800 + TRACE("TD_DRAIN BUG!!!!!!!!!!\n");
5801 +
5802 + switch (res->state) {
5803 + case RESERVATION_DEPLETED:
5804 + case RESERVATION_INACTIVE:
5805 + //BUG();
5806 + TRACE("TD_DRAIN!!!!!!!!! RES_STATE = %d\n", res->state);
5807 + break;
5808 +
5809 + case RESERVATION_ACTIVE_IDLE:
5810 + case RESERVATION_ACTIVE:
5811 + res->cur_budget = td_time_remaining_until_end(tdres);
5812 + //TRACE("td_drain_budget(%u): drained to budget=%llu\n", res->id, res->cur_budget);
5813 + if (!res->cur_budget) {
5814 + res->env->change_state(res->env, res,
5815 + RESERVATION_DEPLETED);
5816 + } else {
5817 + /* sanity check budget calculation */
5818 + //BUG_ON(res->env->current_time >= tdres->cur_interval.end);
5819 + //BUG_ON(res->env->current_time < tdres->cur_interval.start);
5820 + if (res->env->current_time >= tdres->cur_interval.end)
5821 + printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING1\n");
5822 + if (res->env->current_time < tdres->cur_interval.start)
5823 + printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING2\n");
5824 + }
5825 +
5826 + break;
5827 + }
5828 +}
5829 +
5830 +static struct task_struct* td_dispatch_client(
5831 + struct reservation *res,
5832 + lt_t *for_at_most)
5833 +{
5834 + struct task_struct *t;
5835 + struct table_driven_reservation *tdres =
5836 + container_of(res, struct table_driven_reservation, res);
5837 +
5838 + /* usual logic for selecting a client */
5839 + t = default_dispatch_client(res, for_at_most);
5840 +
5841 + TRACE_TASK(t, "td_dispatch_client(%u): selected, budget=%llu\n",
5842 + res->id, res->cur_budget);
5843 +
5844 + /* check how much budget we have left in this time slot */
5845 + res->cur_budget = td_time_remaining_until_end(tdres);
5846 +
5847 + TRACE_TASK(t, "td_dispatch_client(%u): updated to budget=%llu next=%d\n",
5848 + res->id, res->cur_budget, tdres->next_interval);
5849 +
5850 + if (unlikely(!res->cur_budget)) {
5851 + /* Unlikely case: if we ran out of budget, the user configured
5852 + * a broken scheduling table (overlapping table slots).
5853 + * Not much we can do about this, but we can't dispatch a job
5854 + * now without causing overload. So let's register this reservation
5855 + * as depleted and wait for the next allocation. */
5856 + TRACE("td_dispatch_client(%u): budget unexpectedly depleted "
5857 + "(check scheduling table for unintended overlap)\n",
5858 + res->id);
5859 + res->env->change_state(res->env, res,
5860 + RESERVATION_DEPLETED);
5861 + return NULL;
5862 + } else
5863 + return t;
5864 +}
5865 +
5866 +static struct reservation_ops td_ops = {
5867 + .dispatch_client = td_dispatch_client,
5868 + .client_arrives = td_client_arrives,
5869 + .client_departs = td_client_departs,
5870 + .replenish = td_replenish,
5871 + .drain_budget = td_drain_budget,
5872 +};
5873 +
5874 +void table_driven_reservation_init(
5875 + struct table_driven_reservation *tdres,
5876 + lt_t major_cycle,
5877 + struct lt_interval *intervals,
5878 + unsigned int num_intervals)
5879 +{
5880 + unsigned int i;
5881 +
5882 + /* sanity checking */
5883 + BUG_ON(!num_intervals);
5884 + for (i = 0; i < num_intervals; i++)
5885 + BUG_ON(intervals[i].end <= intervals[i].start);
5886 + for (i = 0; i + 1 < num_intervals; i++)
5887 + BUG_ON(intervals[i + 1].start <= intervals[i].end);
5888 + BUG_ON(intervals[num_intervals - 1].end > major_cycle);
5889 +
5890 + reservation_init(&tdres->res);
5891 + tdres->major_cycle = major_cycle;
5892 + tdres->intervals = intervals;
5893 + tdres->cur_interval.start = 0;
5894 + tdres->cur_interval.end = 0;
5895 + tdres->num_intervals = num_intervals;
5896 + tdres->res.ops = &td_ops;
5897 +}
5898 diff --git litmus/replicate_lib.c litmus/replicate_lib.c
5899 new file mode 100644
5900 index 0000000..0eac667
5901 --- /dev/null
5902 +++ litmus/replicate_lib.c
5903 @@ -0,0 +1,50 @@
5904 +#include <asm/uaccess.h>
5905 +#include <linux/uaccess.h>
5906 +#include <linux/init.h>
5907 +#include <linux/types.h>
5908 +#include <linux/kernel.h>
5909 +#include <linux/module.h>
5910 +#include <linux/sysctl.h>
5911 +#include <linux/slab.h>
5912 +#include <linux/io.h>
5913 +#include <linux/mutex.h>
5914 +#include <linux/time.h>
5915 +#include <linux/migrate.h>
5916 +#include <linux/mm.h>
5917 +#include <linux/memcontrol.h>
5918 +#include <linux/mm_inline.h>
5919 +
5920 +#include <litmus/litmus_proc.h>
5921 +#include <litmus/sched_trace.h>
5922 +#include <litmus/cache_proc.h>
5923 +#include <litmus/mc2_common.h>
5924 +#include <litmus/replicate_lib.h>
5925 +
5926 +DEFINE_PER_CPU(struct list_head, shared_lib_page_list);
5927 +
5928 +#define shared_lib_pages_for(cpu_id) (&per_cpu(shared_lib_page_list, cpu_id))
5929 +#define local_shared_lib_pages() (this_cpu_ptr(&shared_lib_page_list))
5930 +
5931 +#define INVALID_PFN (0xffffffff)
5932 +
5933 +static int __init litmus_replicate_lib_init(void)
5934 +{
5935 + int cpu, ret = 0;
5936 +
5937 + printk(KERN_INFO "Registering LITMUS^RT Per-core Shared Library module.\n");
5938 +
5939 + for_each_online_cpu(cpu) {
5940 + INIT_LIST_HEAD(shared_lib_pages_for(cpu));
5941 + printk(KERN_INFO "CPU%d PSL-list initialized.\n", cpu);
5942 + }
5943 +
5944 + return ret;
5945 +}
5946 +
5947 +static void litmus_replicate_lib_exit(void)
5948 +{
5949 + return;
5950 +}
5951 +
5952 +module_init(litmus_replicate_lib_init);
5953 +module_exit(litmus_replicate_lib_exit);
5954 \ No newline at end of file
5955 diff --git litmus/reservation.c litmus/reservation.c
5956 new file mode 100644
5957 index 0000000..5eee01a
5958 --- /dev/null
5959 +++ litmus/reservation.c
5960 @@ -0,0 +1,706 @@
5961 +#include <linux/sched.h>
5962 +#include <linux/slab.h>
5963 +
5964 +#include <litmus/litmus.h>
5965 +#include <litmus/reservation.h>
5966 +
5967 +#define BUDGET_ENFORCEMENT_AT_C 0
5968 +
5969 +void reservation_init(struct reservation *res)
5970 +{
5971 + memset(res, sizeof(*res), 0);
5972 + res->state = RESERVATION_INACTIVE;
5973 + INIT_LIST_HEAD(&res->clients);
5974 +}
5975 +
5976 +struct task_struct* default_dispatch_client(
5977 + struct reservation *res,
5978 + lt_t *for_at_most)
5979 +{
5980 + struct reservation_client *client, *next;
5981 + struct task_struct* tsk;
5982 +
5983 + BUG_ON(res->state != RESERVATION_ACTIVE);
5984 + *for_at_most = 0;
5985 +
5986 + list_for_each_entry_safe(client, next, &res->clients, list) {
5987 + tsk = client->dispatch(client);
5988 + if (likely(tsk)) {
5989 + return tsk;
5990 + }
5991 + }
5992 + return NULL;
5993 +}
5994 +
5995 +static struct task_struct * task_client_dispatch(struct reservation_client *client)
5996 +{
5997 + struct task_client *tc = container_of(client, struct task_client, client);
5998 + return tc->task;
5999 +}
6000 +
6001 +void task_client_init(struct task_client *tc, struct task_struct *tsk,
6002 + struct reservation *res)
6003 +{
6004 + memset(&tc->client, sizeof(tc->client), 0);
6005 + tc->client.dispatch = task_client_dispatch;
6006 + tc->client.reservation = res;
6007 + tc->task = tsk;
6008 +}
6009 +
6010 +static void sup_scheduler_update_at(
6011 + struct sup_reservation_environment* sup_env,
6012 + lt_t when)
6013 +{
6014 + //TRACE("SCHEDULER_UPDATE_AT update: %llu > when %llu\n", sup_env->next_scheduler_update, when);
6015 + if (sup_env->next_scheduler_update > when)
6016 + sup_env->next_scheduler_update = when;
6017 +}
6018 +
6019 +void sup_scheduler_update_after(
6020 + struct sup_reservation_environment* sup_env,
6021 + lt_t timeout)
6022 +{
6023 + sup_scheduler_update_at(sup_env, sup_env->env.current_time + timeout);
6024 +}
6025 +
6026 +static int _sup_queue_depleted(
6027 + struct sup_reservation_environment* sup_env,
6028 + struct reservation *res)
6029 +{
6030 + struct list_head *pos;
6031 + struct reservation *queued;
6032 + int passed_earlier = 0;
6033 +
6034 + list_for_each(pos, &sup_env->depleted_reservations) {
6035 + queued = list_entry(pos, struct reservation, list);
6036 + if (queued->next_replenishment > res->next_replenishment) {
6037 + list_add(&res->list, pos->prev);
6038 + return passed_earlier;
6039 + } else
6040 + passed_earlier = 1;
6041 + }
6042 +
6043 + list_add_tail(&res->list, &sup_env->depleted_reservations);
6044 +
6045 + return passed_earlier;
6046 +}
6047 +
6048 +static void sup_queue_depleted(
6049 + struct sup_reservation_environment* sup_env,
6050 + struct reservation *res)
6051 +{
6052 + int passed_earlier = _sup_queue_depleted(sup_env, res);
6053 +
6054 + /* check for updated replenishment time */
6055 + if (!passed_earlier)
6056 + sup_scheduler_update_at(sup_env, res->next_replenishment);
6057 +}
6058 +
6059 +static int _sup_queue_active(
6060 + struct sup_reservation_environment* sup_env,
6061 + struct reservation *res)
6062 +{
6063 + struct list_head *pos;
6064 + struct reservation *queued;
6065 + int passed_active = 0;
6066 +
6067 + list_for_each(pos, &sup_env->active_reservations) {
6068 + queued = list_entry(pos, struct reservation, list);
6069 + if (queued->priority > res->priority) {
6070 + list_add(&res->list, pos->prev);
6071 + return passed_active;
6072 + } else if (queued->state == RESERVATION_ACTIVE)
6073 + passed_active = 1;
6074 + }
6075 +
6076 + list_add_tail(&res->list, &sup_env->active_reservations);
6077 + return passed_active;
6078 +}
6079 +
6080 +static void sup_queue_active(
6081 + struct sup_reservation_environment* sup_env,
6082 + struct reservation *res)
6083 +{
6084 + int passed_active = _sup_queue_active(sup_env, res);
6085 +
6086 + /* check for possible preemption */
6087 + if (res->state == RESERVATION_ACTIVE && !passed_active)
6088 + sup_env->next_scheduler_update = SUP_RESCHEDULE_NOW;
6089 + else {
6090 + /* Active means this reservation is draining budget => make sure
6091 + * the scheduler is called to notice when the reservation budget has been
6092 + * drained completely. */
6093 + sup_scheduler_update_after(sup_env, res->cur_budget);
6094 + }
6095 +}
6096 +
6097 +static void sup_queue_reservation(
6098 + struct sup_reservation_environment* sup_env,
6099 + struct reservation *res)
6100 +{
6101 + switch (res->state) {
6102 + case RESERVATION_INACTIVE:
6103 + list_add(&res->list, &sup_env->inactive_reservations);
6104 + break;
6105 +
6106 + case RESERVATION_DEPLETED:
6107 + sup_queue_depleted(sup_env, res);
6108 + break;
6109 +
6110 + case RESERVATION_ACTIVE_IDLE:
6111 + case RESERVATION_ACTIVE:
6112 + sup_queue_active(sup_env, res);
6113 + break;
6114 + }
6115 +}
6116 +
6117 +void sup_add_new_reservation(
6118 + struct sup_reservation_environment* sup_env,
6119 + struct reservation* new_res)
6120 +{
6121 + new_res->env = &sup_env->env;
6122 + sup_queue_reservation(sup_env, new_res);
6123 +}
6124 +
6125 +struct reservation* sup_find_by_id(struct sup_reservation_environment* sup_env,
6126 + unsigned int id)
6127 +{
6128 + struct reservation *res;
6129 +
6130 + list_for_each_entry(res, &sup_env->active_reservations, list) {
6131 + if (res->id == id)
6132 + return res;
6133 + }
6134 + list_for_each_entry(res, &sup_env->inactive_reservations, list) {
6135 + if (res->id == id)
6136 + return res;
6137 + }
6138 + list_for_each_entry(res, &sup_env->depleted_reservations, list) {
6139 + if (res->id == id)
6140 + return res;
6141 + }
6142 +
6143 + return NULL;
6144 +}
6145 +
6146 +static void sup_charge_budget(
6147 + struct sup_reservation_environment* sup_env,
6148 + lt_t delta)
6149 +{
6150 + struct list_head *pos, *next;
6151 + struct reservation *res;
6152 +
6153 + int encountered_active = 0;
6154 +
6155 + list_for_each_safe(pos, next, &sup_env->active_reservations) {
6156 + /* charge all ACTIVE_IDLE up to the first ACTIVE reservation */
6157 + res = list_entry(pos, struct reservation, list);
6158 + if (res->state == RESERVATION_ACTIVE) {
6159 + TRACE("sup_charge_budget ACTIVE R%u drain %llu\n", res->id, delta);
6160 + if (encountered_active == 0 && res->blocked_by_ghost == 0) {
6161 + TRACE("DRAIN !!\n");
6162 + res->ops->drain_budget(res, delta);
6163 + encountered_active = 1;
6164 + }
6165 + } else {
6166 + //BUG_ON(res->state != RESERVATION_ACTIVE_IDLE);
6167 + TRACE("sup_charge_budget INACTIVE R%u drain %llu\n", res->id, delta);
6168 + res->ops->drain_budget(res, delta);
6169 + }
6170 + if (res->state == RESERVATION_ACTIVE ||
6171 + res->state == RESERVATION_ACTIVE_IDLE)
6172 + {
6173 + /* make sure scheduler is invoked when this reservation expires
6174 + * its remaining budget */
6175 + TRACE("requesting scheduler update for reservation %u in %llu nanoseconds\n",
6176 + res->id, res->cur_budget);
6177 + sup_scheduler_update_after(sup_env, res->cur_budget);
6178 + }
6179 + //if (encountered_active == 2)
6180 + /* stop at the first ACTIVE reservation */
6181 + // break;
6182 + }
6183 + //TRACE("finished charging budgets\n");
6184 +}
6185 +
6186 +static void sup_replenish_budgets(struct sup_reservation_environment* sup_env)
6187 +{
6188 + struct list_head *pos, *next;
6189 + struct reservation *res;
6190 +
6191 + list_for_each_safe(pos, next, &sup_env->depleted_reservations) {
6192 + res = list_entry(pos, struct reservation, list);
6193 + if (res->next_replenishment <= sup_env->env.current_time) {
6194 + res->ops->replenish(res);
6195 + } else {
6196 + /* list is ordered by increasing depletion times */
6197 + break;
6198 + }
6199 + }
6200 + //TRACE("finished replenishing budgets\n");
6201 +
6202 + /* request a scheduler update at the next replenishment instant */
6203 + res = list_first_entry_or_null(&sup_env->depleted_reservations,
6204 + struct reservation, list);
6205 + if (res)
6206 + sup_scheduler_update_at(sup_env, res->next_replenishment);
6207 +}
6208 +
6209 +void sup_update_time(
6210 + struct sup_reservation_environment* sup_env,
6211 + lt_t now)
6212 +{
6213 + lt_t delta;
6214 +
6215 + /* If the time didn't advance, there is nothing to do.
6216 + * This check makes it safe to call sup_advance_time() potentially
6217 + * multiple times (e.g., via different code paths. */
6218 + //TRACE("(sup_update_time) now: %llu, current_time: %llu\n", now, sup_env->env.current_time);
6219 + if (unlikely(now <= sup_env->env.current_time))
6220 + return;
6221 +
6222 + delta = now - sup_env->env.current_time;
6223 + sup_env->env.current_time = now;
6224 +
6225 + /* check if future updates are required */
6226 + if (sup_env->next_scheduler_update <= sup_env->env.current_time)
6227 + sup_env->next_scheduler_update = SUP_NO_SCHEDULER_UPDATE;
6228 +
6229 + /* deplete budgets by passage of time */
6230 + //TRACE("CHARGE###\n");
6231 + sup_charge_budget(sup_env, delta);
6232 +
6233 + /* check if any budgets where replenished */
6234 + //TRACE("REPLENISH###\n");
6235 + sup_replenish_budgets(sup_env);
6236 +}
6237 +
6238 +struct task_struct* sup_dispatch(struct sup_reservation_environment* sup_env)
6239 +{
6240 + struct reservation *res, *next;
6241 + struct task_struct *tsk = NULL;
6242 + lt_t time_slice;
6243 +
6244 + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) {
6245 + if (res->state == RESERVATION_ACTIVE) {
6246 + tsk = res->ops->dispatch_client(res, &time_slice);
6247 + if (likely(tsk)) {
6248 + if (time_slice)
6249 + sup_scheduler_update_after(sup_env, time_slice);
6250 + sup_scheduler_update_after(sup_env, res->cur_budget);
6251 + return tsk;
6252 + }
6253 + }
6254 + }
6255 +
6256 + return NULL;
6257 +}
6258 +
6259 +static void sup_res_change_state(
6260 + struct reservation_environment* env,
6261 + struct reservation *res,
6262 + reservation_state_t new_state)
6263 +{
6264 + struct sup_reservation_environment* sup_env;
6265 +
6266 + sup_env = container_of(env, struct sup_reservation_environment, env);
6267 +
6268 + TRACE("reservation R%d state %d->%d at %llu\n",
6269 + res->id, res->state, new_state, env->current_time);
6270 +
6271 + list_del(&res->list);
6272 + /* check if we need to reschedule because we lost an active reservation */
6273 + if (res->state == RESERVATION_ACTIVE && !sup_env->will_schedule)
6274 + sup_env->next_scheduler_update = SUP_RESCHEDULE_NOW;
6275 + res->state = new_state;
6276 + sup_queue_reservation(sup_env, res);
6277 +}
6278 +
6279 +void sup_init(struct sup_reservation_environment* sup_env)
6280 +{
6281 + memset(sup_env, sizeof(*sup_env), 0);
6282 +
6283 + INIT_LIST_HEAD(&sup_env->active_reservations);
6284 + INIT_LIST_HEAD(&sup_env->depleted_reservations);
6285 + INIT_LIST_HEAD(&sup_env->inactive_reservations);
6286 +
6287 + sup_env->env.change_state = sup_res_change_state;
6288 +
6289 + sup_env->next_scheduler_update = SUP_NO_SCHEDULER_UPDATE;
6290 +}
6291 +
6292 +struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env,
6293 + unsigned int id)
6294 +{
6295 + struct reservation *res;
6296 +
6297 + list_for_each_entry(res, &gmp_env->active_reservations, list) {
6298 + if (res->id == id)
6299 + return res;
6300 + }
6301 + list_for_each_entry(res, &gmp_env->inactive_reservations, list) {
6302 + if (res->id == id)
6303 + return res;
6304 + }
6305 + list_for_each_entry(res, &gmp_env->depleted_reservations, list) {
6306 + if (res->id == id)
6307 + return res;
6308 + }
6309 +
6310 + return NULL;
6311 +}
6312 +
6313 +
6314 +struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env,
6315 + unsigned int id)
6316 +{
6317 + struct next_timer_event *event;
6318 +
6319 + list_for_each_entry(event, &gmp_env->next_events, list) {
6320 + if (event->id == id)
6321 + return event;
6322 + }
6323 +
6324 + return NULL;
6325 +}
6326 +
6327 +
6328 +struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env,
6329 + lt_t when)
6330 +{
6331 + struct next_timer_event *event;
6332 +
6333 + list_for_each_entry(event, &gmp_env->next_events, list) {
6334 + if (event->next_update == when)
6335 + return event;
6336 + }
6337 +
6338 + return NULL;
6339 +}
6340 +
6341 +#define TIMER_RESOLUTION 100000L
6342 +
6343 +static void gmp_add_event(
6344 + struct gmp_reservation_environment* gmp_env,
6345 + lt_t when, unsigned int id, event_type_t type)
6346 +{
6347 + struct next_timer_event *nevent, *queued;
6348 + struct list_head *pos;
6349 + int found = 0, update = 0;
6350 +
6351 + //when = div64_u64(when, TIMER_RESOLUTION);
6352 + //when *= TIMER_RESOLUTION;
6353 +//printk(KERN_ALERT "GMP_ADD id=%d type=%d when=%llu\n", id, type, when);
6354 + nevent = gmp_find_event_by_id(gmp_env, id);
6355 +
6356 + if (nevent)
6357 + TRACE("EVENT R%d update prev = %llu, new = %llu\n", nevent->id, nevent->next_update, when);
6358 +
6359 + if (nevent && nevent->next_update > when) {
6360 + list_del(&nevent->list);
6361 + update = 1;
6362 +
6363 + }
6364 +
6365 + if (!nevent || nevent->type != type || update == 1) {
6366 + if (update == 0)
6367 + nevent = kzalloc(sizeof(*nevent), GFP_ATOMIC);
6368 + BUG_ON(!nevent);
6369 + nevent->next_update = when;
6370 + nevent->id = id;
6371 + nevent->type = type;
6372 + nevent->timer_armed_on = NO_CPU;
6373 +
6374 + list_for_each(pos, &gmp_env->next_events) {
6375 + queued = list_entry(pos, struct next_timer_event, list);
6376 + if (queued->next_update > nevent->next_update) {
6377 + list_add(&nevent->list, pos->prev);
6378 + found = 1;
6379 + TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at before %llu\n", nevent->id, nevent->type, nevent->next_update, queued->next_update);
6380 + break;
6381 + }
6382 + }
6383 +
6384 + if (!found) {
6385 + list_add_tail(&nevent->list, &gmp_env->next_events);
6386 + TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at TAIL\n", nevent->id, nevent->type, nevent->next_update);
6387 + }
6388 + } else {
6389 + //TRACE("EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when);
6390 +; //printk(KERN_ALERT "EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when);
6391 + }
6392 +
6393 + TRACE("======START PRINTING EVENT LIST======\n");
6394 + gmp_print_events(gmp_env, litmus_clock());
6395 + TRACE("======FINISH PRINTING EVENT LIST======\n");
6396 +}
6397 +
6398 +void gmp_add_event_after(
6399 + struct gmp_reservation_environment* gmp_env, lt_t timeout, unsigned int id, event_type_t type)
6400 +{
6401 + //printk(KERN_ALERT "ADD_EVENT_AFTER id = %d\n", id);
6402 + gmp_add_event(gmp_env, gmp_env->env.current_time + timeout, id, type);
6403 +}
6404 +
6405 +static void gmp_queue_depleted(
6406 + struct gmp_reservation_environment* gmp_env,
6407 + struct reservation *res)
6408 +{
6409 + struct list_head *pos;
6410 + struct reservation *queued;
6411 + int found = 0;
6412 +
6413 +//printk(KERN_ALERT "R%d request to enqueue depleted_list\n", res->id);
6414 +
6415 + list_for_each(pos, &gmp_env->depleted_reservations) {
6416 + queued = list_entry(pos, struct reservation, list);
6417 + if (queued && (queued->next_replenishment > res->next_replenishment)) {
6418 +//printk(KERN_ALERT "QUEUED R%d %llu\n", queued->id, queued->next_replenishment);
6419 + list_add(&res->list, pos->prev);
6420 + found = 1;
6421 + break;
6422 + }
6423 + }
6424 +
6425 + if (!found)
6426 + list_add_tail(&res->list, &gmp_env->depleted_reservations);
6427 +
6428 + TRACE("R%d queued to depleted_list\n", res->id);
6429 +//printk(KERN_ALERT "R%d queued to depleted_list\n", res->id);
6430 + gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
6431 +}
6432 +
6433 +static void gmp_queue_active(
6434 + struct gmp_reservation_environment* gmp_env,
6435 + struct reservation *res)
6436 +{
6437 + struct list_head *pos;
6438 + struct reservation *queued;
6439 + int check_preempt = 1, found = 0;
6440 +
6441 + list_for_each(pos, &gmp_env->active_reservations) {
6442 + queued = list_entry(pos, struct reservation, list);
6443 + if (queued->priority > res->priority) {
6444 + list_add(&res->list, pos->prev);
6445 + found = 1;
6446 + break;
6447 + } else if (queued->scheduled_on == NO_CPU)
6448 + check_preempt = 0;
6449 + }
6450 +
6451 + if (!found)
6452 + list_add_tail(&res->list, &gmp_env->active_reservations);
6453 +
6454 + /* check for possible preemption */
6455 + if (res->state == RESERVATION_ACTIVE && check_preempt)
6456 + gmp_env->schedule_now++;
6457 +
6458 +#if BUDGET_ENFORCEMENT_AT_C
6459 + gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN);
6460 +#endif
6461 + res->event_added = 1;
6462 +}
6463 +
6464 +static void gmp_queue_reservation(
6465 + struct gmp_reservation_environment* gmp_env,
6466 + struct reservation *res)
6467 +{
6468 +
6469 +//printk(KERN_ALERT "DEBUG: Passed %s %d %p R%d STATE %d\n",__FUNCTION__,__LINE__, gmp_env, res->id, res->state);
6470 + switch (res->state) {
6471 + case RESERVATION_INACTIVE:
6472 + list_add(&res->list, &gmp_env->inactive_reservations);
6473 + break;
6474 +
6475 + case RESERVATION_DEPLETED:
6476 + gmp_queue_depleted(gmp_env, res);
6477 + break;
6478 +
6479 + case RESERVATION_ACTIVE_IDLE:
6480 + case RESERVATION_ACTIVE:
6481 + gmp_queue_active(gmp_env, res);
6482 + break;
6483 + }
6484 +}
6485 +
6486 +void gmp_add_new_reservation(
6487 + struct gmp_reservation_environment* gmp_env,
6488 + struct reservation* new_res)
6489 +{
6490 + new_res->env = &gmp_env->env;
6491 + gmp_queue_reservation(gmp_env, new_res);
6492 +}
6493 +
6494 +#if BUDGET_ENFORCEMENT_AT_C
6495 +static void gmp_charge_budget(
6496 + struct gmp_reservation_environment* gmp_env,
6497 + lt_t delta)
6498 +{
6499 + struct list_head *pos, *next;
6500 + struct reservation *res;
6501 +
6502 + list_for_each_safe(pos, next, &gmp_env->active_reservations) {
6503 + int drained = 0;
6504 + /* charge all ACTIVE_IDLE up to the first ACTIVE reservation */
6505 + res = list_entry(pos, struct reservation, list);
6506 + if (res->state == RESERVATION_ACTIVE) {
6507 + TRACE("gmp_charge_budget ACTIVE R%u scheduled_on=%d drain %llu\n", res->id, res->scheduled_on, delta);
6508 + if (res->scheduled_on != NO_CPU && res->blocked_by_ghost == 0) {
6509 + TRACE("DRAIN !!\n");
6510 + drained = 1;
6511 + res->ops->drain_budget(res, delta);
6512 + } else {
6513 + TRACE("NO DRAIN (not scheduled)!!\n");
6514 + }
6515 + } else {
6516 + //BUG_ON(res->state != RESERVATION_ACTIVE_IDLE);
6517 + if (res->state != RESERVATION_ACTIVE_IDLE)
6518 + TRACE("BUG!!!!!!!!!!!! gmp_charge_budget()\n");
6519 + TRACE("gmp_charge_budget INACTIVE R%u drain %llu\n", res->id, delta);
6520 + //if (res->is_ghost != NO_CPU) {
6521 + TRACE("DRAIN !!\n");
6522 + drained = 1;
6523 + res->ops->drain_budget(res, delta);
6524 + //}
6525 + }
6526 + if ((res->state == RESERVATION_ACTIVE ||
6527 + res->state == RESERVATION_ACTIVE_IDLE) && (drained == 1))
6528 + {
6529 + /* make sure scheduler is invoked when this reservation expires
6530 + * its remaining budget */
6531 + TRACE("requesting gmp_scheduler update for reservation %u in %llu nanoseconds\n", res->id, res->cur_budget);
6532 + gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN);
6533 + res->event_added = 1;
6534 + }
6535 + //if (encountered_active == 2)
6536 + /* stop at the first ACTIVE reservation */
6537 + // break;
6538 + }
6539 + //TRACE("finished charging budgets\n");
6540 +}
6541 +#else
6542 +
6543 +static void gmp_charge_budget(
6544 + struct gmp_reservation_environment* gmp_env,
6545 + lt_t delta)
6546 +{
6547 + return;
6548 +}
6549 +
6550 +#endif
6551 +
6552 +static void gmp_replenish_budgets(struct gmp_reservation_environment* gmp_env)
6553 +{
6554 + struct list_head *pos, *next;
6555 + struct reservation *res;
6556 +
6557 + list_for_each_safe(pos, next, &gmp_env->depleted_reservations) {
6558 + res = list_entry(pos, struct reservation, list);
6559 + if (res->next_replenishment <= gmp_env->env.current_time) {
6560 + res->ops->replenish(res);
6561 + if (res->is_ghost != NO_CPU) {
6562 + TRACE("R%d replenished! scheduled_on=%d\n", res->id, res->scheduled_on);
6563 + }
6564 + } else {
6565 + /* list is ordered by increasing depletion times */
6566 + break;
6567 + }
6568 + }
6569 + //TRACE("finished replenishing budgets\n");
6570 +}
6571 +
6572 +#define EPSILON 50
6573 +
6574 +/* return schedule_now */
6575 +int gmp_update_time(
6576 + struct gmp_reservation_environment* gmp_env,
6577 + lt_t now)
6578 +{
6579 + struct next_timer_event *event, *next;
6580 + lt_t delta, ret;
6581 +
6582 + /* If the time didn't advance, there is nothing to do.
6583 + * This check makes it safe to call sup_advance_time() potentially
6584 + * multiple times (e.g., via different code paths. */
6585 + //TRACE("(gmp_update_time) now: %llu, current_time: %llu\n", now, gmp_env->env.current_time);
6586 + if (unlikely(now <= gmp_env->env.current_time + EPSILON))
6587 + return 0;
6588 +
6589 + delta = now - gmp_env->env.current_time;
6590 + gmp_env->env.current_time = now;
6591 +
6592 +
6593 + //gmp_print_events(gmp_env, now);
6594 + /* deplete budgets by passage of time */
6595 + //TRACE("CHARGE###\n");
6596 + gmp_charge_budget(gmp_env, delta);
6597 +
6598 + /* check if any budgets where replenished */
6599 + //TRACE("REPLENISH###\n");
6600 + gmp_replenish_budgets(gmp_env);
6601 +
6602 +
6603 + list_for_each_entry_safe(event, next, &gmp_env->next_events, list) {
6604 + if (event->next_update < now) {
6605 + list_del(&event->list);
6606 + //TRACE("EVENT at %llu IS DELETED\n", event->next_update);
6607 + kfree(event);
6608 + } else {
6609 + break;
6610 + }
6611 + }
6612 +
6613 + //gmp_print_events(gmp_env, litmus_clock());
6614 +
6615 + ret = min(gmp_env->schedule_now, NR_CPUS);
6616 + gmp_env->schedule_now = 0;
6617 +
6618 + return ret;
6619 +}
6620 +
6621 +void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now)
6622 +{
6623 + struct next_timer_event *event, *next;
6624 +
6625 + TRACE("GLOBAL EVENTS now=%llu\n", now);
6626 + list_for_each_entry_safe(event, next, &gmp_env->next_events, list) {
6627 + TRACE("at %llu type=%d id=%d armed_on=%d\n", event->next_update, event->type, event->id, event->timer_armed_on);
6628 + }
6629 +}
6630 +
6631 +static void gmp_res_change_state(
6632 + struct reservation_environment* env,
6633 + struct reservation *res,
6634 + reservation_state_t new_state)
6635 +{
6636 + struct gmp_reservation_environment* gmp_env;
6637 +
6638 + gmp_env = container_of(env, struct gmp_reservation_environment, env);
6639 +
6640 + TRACE("GMP reservation R%d state %d->%d at %llu\n",
6641 + res->id, res->state, new_state, env->current_time);
6642 +
6643 + list_del(&res->list);
6644 + /* check if we need to reschedule because we lost an active reservation */
6645 + if (res->state == RESERVATION_ACTIVE)
6646 + gmp_env->schedule_now++;
6647 + res->state = new_state;
6648 + gmp_queue_reservation(gmp_env, res);
6649 +}
6650 +
6651 +void gmp_init(struct gmp_reservation_environment* gmp_env)
6652 +{
6653 + memset(gmp_env, sizeof(*gmp_env), 0);
6654 +
6655 + INIT_LIST_HEAD(&gmp_env->active_reservations);
6656 + INIT_LIST_HEAD(&gmp_env->depleted_reservations);
6657 + INIT_LIST_HEAD(&gmp_env->inactive_reservations);
6658 + INIT_LIST_HEAD(&gmp_env->next_events);
6659 +
6660 + gmp_env->env.change_state = gmp_res_change_state;
6661 +
6662 + gmp_env->schedule_now = 0;
6663 + gmp_env->will_schedule = false;
6664 +
6665 + raw_spin_lock_init(&gmp_env->lock);
6666 +}
6667 diff --git litmus/sched_mc2.c litmus/sched_mc2.c
6668 new file mode 100644
6669 index 0000000..b4b159b
6670 --- /dev/null
6671 +++ litmus/sched_mc2.c
6672 @@ -0,0 +1,1593 @@
6673 +/*
6674 + * litmus/sched_mc2.c
6675 + *
6676 + * Implementation of the Mixed-Criticality on MultiCore scheduler
6677 + *
6678 + * This plugin implements a scheduling algorithm proposed in
6679 + * "Mixed-Criticality Real-Time Scheduling for Multicore System" paper.
6680 + */
6681 +
6682 +#include <linux/percpu.h>
6683 +#include <linux/slab.h>
6684 +#include <asm/uaccess.h>
6685 +
6686 +#include <litmus/sched_plugin.h>
6687 +#include <litmus/preempt.h>
6688 +#include <litmus/debug_trace.h>
6689 +
6690 +#include <litmus/litmus.h>
6691 +#include <litmus/jobs.h>
6692 +#include <litmus/budget.h>
6693 +#include <litmus/litmus_proc.h>
6694 +#include <litmus/sched_trace.h>
6695 +#include <litmus/cache_proc.h>
6696 +#include <litmus/trace.h>
6697 +
6698 +#include <litmus/mc2_common.h>
6699 +#include <litmus/reservation.h>
6700 +#include <litmus/polling_reservations.h>
6701 +
6702 +#define BUDGET_ENFORCEMENT_AT_C 0
6703 +
6704 +extern void do_partition(enum crit_level lv, int cpu);
6705 +
6706 +/* _global_env - reservation container for level-C tasks*/
6707 +struct gmp_reservation_environment _global_env;
6708 +
6709 +/* cpu_entry - keep track of a running task on a cpu
6710 + * This state is used to decide the lowest priority cpu
6711 + */
6712 +struct cpu_entry {
6713 + struct task_struct *scheduled;
6714 + lt_t deadline;
6715 + int cpu;
6716 + enum crit_level lv;
6717 + /* if will_schedule is true, this cpu is already selected and
6718 + call mc2_schedule() soon. */
6719 + bool will_schedule;
6720 +};
6721 +
6722 +/* cpu_priority - a global state for choosing the lowest priority CPU */
6723 +struct cpu_priority {
6724 + raw_spinlock_t lock;
6725 + struct cpu_entry cpu_entries[NR_CPUS];
6726 +};
6727 +
6728 +struct cpu_priority _lowest_prio_cpu;
6729 +
6730 +/* mc2_task_state - a task state structure */
6731 +struct mc2_task_state {
6732 + struct task_client res_info;
6733 + /* if cpu == -1, this task is a global task (level C) */
6734 + int cpu;
6735 + bool has_departed;
6736 + struct mc2_task mc2_param;
6737 +};
6738 +
6739 +/* crit_entry - maintain the logically running job (ghost job) */
6740 +struct crit_entry {
6741 + enum crit_level level;
6742 + struct task_struct *running;
6743 +};
6744 +
6745 +/* mc2_cpu_state - maintain the scheduled state and ghost jobs
6746 + * timer : timer for partitioned tasks (level A and B)
6747 + * g_timer : timer for global tasks (level C)
6748 + */
6749 +struct mc2_cpu_state {
6750 + raw_spinlock_t lock;
6751 +
6752 + struct sup_reservation_environment sup_env;
6753 + struct hrtimer timer;
6754 +
6755 + int cpu;
6756 + struct task_struct* scheduled;
6757 + struct crit_entry crit_entries[NUM_CRIT_LEVELS];
6758 +};
6759 +
6760 +static int resched_cpu[NR_CPUS];
6761 +static DEFINE_PER_CPU(struct mc2_cpu_state, mc2_cpu_state);
6762 +static int level_a_priorities[NR_CPUS];
6763 +
6764 +#define cpu_state_for(cpu_id) (&per_cpu(mc2_cpu_state, cpu_id))
6765 +#define local_cpu_state() (this_cpu_ptr(&mc2_cpu_state))
6766 +
6767 +/* get_mc2_state - get the task's state */
6768 +static struct mc2_task_state* get_mc2_state(struct task_struct *tsk)
6769 +{
6770 + struct mc2_task_state* tinfo;
6771 +
6772 + tinfo = (struct mc2_task_state*)tsk_rt(tsk)->plugin_state;
6773 +
6774 + if (tinfo)
6775 + return tinfo;
6776 + else
6777 + return NULL;
6778 +}
6779 +
6780 +/* get_task_crit_level - return the criticaility level of a task */
6781 +static enum crit_level get_task_crit_level(struct task_struct *tsk)
6782 +{
6783 + struct mc2_task *mp;
6784 +
6785 + if (!tsk || !is_realtime(tsk))
6786 + return NUM_CRIT_LEVELS;
6787 +
6788 + mp = tsk_rt(tsk)->mc2_data;
6789 +
6790 + if (!mp)
6791 + return NUM_CRIT_LEVELS;
6792 + else
6793 + return mp->crit;
6794 +}
6795 +
6796 +/* task_depart - remove a task from its reservation
6797 + * If the job has remaining budget, convert it to a ghost job
6798 + * and update crit_entries[]
6799 + *
6800 + * @job_complete indicate whether job completes or not
6801 + */
6802 +static void task_departs(struct task_struct *tsk, int job_complete)
6803 +{
6804 + struct mc2_task_state* tinfo = get_mc2_state(tsk);
6805 +
6806 + struct reservation* res = NULL;
6807 + struct reservation_client *client = NULL;
6808 +
6809 + BUG_ON(!is_realtime(tsk));
6810 +
6811 + res = tinfo->res_info.client.reservation;
6812 + client = &tinfo->res_info.client;
6813 + BUG_ON(!res);
6814 + BUG_ON(!client);
6815 +
6816 + /* No ghost job handling, empty remaining budget */
6817 + if (job_complete) {
6818 + res->cur_budget = 0;
6819 + sched_trace_task_completion(tsk, 0);
6820 + }
6821 +
6822 + res->ops->client_departs(res, client, job_complete);
6823 + tinfo->has_departed = true;
6824 + TRACE_TASK(tsk, "Client departs with budget %llu at %llu\n", res->cur_budget, litmus_clock());
6825 +}
6826 +
6827 +/* task_arrive - put a task into its reservation
6828 + * If the job was a ghost job, remove it from crit_entries[]
6829 + */
6830 +static void task_arrives(struct mc2_cpu_state *state, struct task_struct *tsk)
6831 +{
6832 + struct mc2_task_state* tinfo = get_mc2_state(tsk);
6833 + struct reservation* res;
6834 + struct reservation_client *client;
6835 + //enum crit_level lv = get_task_crit_level(tsk);
6836 +
6837 + res = tinfo->res_info.client.reservation;
6838 + client = &tinfo->res_info.client;
6839 +
6840 + tinfo->has_departed = false;
6841 +
6842 + res->ops->client_arrives(res, client);
6843 + TRACE_TASK(tsk, "Client arrives at %llu\n", litmus_clock());
6844 +/*
6845 + if (lv != NUM_CRIT_LEVELS) {
6846 + struct crit_entry *ce;
6847 + ce = &state->crit_entries[lv];
6848 + // if the currrent task is a ghost job, remove it
6849 + if (ce->running == tsk)
6850 + ce->running = NULL;
6851 + }
6852 +*/
6853 +}
6854 +
6855 +/* get_lowest_prio_cpu - return the lowest priority cpu
6856 + * This will be used for scheduling level-C tasks.
6857 + * If all CPUs are running tasks which has
6858 + * higher priority than level C, return NO_CPU.
6859 + */
6860 +static int get_lowest_prio_cpu(lt_t priority)
6861 +{
6862 + struct cpu_entry *ce;
6863 + int cpu, ret = NO_CPU;
6864 + lt_t latest_deadline = 0;
6865 +
6866 + if (priority == LITMUS_NO_PRIORITY)
6867 + return ret;
6868 +
6869 + ce = &_lowest_prio_cpu.cpu_entries[local_cpu_state()->cpu];
6870 + if (!ce->will_schedule && !ce->scheduled) {
6871 + TRACE("CPU %d (local) is the lowest!\n", ce->cpu);
6872 + return ce->cpu;
6873 + } else {
6874 + TRACE("Local CPU will_schedule=%d, scheduled=(%s/%d)\n", ce->will_schedule, ce->scheduled ? (ce->scheduled)->comm : "null", ce->scheduled ? (ce->scheduled)->pid : 0);
6875 + }
6876 +
6877 + for_each_online_cpu(cpu) {
6878 + ce = &_lowest_prio_cpu.cpu_entries[cpu];
6879 + /* If a CPU will call schedule() in the near future, we don't
6880 + return that CPU. */
6881 + /*
6882 + TRACE("CPU %d will_schedule=%d, scheduled=(%s/%d:%d)\n", cpu, ce->will_schedule,
6883 + ce->scheduled ? (ce->scheduled)->comm : "null",
6884 + ce->scheduled ? (ce->scheduled)->pid : 0,
6885 + ce->scheduled ? (ce->scheduled)->rt_param.job_params.job_no : 0);
6886 + */
6887 + if (!ce->will_schedule) {
6888 + if (!ce->scheduled) {
6889 + /* Idle cpu, return this. */
6890 + TRACE("CPU %d is the lowest!\n", ce->cpu);
6891 + return ce->cpu;
6892 + } else if (ce->lv == CRIT_LEVEL_C &&
6893 + ce->deadline > latest_deadline) {
6894 + latest_deadline = ce->deadline;
6895 + ret = ce->cpu;
6896 + }
6897 + }
6898 + }
6899 +
6900 + if (priority >= latest_deadline)
6901 + ret = NO_CPU;
6902 +
6903 + TRACE("CPU %d is the lowest!\n", ret);
6904 +
6905 + return ret;
6906 +}
6907 +
6908 +/* NOTE: drops state->lock */
6909 +/* mc2_update_timer_and_unlock - set a timer and g_timer and unlock
6910 + * Whenever res_env.current_time is updated,
6911 + * we check next_scheduler_update and set
6912 + * a timer.
6913 + * If there exist a global event which is
6914 + * not armed on any CPU and g_timer is not
6915 + * active, set a g_timer for that event.
6916 + */
6917 +static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
6918 +{
6919 + int local, cpus;
6920 + lt_t update, now;
6921 + struct next_timer_event *event, *next;
6922 + int reschedule[NR_CPUS];
6923 + unsigned long flags;
6924 +
6925 + local_irq_save(flags);
6926 +
6927 + for (cpus = 0; cpus<NR_CPUS; cpus++)
6928 + reschedule[cpus] = 0;
6929 +
6930 + update = state->sup_env.next_scheduler_update;
6931 + now = state->sup_env.env.current_time;
6932 +
6933 + /* Be sure we're actually running on the right core,
6934 + * as pres_update_timer() is also called from pres_task_resume(),
6935 + * which might be called on any CPU when a thread resumes.
6936 + */
6937 + local = local_cpu_state() == state;
6938 +
6939 + raw_spin_lock(&_global_env.lock);
6940 +
6941 + list_for_each_entry_safe(event, next, &_global_env.next_events, list) {
6942 + /* If the event time is already passed, we call schedule() on
6943 + the lowest priority cpu */
6944 + if (event->next_update >= update) {
6945 + break;
6946 + }
6947 +
6948 + if (event->next_update < litmus_clock()) {
6949 + if (event->timer_armed_on == NO_CPU) {
6950 + struct reservation *res = gmp_find_by_id(&_global_env, event->id);
6951 + int cpu = get_lowest_prio_cpu(res?res->priority:0);
6952 + //TRACE("GLOBAL EVENT PASSED!! poking CPU %d to reschedule\n", cpu);
6953 + list_del(&event->list);
6954 + kfree(event);
6955 + if (cpu != NO_CPU) {
6956 + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
6957 + reschedule[cpu] = 1;
6958 + }
6959 + }
6960 + } else if (event->next_update < update && (event->timer_armed_on == NO_CPU || event->timer_armed_on == state->cpu)) {
6961 + event->timer_armed_on = state->cpu;
6962 + update = event->next_update;
6963 + break;
6964 + }
6965 + }
6966 +
6967 + /* Must drop state lock before calling into hrtimer_start(), which
6968 + * may raise a softirq, which in turn may wake ksoftirqd. */
6969 + raw_spin_unlock(&_global_env.lock);
6970 + local_irq_restore(flags);
6971 + raw_spin_unlock(&state->lock);
6972 +
6973 + if (update <= now || reschedule[state->cpu]) {
6974 + reschedule[state->cpu] = 0;
6975 + litmus_reschedule(state->cpu);
6976 + } else if (likely(local && update != SUP_NO_SCHEDULER_UPDATE)) {
6977 + /* Reprogram only if not already set correctly. */
6978 + if (!hrtimer_active(&state->timer) ||
6979 + ktime_to_ns(hrtimer_get_expires(&state->timer)) != update) {
6980 + TRACE("canceling timer...at %llu\n",
6981 + ktime_to_ns(hrtimer_get_expires(&state->timer)));
6982 + hrtimer_cancel(&state->timer);
6983 + TRACE("setting scheduler timer for %llu\n", update);
6984 + /* We cannot use hrtimer_start() here because the
6985 + * wakeup flag must be set to zero. */
6986 + __hrtimer_start_range_ns(&state->timer,
6987 + ns_to_ktime(update),
6988 + 0 /* timer coalescing slack */,
6989 + HRTIMER_MODE_ABS_PINNED,
6990 + 0 /* wakeup */);
6991 + if (update < litmus_clock()) {
6992 + /* uh oh, timer expired while trying to set it */
6993 + TRACE("timer expired during setting "
6994 + "update:%llu now:%llu actual:%llu\n",
6995 + update, now, litmus_clock());
6996 + /* The timer HW may not have been reprogrammed
6997 + * correctly; force rescheduling now. */
6998 + litmus_reschedule(state->cpu);
6999 + }
7000 + }
7001 + } else if (unlikely(!local && update != SUP_NO_SCHEDULER_UPDATE)) {
7002 + /* Poke remote core only if timer needs to be set earlier than
7003 + * it is currently set.
7004 + */
7005 + TRACE("mc2_update_timer for remote CPU %d (update=%llu, "
7006 + "active:%d, set:%llu)\n",
7007 + state->cpu, update, hrtimer_active(&state->timer),
7008 + ktime_to_ns(hrtimer_get_expires(&state->timer)));
7009 + if (!hrtimer_active(&state->timer) ||
7010 + ktime_to_ns(hrtimer_get_expires(&state->timer)) > update) {
7011 + TRACE("poking CPU %d so that it can update its "
7012 + "scheduling timer (active:%d, set:%llu)\n",
7013 + state->cpu,
7014 + hrtimer_active(&state->timer),
7015 + ktime_to_ns(hrtimer_get_expires(&state->timer)));
7016 + //raw_spin_lock(&state->lock);
7017 + //preempt_if_preemptable(state->scheduled, state->cpu);
7018 + //raw_spin_unlock(&state->lock);
7019 + //reschedule[state->cpu] = 0;
7020 + }
7021 + }
7022 +
7023 + for (cpus = 0; cpus<NR_CPUS; cpus++) {
7024 + if (reschedule[cpus]) {
7025 + litmus_reschedule(cpus);
7026 + }
7027 + }
7028 +
7029 +}
7030 +
7031 +/* update_cpu_prio - Update cpu's priority
7032 + * When a cpu picks a new task, call this function
7033 + * to update cpu priorities.
7034 + */
7035 +static void update_cpu_prio(struct mc2_cpu_state *state)
7036 +{
7037 + struct cpu_entry *ce = &_lowest_prio_cpu.cpu_entries[state->cpu];
7038 + enum crit_level lv = get_task_crit_level(state->scheduled);
7039 +
7040 + if (!state->scheduled) {
7041 + /* cpu is idle. */
7042 + ce->scheduled = NULL;
7043 + ce->deadline = ULLONG_MAX;
7044 + ce->lv = NUM_CRIT_LEVELS;
7045 + } else if (lv == CRIT_LEVEL_C) {
7046 + ce->scheduled = state->scheduled;
7047 + ce->deadline = get_deadline(state->scheduled);
7048 + ce->lv = lv;
7049 + } else if (lv < CRIT_LEVEL_C) {
7050 + /* If cpu is running level A or B tasks, it is not eligible
7051 + to run level-C tasks */
7052 + ce->scheduled = state->scheduled;
7053 + ce->deadline = 0;
7054 + ce->lv = lv;
7055 + }
7056 +};
7057 +
7058 +/* on_scheduling_timer - timer event for partitioned tasks
7059 + */
7060 +static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
7061 +{
7062 + unsigned long flags;
7063 + enum hrtimer_restart restart = HRTIMER_NORESTART;
7064 + struct mc2_cpu_state *state;
7065 + lt_t update, now;
7066 + int global_schedule_now;
7067 + int reschedule[NR_CPUS];
7068 + int cpus;
7069 +
7070 + for (cpus = 0; cpus<NR_CPUS; cpus++)
7071 + reschedule[cpus] = 0;
7072 +
7073 + state = container_of(timer, struct mc2_cpu_state, timer);
7074 +
7075 + /* The scheduling timer should only fire on the local CPU, because
7076 + * otherwise deadlocks via timer_cancel() are possible.
7077 + * Note: this does not interfere with dedicated interrupt handling, as
7078 + * even under dedicated interrupt handling scheduling timers for
7079 + * budget enforcement must occur locally on each CPU.
7080 + */
7081 + BUG_ON(state->cpu != raw_smp_processor_id());
7082 +
7083 + TS_ISR_START;
7084 +
7085 + TRACE("Timer fired at %llu\n", litmus_clock());
7086 + raw_spin_lock_irqsave(&state->lock, flags);
7087 + now = litmus_clock();
7088 + sup_update_time(&state->sup_env, now);
7089 +
7090 + update = state->sup_env.next_scheduler_update;
7091 + now = state->sup_env.env.current_time;
7092 +
7093 + if (update <= now) {
7094 + litmus_reschedule_local();
7095 + } else if (update != SUP_NO_SCHEDULER_UPDATE) {
7096 + hrtimer_set_expires(timer, ns_to_ktime(update));
7097 + restart = HRTIMER_RESTART;
7098 + }
7099 +
7100 + raw_spin_lock(&_global_env.lock);
7101 + global_schedule_now = gmp_update_time(&_global_env, now);
7102 +
7103 + BUG_ON(global_schedule_now < 0 || global_schedule_now > 4);
7104 +
7105 + /* Find the lowest cpu, and call reschedule */
7106 + while (global_schedule_now--) {
7107 + int cpu = get_lowest_prio_cpu(0);
7108 + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) {
7109 + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
7110 + if (cpu == state->cpu && update > now)
7111 + ; //litmus_reschedule_local();
7112 + else
7113 + reschedule[cpu] = 1;
7114 + }
7115 + }
7116 + raw_spin_unlock(&_global_env.lock);
7117 + raw_spin_unlock_irqrestore(&state->lock, flags);
7118 +
7119 + TS_ISR_END;
7120 +
7121 + for (cpus = 0; cpus<NR_CPUS; cpus++) {
7122 + if (reschedule[cpus]) {
7123 + litmus_reschedule(cpus);
7124 + }
7125 + }
7126 +
7127 + return restart;
7128 +}
7129 +
7130 +/* mc2_complete_job - syscall backend for job completions
7131 + */
7132 +static long mc2_complete_job(void)
7133 +{
7134 + ktime_t next_release;
7135 + long err;
7136 +
7137 + tsk_rt(current)->completed = 1;
7138 +
7139 + /* If this the first job instance, we need to reset replenish
7140 + time to the next release time */
7141 + if (tsk_rt(current)->sporadic_release) {
7142 + struct mc2_cpu_state *state;
7143 + struct reservation_environment *env;
7144 + struct mc2_task_state *tinfo;
7145 + struct reservation *res = NULL;
7146 + unsigned long flags;
7147 + enum crit_level lv;
7148 +
7149 + //preempt_disable();
7150 + local_irq_save(flags);
7151 +
7152 + tinfo = get_mc2_state(current);
7153 + lv = get_task_crit_level(current);
7154 +
7155 + if (lv < CRIT_LEVEL_C) {
7156 + state = cpu_state_for(tinfo->cpu);
7157 + raw_spin_lock(&state->lock);
7158 + env = &(state->sup_env.env);
7159 + res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id);
7160 + env->time_zero = tsk_rt(current)->sporadic_release_time;
7161 + }
7162 + else if (lv == CRIT_LEVEL_C) {
7163 + state = local_cpu_state();
7164 + raw_spin_lock(&state->lock);
7165 + raw_spin_lock(&_global_env.lock);
7166 + res = gmp_find_by_id(&_global_env, tinfo->mc2_param.res_id);
7167 + _global_env.env.time_zero = tsk_rt(current)->sporadic_release_time;
7168 + }
7169 + else
7170 + BUG();
7171 +
7172 + /* set next_replenishtime to synchronous release time */
7173 + BUG_ON(!res);
7174 + res->next_replenishment = tsk_rt(current)->sporadic_release_time;
7175 + res->cur_budget = 0;
7176 + res->env->change_state(res->env, res, RESERVATION_DEPLETED);
7177 +
7178 + if (lv == CRIT_LEVEL_C)
7179 + raw_spin_unlock(&_global_env.lock);
7180 +
7181 + raw_spin_unlock(&state->lock);
7182 + local_irq_restore(flags);
7183 + //preempt_enable();
7184 + }
7185 +
7186 + sched_trace_task_completion(current, 0);
7187 + /* update the next release time and deadline */
7188 + prepare_for_next_period(current);
7189 + sched_trace_task_release(current);
7190 + next_release = ns_to_ktime(get_release(current));
7191 + preempt_disable();
7192 + TRACE_CUR("next_release=%llu\n", get_release(current));
7193 + if (get_release(current) > litmus_clock()) {
7194 + /* sleep until next_release */
7195 + set_current_state(TASK_INTERRUPTIBLE);
7196 + preempt_enable_no_resched();
7197 + err = schedule_hrtimeout(&next_release, HRTIMER_MODE_ABS);
7198 + } else {
7199 + /* release the next job immediately */
7200 + err = 0;
7201 + TRACE_CUR("TARDY: release=%llu now=%llu\n", get_release(current), litmus_clock());
7202 + preempt_enable();
7203 + }
7204 +
7205 + TRACE_CUR("mc2_complete_job returns at %llu\n", litmus_clock());
7206 +
7207 + tsk_rt(current)->completed = 0;
7208 + return err;
7209 +}
7210 +
7211 +/* mc2_dispatch - Select the next task to schedule.
7212 + */
7213 +struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, struct mc2_cpu_state* state)
7214 +{
7215 + struct reservation *res, *next;
7216 + struct task_struct *tsk = NULL;
7217 + struct crit_entry *ce;
7218 + enum crit_level lv;
7219 + lt_t time_slice;
7220 +
7221 + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) {
7222 + if (res->state == RESERVATION_ACTIVE) {
7223 + tsk = res->ops->dispatch_client(res, &time_slice);
7224 + if (likely(tsk)) {
7225 + lv = get_task_crit_level(tsk);
7226 + if (lv == NUM_CRIT_LEVELS) {
7227 + sup_scheduler_update_after(sup_env, res->cur_budget);
7228 + return tsk;
7229 + } else {
7230 + ce = &state->crit_entries[lv];
7231 + sup_scheduler_update_after(sup_env, res->cur_budget);
7232 + res->blocked_by_ghost = 0;
7233 + res->is_ghost = NO_CPU;
7234 + return tsk;
7235 + }
7236 + }
7237 + }
7238 + }
7239 +
7240 + return NULL;
7241 +}
7242 +
7243 +struct task_struct* mc2_global_dispatch(struct mc2_cpu_state* state)
7244 +{
7245 + struct reservation *res, *next;
7246 + struct task_struct *tsk = NULL;
7247 + enum crit_level lv;
7248 + lt_t time_slice;
7249 +
7250 + list_for_each_entry_safe(res, next, &_global_env.active_reservations, list) {
7251 + BUG_ON(!res);
7252 + if (res->state == RESERVATION_ACTIVE && res->scheduled_on == NO_CPU) {
7253 + tsk = res->ops->dispatch_client(res, &time_slice);
7254 + if (likely(tsk)) {
7255 + lv = get_task_crit_level(tsk);
7256 + if (lv != CRIT_LEVEL_C)
7257 + BUG();
7258 +#if BUDGET_ENFORCEMENT_AT_C
7259 + gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
7260 +#endif
7261 + res->event_added = 1;
7262 + res->blocked_by_ghost = 0;
7263 + res->is_ghost = NO_CPU;
7264 + res->scheduled_on = state->cpu;
7265 + return tsk;
7266 + }
7267 + }
7268 + }
7269 +
7270 + return NULL;
7271 +}
7272 +
7273 +static inline void pre_schedule(struct task_struct *prev, int cpu)
7274 +{
7275 + TS_SCHED_A_START;
7276 + TS_SCHED_C_START;
7277 +
7278 + if (!prev || !is_realtime(prev))
7279 + return;
7280 +
7281 + do_partition(CRIT_LEVEL_C, cpu);
7282 +}
7283 +
7284 +static inline void post_schedule(struct task_struct *next, int cpu)
7285 +{
7286 + enum crit_level lev;
7287 + if ((!next) || !is_realtime(next)) {
7288 + //do_partition(NUM_CRIT_LEVELS, -1);
7289 + return;
7290 + }
7291 +
7292 + lev = get_task_crit_level(next);
7293 + do_partition(lev, cpu);
7294 +
7295 + switch(lev) {
7296 + case CRIT_LEVEL_A:
7297 + case CRIT_LEVEL_B:
7298 + TS_SCHED_A_END(next);
7299 + break;
7300 + case CRIT_LEVEL_C:
7301 + TS_SCHED_C_END(next);
7302 + break;
7303 + default:
7304 + break;
7305 + }
7306 +
7307 +}
7308 +
7309 +/* mc2_schedule - main scheduler function. pick the next task to run
7310 + */
7311 +static struct task_struct* mc2_schedule(struct task_struct * prev)
7312 +{
7313 + int np, blocks, exists, cpu; //preempt, to_schedule;
7314 + /* next == NULL means "schedule background work". */
7315 + lt_t now = litmus_clock();
7316 + struct mc2_cpu_state *state = local_cpu_state();
7317 +
7318 + raw_spin_lock(&state->lock);
7319 +
7320 + pre_schedule(prev, state->cpu);
7321 +
7322 + if (state->scheduled && state->scheduled != prev)
7323 + printk(KERN_ALERT "BUG1!!!!!!!! %s %s\n", state->scheduled ? (state->scheduled)->comm : "null", prev ? (prev)->comm : "null");
7324 + if (state->scheduled && !is_realtime(prev))
7325 + printk(KERN_ALERT "BUG2!!!!!!!! \n");
7326 +
7327 + /* (0) Determine state */
7328 + exists = state->scheduled != NULL;
7329 + blocks = exists && !is_current_running();
7330 + np = exists && is_np(state->scheduled);
7331 +
7332 + /* update time */
7333 + state->sup_env.will_schedule = true;
7334 + sup_update_time(&state->sup_env, now);
7335 +
7336 + if (is_realtime(current) && blocks) {
7337 + if (get_task_crit_level(current) == CRIT_LEVEL_C)
7338 + raw_spin_lock(&_global_env.lock);
7339 + task_departs(current, is_completed(current));
7340 + if (get_task_crit_level(current) == CRIT_LEVEL_C)
7341 + raw_spin_unlock(&_global_env.lock);
7342 + }
7343 +
7344 + /* figure out what to schedule next */
7345 + if (!np)
7346 + state->scheduled = mc2_dispatch(&state->sup_env, state);
7347 +
7348 + if (!state->scheduled) {
7349 + raw_spin_lock(&_global_env.lock);
7350 + if (is_realtime(prev))
7351 + gmp_update_time(&_global_env, now);
7352 + state->scheduled = mc2_global_dispatch(state);
7353 + _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false;
7354 + update_cpu_prio(state);
7355 + raw_spin_unlock(&_global_env.lock);
7356 + } else {
7357 + raw_spin_lock(&_global_env.lock);
7358 + _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false;
7359 + update_cpu_prio(state);
7360 + raw_spin_unlock(&_global_env.lock);
7361 + }
7362 +
7363 + /* Notify LITMUS^RT core that we've arrived at a scheduling decision. */
7364 + sched_state_task_picked();
7365 +
7366 + /* program scheduler timer */
7367 + state->sup_env.will_schedule = false;
7368 +
7369 + /* NOTE: drops state->lock */
7370 + mc2_update_timer_and_unlock(state);
7371 +
7372 + raw_spin_lock(&state->lock);
7373 + if (prev != state->scheduled && is_realtime(prev)) {
7374 + struct mc2_task_state* tinfo = get_mc2_state(prev);
7375 + struct reservation* res = tinfo->res_info.client.reservation;
7376 + res->scheduled_on = NO_CPU;
7377 + TRACE_TASK(prev, "descheduled at %llu.\n", litmus_clock());
7378 + /* if prev is preempted and a global task, find the lowest cpu and reschedule */
7379 + if (tinfo->has_departed == false && get_task_crit_level(prev) == CRIT_LEVEL_C) {
7380 + int cpu;
7381 + raw_spin_lock(&_global_env.lock);
7382 + cpu = get_lowest_prio_cpu(res?res->priority:LITMUS_NO_PRIORITY);
7383 + //TRACE("LEVEL-C TASK PREEMPTED!! poking CPU %d to reschedule\n", cpu);
7384 + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) {
7385 + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
7386 + resched_cpu[cpu] = 1;
7387 + }
7388 + raw_spin_unlock(&_global_env.lock);
7389 + }
7390 + }
7391 +
7392 +/*
7393 + if (to_schedule != 0) {
7394 + raw_spin_lock(&_global_env.lock);
7395 + while (to_schedule--) {
7396 + int cpu = get_lowest_prio_cpu(0);
7397 + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) {
7398 + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
7399 + resched_cpu[cpu] = 1;
7400 + }
7401 + }
7402 + raw_spin_unlock(&_global_env.lock);
7403 + }
7404 +*/
7405 +
7406 + post_schedule(state->scheduled, state->cpu);
7407 +
7408 + raw_spin_unlock(&state->lock);
7409 + if (state->scheduled) {
7410 + TRACE_TASK(state->scheduled, "scheduled.\n");
7411 + }
7412 +
7413 + return state->scheduled;
7414 +}
7415 +
7416 +/* mc2_task_resume - Called when the state of tsk changes back to
7417 + * TASK_RUNNING. We need to requeue the task.
7418 + */
7419 +static void mc2_task_resume(struct task_struct *tsk)
7420 +{
7421 + unsigned long flags;
7422 + struct mc2_task_state* tinfo;
7423 + struct mc2_cpu_state *state;
7424 +
7425 + TRACE_TASK(tsk, "thread wakes up at %llu\n", litmus_clock());
7426 +
7427 + preempt_disable();
7428 + tinfo = get_mc2_state(tsk);
7429 + if (tinfo->cpu != -1)
7430 + state = cpu_state_for(tinfo->cpu);
7431 + else
7432 + state = local_cpu_state();
7433 + preempt_enable();
7434 +
7435 + /* Requeue only if self-suspension was already processed. */
7436 + if (tinfo->has_departed)
7437 + {
7438 +#ifdef CONFIG_SCHED_OVERHEAD_TRACE
7439 + switch(get_task_crit_level(tsk)) {
7440 + case CRIT_LEVEL_A:
7441 + TS_RELEASE_LATENCY_A(get_release(tsk));
7442 + break;
7443 + case CRIT_LEVEL_B:
7444 + TS_RELEASE_LATENCY_B(get_release(tsk));
7445 + break;
7446 + case CRIT_LEVEL_C:
7447 + TS_RELEASE_LATENCY_C(get_release(tsk));
7448 + break;
7449 + default:
7450 + break;
7451 + }
7452 +#endif
7453 +
7454 + raw_spin_lock_irqsave(&state->lock, flags);
7455 +
7456 + /* Assumption: litmus_clock() is synchronized across cores,
7457 + * since we might not actually be executing on tinfo->cpu
7458 + * at the moment. */
7459 + if (tinfo->cpu != -1) {
7460 + sup_update_time(&state->sup_env, litmus_clock());
7461 + task_arrives(state, tsk);
7462 + } else {
7463 + raw_spin_lock(&_global_env.lock);
7464 + gmp_update_time(&_global_env, litmus_clock());
7465 + task_arrives(state, tsk);
7466 + raw_spin_unlock(&_global_env.lock);
7467 + }
7468 +
7469 + /* NOTE: drops state->lock */
7470 + TRACE_TASK(tsk, "mc2_resume()\n");
7471 + raw_spin_unlock_irqrestore(&state->lock, flags);
7472 +
7473 + raw_spin_lock(&state->lock);
7474 + mc2_update_timer_and_unlock(state);
7475 + } else {
7476 + TRACE_TASK(tsk, "resume event ignored, still scheduled\n");
7477 + }
7478 +
7479 +}
7480 +
7481 +
7482 +/* mc2_admit_task - Setup mc2 task parameters
7483 + */
7484 +static long mc2_admit_task(struct task_struct *tsk)
7485 +{
7486 + long err = 0;
7487 + unsigned long flags;
7488 + struct reservation *res;
7489 + struct mc2_cpu_state *state;
7490 + struct mc2_task_state *tinfo = kzalloc(sizeof(*tinfo), GFP_ATOMIC);
7491 + struct mc2_task *mp = tsk_rt(tsk)->mc2_data;
7492 + enum crit_level lv;
7493 +
7494 + if (!tinfo)
7495 + return -ENOMEM;
7496 +
7497 + if (!mp) {
7498 + printk(KERN_ERR "mc2_admit_task: criticality level has not been set\n");
7499 + return -ESRCH;
7500 + }
7501 +
7502 + lv = mp->crit;
7503 +
7504 + if (lv < CRIT_LEVEL_C) {
7505 + state = cpu_state_for(task_cpu(tsk));
7506 + raw_spin_lock_irqsave(&state->lock, flags);
7507 +
7508 + res = sup_find_by_id(&state->sup_env, mp->res_id);
7509 +
7510 + /* found the appropriate reservation */
7511 + if (res) {
7512 + TRACE_TASK(tsk, "SUP FOUND RES ID\n");
7513 + tinfo->mc2_param.crit = mp->crit;
7514 + tinfo->mc2_param.res_id = mp->res_id;
7515 +
7516 + /* initial values */
7517 + err = mc2_task_client_init(&tinfo->res_info, &tinfo->mc2_param, tsk, res);
7518 + tinfo->cpu = task_cpu(tsk);
7519 + tinfo->has_departed = true;
7520 + tsk_rt(tsk)->plugin_state = tinfo;
7521 +
7522 + /* disable LITMUS^RT's per-thread budget enforcement */
7523 + tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT;
7524 + }
7525 + else {
7526 + err = -ESRCH;
7527 + }
7528 +
7529 + raw_spin_unlock_irqrestore(&state->lock, flags);
7530 + } else if (lv == CRIT_LEVEL_C) {
7531 + state = local_cpu_state();
7532 + raw_spin_lock_irqsave(&state->lock, flags);
7533 + raw_spin_lock(&_global_env.lock);
7534 +
7535 + res = gmp_find_by_id(&_global_env, mp->res_id);
7536 +
7537 + /* found the appropriate reservation (or vCPU) */
7538 + if (res) {
7539 + TRACE_TASK(tsk, "GMP FOUND RES ID\n");
7540 + tinfo->mc2_param.crit = mp->crit;
7541 + tinfo->mc2_param.res_id = mp->res_id;
7542 +
7543 + /* initial values */
7544 + err = mc2_task_client_init(&tinfo->res_info, &tinfo->mc2_param, tsk, res);
7545 + tinfo->cpu = -1;
7546 + tinfo->has_departed = true;
7547 + tsk_rt(tsk)->plugin_state = tinfo;
7548 +
7549 + /* disable LITMUS^RT's per-thread budget enforcement */
7550 + tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT;
7551 + }
7552 + else {
7553 + err = -ESRCH;
7554 + }
7555 +
7556 + raw_spin_unlock(&_global_env.lock);
7557 + raw_spin_unlock_irqrestore(&state->lock, flags);
7558 + }
7559 +
7560 + if (err)
7561 + kfree(tinfo);
7562 +
7563 + return err;
7564 +}
7565 +
7566 +/* mc2_task_new - A new real-time job is arrived. Release the next job
7567 + * at the next reservation replenish time
7568 + */
7569 +static void mc2_task_new(struct task_struct *tsk, int on_runqueue,
7570 + int is_running)
7571 +{
7572 + unsigned long flags;
7573 + struct mc2_task_state* tinfo = get_mc2_state(tsk);
7574 + struct mc2_cpu_state *state; // = cpu_state_for(tinfo->cpu);
7575 + struct reservation *res;
7576 + enum crit_level lv = get_task_crit_level(tsk);
7577 + lt_t release = 0;
7578 +
7579 + BUG_ON(lv < CRIT_LEVEL_A || lv > CRIT_LEVEL_C);
7580 +
7581 + TRACE_TASK(tsk, "new RT task %llu (on_rq:%d, running:%d)\n",
7582 + litmus_clock(), on_runqueue, is_running);
7583 +
7584 + if (tinfo->cpu == -1)
7585 + state = local_cpu_state();
7586 + else
7587 + state = cpu_state_for(tinfo->cpu);
7588 +
7589 +
7590 + if (is_running) {
7591 + state->scheduled = tsk;
7592 + /* make sure this task should actually be running */
7593 + litmus_reschedule_local();
7594 + }
7595 +
7596 + /* acquire the lock protecting the state and disable interrupts */
7597 + local_irq_save(flags);
7598 + raw_spin_lock(&state->lock);
7599 +
7600 + if (lv == CRIT_LEVEL_C) {
7601 + raw_spin_lock(&_global_env.lock);
7602 + res = gmp_find_by_id(&_global_env, tinfo->mc2_param.res_id);
7603 + }
7604 + else {
7605 + res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id);
7606 + }
7607 +
7608 + if (on_runqueue || is_running) {
7609 + /* Assumption: litmus_clock() is synchronized across cores
7610 + * [see comment in pres_task_resume()] */
7611 + if (lv == CRIT_LEVEL_C) {
7612 + gmp_update_time(&_global_env, litmus_clock());
7613 + }
7614 + else
7615 + sup_update_time(&state->sup_env, litmus_clock());
7616 +
7617 + task_arrives(state, tsk);
7618 + if (lv == CRIT_LEVEL_C)
7619 + raw_spin_unlock(&_global_env.lock);
7620 + /* NOTE: drops state->lock */
7621 + raw_spin_unlock(&state->lock);
7622 + local_irq_restore(flags);
7623 +
7624 + TRACE("mc2_new()\n");
7625 +
7626 + raw_spin_lock(&state->lock);
7627 + mc2_update_timer_and_unlock(state);
7628 + } else {
7629 + if (lv == CRIT_LEVEL_C)
7630 + raw_spin_unlock(&_global_env.lock);
7631 + raw_spin_unlock(&state->lock);
7632 + local_irq_restore(flags);
7633 + }
7634 + release = res->next_replenishment;
7635 +
7636 + if (!release) {
7637 + TRACE_TASK(tsk, "mc2_task_new() next_release = %llu\n", release);
7638 + BUG();
7639 + }
7640 + else
7641 + TRACE_TASK(tsk, "mc2_task_new() next_release = NULL\n");
7642 +}
7643 +
7644 +/* mc2_reservation_destroy - reservation_destroy system call backend
7645 + */
7646 +static long mc2_reservation_destroy(unsigned int reservation_id, int cpu)
7647 +{
7648 + long ret = -EINVAL;
7649 + struct mc2_cpu_state *state;
7650 + struct reservation *res = NULL, *next;
7651 + struct sup_reservation_environment *sup_env;
7652 + int found = 0;
7653 + unsigned long flags;
7654 +
7655 + if (cpu == -1) {
7656 + struct next_timer_event *event, *e_next;
7657 +
7658 + /* if the reservation is global reservation */
7659 +
7660 + local_irq_save(flags);
7661 + raw_spin_lock(&_global_env.lock);
7662 +
7663 + list_for_each_entry_safe(res, next, &_global_env.depleted_reservations, list) {
7664 + if (res->id == reservation_id) {
7665 + list_del(&res->list);
7666 + kfree(res);
7667 + found = 1;
7668 + ret = 0;
7669 + }
7670 + }
7671 + if (!found) {
7672 + list_for_each_entry_safe(res, next, &_global_env.inactive_reservations, list) {
7673 + if (res->id == reservation_id) {
7674 + list_del(&res->list);
7675 + kfree(res);
7676 + found = 1;
7677 + ret = 0;
7678 + }
7679 + }
7680 + }
7681 + if (!found) {
7682 + list_for_each_entry_safe(res, next, &_global_env.active_reservations, list) {
7683 + if (res->id == reservation_id) {
7684 + list_del(&res->list);
7685 + kfree(res);
7686 + found = 1;
7687 + ret = 0;
7688 + }
7689 + }
7690 + }
7691 + /* delete corresponding events */
7692 + list_for_each_entry_safe(event, e_next, &_global_env.next_events, list) {
7693 + if (event->id == reservation_id) {
7694 + list_del(&event->list);
7695 + kfree(event);
7696 + }
7697 + }
7698 +
7699 + raw_spin_unlock(&_global_env.lock);
7700 + local_irq_restore(flags);
7701 + } else {
7702 + /* if the reservation is partitioned reservation */
7703 + state = cpu_state_for(cpu);
7704 + local_irq_save(flags);
7705 + raw_spin_lock(&state->lock);
7706 +
7707 + sup_env = &state->sup_env;
7708 + list_for_each_entry_safe(res, next, &sup_env->depleted_reservations, list) {
7709 + if (res->id == reservation_id) {
7710 + list_del(&res->list);
7711 + kfree(res);
7712 + found = 1;
7713 + ret = 0;
7714 + }
7715 + }
7716 + if (!found) {
7717 + list_for_each_entry_safe(res, next, &sup_env->inactive_reservations, list) {
7718 + if (res->id == reservation_id) {
7719 + list_del(&res->list);
7720 + kfree(res);
7721 + found = 1;
7722 + ret = 0;
7723 + }
7724 + }
7725 + }
7726 + if (!found) {
7727 + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) {
7728 + if (res->id == reservation_id) {
7729 + list_del(&res->list);
7730 + kfree(res);
7731 + found = 1;
7732 + ret = 0;
7733 + }
7734 + }
7735 + }
7736 +
7737 + raw_spin_unlock(&state->lock);
7738 + local_irq_restore(flags);
7739 + }
7740 +
7741 + TRACE("Rerservation destroyed ret = %d\n", ret);
7742 + return ret;
7743 +}
7744 +
7745 +/* mc2_task_exit - Task became a normal task (not real-time task)
7746 + */
7747 +static void mc2_task_exit(struct task_struct *tsk)
7748 +{
7749 + unsigned long flags;
7750 + struct mc2_task_state* tinfo = get_mc2_state(tsk);
7751 + struct mc2_cpu_state *state;
7752 + enum crit_level lv = tinfo->mc2_param.crit;
7753 + struct crit_entry* ce;
7754 + int cpu;
7755 +
7756 + local_irq_save(flags);
7757 + if (tinfo->cpu != -1)
7758 + state = cpu_state_for(tinfo->cpu);
7759 + else
7760 + state = local_cpu_state();
7761 +
7762 + raw_spin_lock(&state->lock);
7763 +
7764 + if (state->scheduled == tsk)
7765 + state->scheduled = NULL;
7766 +
7767 + ce = &state->crit_entries[lv];
7768 + if (ce->running == tsk)
7769 + ce->running = NULL;
7770 +
7771 + /* remove from queues */
7772 + if (is_running(tsk)) {
7773 + /* Assumption: litmus_clock() is synchronized across cores
7774 + * [see comment in pres_task_resume()] */
7775 +
7776 + /* update both global and partitioned */
7777 + if (lv < CRIT_LEVEL_C) {
7778 + sup_update_time(&state->sup_env, litmus_clock());
7779 + }
7780 + else if (lv == CRIT_LEVEL_C) {
7781 + raw_spin_lock(&_global_env.lock);
7782 + gmp_update_time(&_global_env, litmus_clock());
7783 + }
7784 + task_departs(tsk, 0);
7785 + if (lv == CRIT_LEVEL_C)
7786 + raw_spin_unlock(&_global_env.lock);
7787 +
7788 + /* NOTE: drops state->lock */
7789 + TRACE("mc2_exit()\n");
7790 +
7791 + mc2_update_timer_and_unlock(state);
7792 + } else {
7793 + raw_spin_unlock(&state->lock);
7794 + }
7795 +
7796 + if (lv == CRIT_LEVEL_C) {
7797 + for_each_online_cpu(cpu) {
7798 + state = cpu_state_for(cpu);
7799 + if (state == local_cpu_state())
7800 + continue;
7801 + raw_spin_lock(&state->lock);
7802 +
7803 + if (state->scheduled == tsk)
7804 + state->scheduled = NULL;
7805 +
7806 + ce = &state->crit_entries[lv];
7807 + if (ce->running == tsk)
7808 + ce->running = NULL;
7809 +
7810 + raw_spin_unlock(&state->lock);
7811 + }
7812 + }
7813 +
7814 + local_irq_restore(flags);
7815 +
7816 + kfree(tsk_rt(tsk)->plugin_state);
7817 + tsk_rt(tsk)->plugin_state = NULL;
7818 + kfree(tsk_rt(tsk)->mc2_data);
7819 + tsk_rt(tsk)->mc2_data = NULL;
7820 +}
7821 +
7822 +/* create_polling_reservation - create a new polling reservation
7823 + */
7824 +static long create_polling_reservation(
7825 + int res_type,
7826 + struct reservation_config *config)
7827 +{
7828 + struct mc2_cpu_state *state;
7829 + struct reservation* res;
7830 + struct polling_reservation *pres;
7831 + unsigned long flags;
7832 + int use_edf = config->priority == LITMUS_NO_PRIORITY;
7833 + int periodic = res_type == PERIODIC_POLLING;
7834 + long err = -EINVAL;
7835 +
7836 + /* sanity checks */
7837 + if (config->polling_params.budget >
7838 + config->polling_params.period) {
7839 + printk(KERN_ERR "invalid polling reservation (%u): "
7840 + "budget > period\n", config->id);
7841 + return -EINVAL;
7842 + }
7843 + if (config->polling_params.budget >
7844 + config->polling_params.relative_deadline
7845 + && config->polling_params.relative_deadline) {
7846 + printk(KERN_ERR "invalid polling reservation (%u): "
7847 + "budget > deadline\n", config->id);
7848 + return -EINVAL;
7849 + }
7850 + if (config->polling_params.offset >
7851 + config->polling_params.period) {
7852 + printk(KERN_ERR "invalid polling reservation (%u): "
7853 + "offset > period\n", config->id);
7854 + return -EINVAL;
7855 + }
7856 +
7857 + /* Allocate before we grab a spin lock.
7858 + * Todo: would be nice to use a core-local allocation.
7859 + */
7860 + pres = kzalloc(sizeof(*pres), GFP_KERNEL);
7861 + if (!pres)
7862 + return -ENOMEM;
7863 +
7864 + if (config->cpu != -1) {
7865 + state = cpu_state_for(config->cpu);
7866 + raw_spin_lock_irqsave(&state->lock, flags);
7867 +
7868 + res = sup_find_by_id(&state->sup_env, config->id);
7869 + if (!res) {
7870 + polling_reservation_init(pres, use_edf, periodic,
7871 + config->polling_params.budget,
7872 + config->polling_params.period,
7873 + config->polling_params.relative_deadline,
7874 + config->polling_params.offset);
7875 + pres->res.id = config->id;
7876 + pres->res.blocked_by_ghost = 0;
7877 + pres->res.is_ghost = NO_CPU;
7878 + if (!use_edf)
7879 + pres->res.priority = config->priority;
7880 + sup_add_new_reservation(&state->sup_env, &pres->res);
7881 + err = config->id;
7882 + TRACE_CUR("reservation created R%d priority : %llu\n", config->id, pres->res.priority);
7883 + } else {
7884 + err = -EEXIST;
7885 + }
7886 +
7887 + raw_spin_unlock_irqrestore(&state->lock, flags);
7888 +
7889 + } else {
7890 + raw_spin_lock_irqsave(&_global_env.lock, flags);
7891 +
7892 + res = gmp_find_by_id(&_global_env, config->id);
7893 + if (!res) {
7894 + polling_reservation_init(pres, use_edf, periodic,
7895 + config->polling_params.budget,
7896 + config->polling_params.period,
7897 + config->polling_params.relative_deadline,
7898 + config->polling_params.offset);
7899 + pres->res.id = config->id;
7900 + pres->res.blocked_by_ghost = 0;
7901 + pres->res.scheduled_on = NO_CPU;
7902 + pres->res.is_ghost = NO_CPU;
7903 + if (!use_edf)
7904 + pres->res.priority = config->priority;
7905 + gmp_add_new_reservation(&_global_env, &pres->res);
7906 + err = config->id;
7907 + } else {
7908 + err = -EEXIST;
7909 + }
7910 + raw_spin_unlock_irqrestore(&_global_env.lock, flags);
7911 + }
7912 +
7913 + if (err < 0)
7914 + kfree(pres);
7915 +
7916 + return err;
7917 +}
7918 +
7919 +#define MAX_INTERVALS 1024
7920 +
7921 +/* create_table_driven_reservation - create a table_driven reservation
7922 + */
7923 +static long create_table_driven_reservation(
7924 + struct reservation_config *config)
7925 +{
7926 + struct mc2_cpu_state *state;
7927 + struct reservation* res;
7928 + struct table_driven_reservation *td_res = NULL;
7929 + struct lt_interval *slots = NULL;
7930 + size_t slots_size;
7931 + unsigned int i, num_slots;
7932 + unsigned long flags;
7933 + long err = -EINVAL;
7934 +
7935 +
7936 + if (!config->table_driven_params.num_intervals) {
7937 + printk(KERN_ERR "invalid table-driven reservation (%u): "
7938 + "no intervals\n", config->id);
7939 + return -EINVAL;
7940 + }
7941 +
7942 + if (config->table_driven_params.num_intervals > MAX_INTERVALS) {
7943 + printk(KERN_ERR "invalid table-driven reservation (%u): "
7944 + "too many intervals (max: %d)\n", config->id, MAX_INTERVALS);
7945 + return -EINVAL;
7946 + }
7947 +
7948 + num_slots = config->table_driven_params.num_intervals;
7949 + slots_size = sizeof(slots[0]) * num_slots;
7950 + slots = kzalloc(slots_size, GFP_KERNEL);
7951 + if (!slots)
7952 + return -ENOMEM;
7953 +
7954 + td_res = kzalloc(sizeof(*td_res), GFP_KERNEL);
7955 + if (!td_res)
7956 + err = -ENOMEM;
7957 + else
7958 + err = copy_from_user(slots,
7959 + config->table_driven_params.intervals, slots_size);
7960 +
7961 + if (!err) {
7962 + /* sanity checks */
7963 + for (i = 0; !err && i < num_slots; i++)
7964 + if (slots[i].end <= slots[i].start) {
7965 + printk(KERN_ERR
7966 + "invalid table-driven reservation (%u): "
7967 + "invalid interval %u => [%llu, %llu]\n",
7968 + config->id, i,
7969 + slots[i].start, slots[i].end);
7970 + err = -EINVAL;
7971 + }
7972 +
7973 + for (i = 0; !err && i + 1 < num_slots; i++)
7974 + if (slots[i + 1].start <= slots[i].end) {
7975 + printk(KERN_ERR
7976 + "invalid table-driven reservation (%u): "
7977 + "overlapping intervals %u, %u\n",
7978 + config->id, i, i + 1);
7979 + err = -EINVAL;
7980 + }
7981 +
7982 + if (slots[num_slots - 1].end >
7983 + config->table_driven_params.major_cycle_length) {
7984 + printk(KERN_ERR
7985 + "invalid table-driven reservation (%u): last "
7986 + "interval ends past major cycle %llu > %llu\n",
7987 + config->id,
7988 + slots[num_slots - 1].end,
7989 + config->table_driven_params.major_cycle_length);
7990 + err = -EINVAL;
7991 + }
7992 + }
7993 +
7994 + if (!err) {
7995 + state = cpu_state_for(config->cpu);
7996 + raw_spin_lock_irqsave(&state->lock, flags);
7997 +
7998 + res = sup_find_by_id(&state->sup_env, config->id);
7999 + if (!res) {
8000 + table_driven_reservation_init(td_res,
8001 + config->table_driven_params.major_cycle_length,
8002 + slots, num_slots);
8003 + td_res->res.id = config->id;
8004 + td_res->res.priority = config->priority;
8005 + td_res->res.blocked_by_ghost = 0;
8006 + sup_add_new_reservation(&state->sup_env, &td_res->res);
8007 + err = config->id;
8008 + } else {
8009 + err = -EEXIST;
8010 + }
8011 +
8012 + raw_spin_unlock_irqrestore(&state->lock, flags);
8013 + }
8014 +
8015 + if (err < 0) {
8016 + kfree(slots);
8017 + kfree(td_res);
8018 + }
8019 +
8020 + return err;
8021 +}
8022 +
8023 +/* mc2_reservation_create - reservation_create system call backend
8024 + */
8025 +static long mc2_reservation_create(int res_type, void* __user _config)
8026 +{
8027 + long ret = -EINVAL;
8028 + struct reservation_config config;
8029 +
8030 + TRACE("Attempt to create reservation (%d)\n", res_type);
8031 +
8032 + if (copy_from_user(&config, _config, sizeof(config)))
8033 + return -EFAULT;
8034 +
8035 + if (config.cpu != -1) {
8036 + if (config.cpu < 0 || !cpu_online(config.cpu)) {
8037 + printk(KERN_ERR "invalid polling reservation (%u): "
8038 + "CPU %d offline\n", config.id, config.cpu);
8039 + return -EINVAL;
8040 + }
8041 + }
8042 +
8043 + switch (res_type) {
8044 + case PERIODIC_POLLING:
8045 + case SPORADIC_POLLING:
8046 + ret = create_polling_reservation(res_type, &config);
8047 + break;
8048 +
8049 + case TABLE_DRIVEN:
8050 + ret = create_table_driven_reservation(&config);
8051 + break;
8052 +
8053 + default:
8054 + return -EINVAL;
8055 + };
8056 +
8057 + return ret;
8058 +}
8059 +
8060 +static struct domain_proc_info mc2_domain_proc_info;
8061 +
8062 +static long mc2_get_domain_proc_info(struct domain_proc_info **ret)
8063 +{
8064 + *ret = &mc2_domain_proc_info;
8065 + return 0;
8066 +}
8067 +
8068 +static void mc2_setup_domain_proc(void)
8069 +{
8070 + int i, cpu;
8071 + int num_rt_cpus = num_online_cpus();
8072 +
8073 + struct cd_mapping *cpu_map, *domain_map;
8074 +
8075 + memset(&mc2_domain_proc_info, sizeof(mc2_domain_proc_info), 0);
8076 + init_domain_proc_info(&mc2_domain_proc_info, num_rt_cpus, num_rt_cpus);
8077 + mc2_domain_proc_info.num_cpus = num_rt_cpus;
8078 + mc2_domain_proc_info.num_domains = num_rt_cpus;
8079 +
8080 + i = 0;
8081 + for_each_online_cpu(cpu) {
8082 + cpu_map = &mc2_domain_proc_info.cpu_to_domains[i];
8083 + domain_map = &mc2_domain_proc_info.domain_to_cpus[i];
8084 +
8085 + cpu_map->id = cpu;
8086 + domain_map->id = i;
8087 + cpumask_set_cpu(i, cpu_map->mask);
8088 + cpumask_set_cpu(cpu, domain_map->mask);
8089 + ++i;
8090 + }
8091 +}
8092 +
8093 +static long mc2_activate_plugin(void)
8094 +{
8095 + int cpu, lv;
8096 + struct mc2_cpu_state *state;
8097 + struct cpu_entry *ce;
8098 +
8099 + gmp_init(&_global_env);
8100 + //raw_spin_lock_init(&_lowest_prio_cpu.lock);
8101 +
8102 + for_each_online_cpu(cpu) {
8103 + TRACE("Initializing CPU%d...\n", cpu);
8104 +
8105 + resched_cpu[cpu] = 0;
8106 + level_a_priorities[cpu] = 0;
8107 + state = cpu_state_for(cpu);
8108 + ce = &_lowest_prio_cpu.cpu_entries[cpu];
8109 +
8110 + ce->cpu = cpu;
8111 + ce->scheduled = NULL;
8112 + ce->deadline = ULLONG_MAX;
8113 + ce->lv = NUM_CRIT_LEVELS;
8114 + ce->will_schedule = false;
8115 +
8116 + raw_spin_lock_init(&state->lock);
8117 + state->cpu = cpu;
8118 + state->scheduled = NULL;
8119 + for (lv = 0; lv < NUM_CRIT_LEVELS; lv++) {
8120 + struct crit_entry *cr_entry = &state->crit_entries[lv];
8121 + cr_entry->level = lv;
8122 + cr_entry->running = NULL;
8123 + }
8124 + sup_init(&state->sup_env);
8125 +
8126 + hrtimer_init(&state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
8127 + state->timer.function = on_scheduling_timer;
8128 + }
8129 +
8130 + mc2_setup_domain_proc();
8131 +
8132 + return 0;
8133 +}
8134 +
8135 +static void mc2_finish_switch(struct task_struct *prev)
8136 +{
8137 + int cpus;
8138 + enum crit_level lv = get_task_crit_level(prev);
8139 + struct mc2_cpu_state *state = local_cpu_state();
8140 +
8141 + state->scheduled = is_realtime(current) ? current : NULL;
8142 + if (lv == CRIT_LEVEL_C) {
8143 + for (cpus = 0; cpus<NR_CPUS; cpus++) {
8144 + if (resched_cpu[cpus] && state->cpu != cpus) {
8145 + resched_cpu[cpus] = 0;
8146 + litmus_reschedule(cpus);
8147 + }
8148 + }
8149 + }
8150 +}
8151 +
8152 +static long mc2_deactivate_plugin(void)
8153 +{
8154 + int cpu;
8155 + struct mc2_cpu_state *state;
8156 + struct reservation *res;
8157 + struct next_timer_event *event;
8158 + struct cpu_entry *ce;
8159 +
8160 + for_each_online_cpu(cpu) {
8161 + state = cpu_state_for(cpu);
8162 + raw_spin_lock(&state->lock);
8163 +
8164 + hrtimer_cancel(&state->timer);
8165 +
8166 + ce = &_lowest_prio_cpu.cpu_entries[cpu];
8167 +
8168 + ce->cpu = cpu;
8169 + ce->scheduled = NULL;
8170 + ce->deadline = ULLONG_MAX;
8171 + ce->lv = NUM_CRIT_LEVELS;
8172 + ce->will_schedule = false;
8173 +
8174 + /* Delete all reservations --- assumes struct reservation
8175 + * is prefix of containing struct. */
8176 +
8177 + while (!list_empty(&state->sup_env.active_reservations)) {
8178 + res = list_first_entry(
8179 + &state->sup_env.active_reservations,
8180 + struct reservation, list);
8181 + list_del(&res->list);
8182 + kfree(res);
8183 + }
8184 +
8185 + while (!list_empty(&state->sup_env.inactive_reservations)) {
8186 + res = list_first_entry(
8187 + &state->sup_env.inactive_reservations,
8188 + struct reservation, list);
8189 + list_del(&res->list);
8190 + kfree(res);
8191 + }
8192 +
8193 + while (!list_empty(&state->sup_env.depleted_reservations)) {
8194 + res = list_first_entry(
8195 + &state->sup_env.depleted_reservations,
8196 + struct reservation, list);
8197 + list_del(&res->list);
8198 + kfree(res);
8199 + }
8200 +
8201 + raw_spin_unlock(&state->lock);
8202 + }
8203 +
8204 + raw_spin_lock(&_global_env.lock);
8205 +
8206 + while (!list_empty(&_global_env.active_reservations)) {
8207 + res = list_first_entry(
8208 + &_global_env.active_reservations,
8209 + struct reservation, list);
8210 + list_del(&res->list);
8211 + kfree(res);
8212 + }
8213 +
8214 + while (!list_empty(&_global_env.inactive_reservations)) {
8215 + res = list_first_entry(
8216 + &_global_env.inactive_reservations,
8217 + struct reservation, list);
8218 + list_del(&res->list);
8219 + kfree(res);
8220 + }
8221 +
8222 + while (!list_empty(&_global_env.depleted_reservations)) {
8223 + res = list_first_entry(
8224 + &_global_env.depleted_reservations,
8225 + struct reservation, list);
8226 + list_del(&res->list);
8227 + kfree(res);
8228 + }
8229 +
8230 + while (!list_empty(&_global_env.next_events)) {
8231 + event = list_first_entry(
8232 + &_global_env.next_events,
8233 + struct next_timer_event, list);
8234 + list_del(&event->list);
8235 + kfree(event);
8236 + }
8237 +
8238 + raw_spin_unlock(&_global_env.lock);
8239 +
8240 + destroy_domain_proc_info(&mc2_domain_proc_info);
8241 + return 0;
8242 +}
8243 +
8244 +static struct sched_plugin mc2_plugin = {
8245 + .plugin_name = "MC2",
8246 + .schedule = mc2_schedule,
8247 + .finish_switch = mc2_finish_switch,
8248 + .task_wake_up = mc2_task_resume,
8249 + .admit_task = mc2_admit_task,
8250 + .task_new = mc2_task_new,
8251 + .task_exit = mc2_task_exit,
8252 + .complete_job = mc2_complete_job,
8253 + .get_domain_proc_info = mc2_get_domain_proc_info,
8254 + .activate_plugin = mc2_activate_plugin,
8255 + .deactivate_plugin = mc2_deactivate_plugin,
8256 + .reservation_create = mc2_reservation_create,
8257 + .reservation_destroy = mc2_reservation_destroy,
8258 +};
8259 +
8260 +static int __init init_mc2(void)
8261 +{
8262 + return register_sched_plugin(&mc2_plugin);
8263 +}
8264 +
8265 +module_init(init_mc2);
8266 diff --git litmus/sched_plugin.c litmus/sched_plugin.c
8267 index edd91e9..7b1eba0 100644
8268 --- litmus/sched_plugin.c
8269 +++ litmus/sched_plugin.c
8270 @@ -13,6 +13,7 @@
8271 #include <litmus/sched_plugin.h>
8272 #include <litmus/preempt.h>
8273 #include <litmus/jobs.h>
8274 +#include <litmus/budget.h>
8275
8276 /*
8277 * Generic function to trigger preemption on either local or remote cpu
8278 @@ -197,6 +198,9 @@ int register_sched_plugin(struct sched_plugin* plugin)
8279 if (!plugin->wait_for_release_at)
8280 plugin->wait_for_release_at = default_wait_for_release_at;
8281
8282 + if (!plugin->current_budget)
8283 + plugin->current_budget = litmus_current_budget;
8284 +
8285 raw_spin_lock(&sched_plugins_lock);
8286 list_add(&plugin->list, &sched_plugins);
8287 raw_spin_unlock(&sched_plugins_lock);
8288 diff --git litmus/sched_psn_edf.c litmus/sched_psn_edf.c
8289 index 2549a3f..216b9f3 100644
8290 --- litmus/sched_psn_edf.c
8291 +++ litmus/sched_psn_edf.c
8292 @@ -23,6 +23,10 @@
8293 #include <litmus/sched_trace.h>
8294 #include <litmus/trace.h>
8295
8296 +#ifdef CONFIG_PGMRT_SUPPORT
8297 +#include <litmus/pgm.h>
8298 +#endif
8299 +
8300 /* to set up domain/cpu mappings */
8301 #include <litmus/litmus_proc.h>
8302
8303 @@ -199,6 +203,62 @@ static struct task_struct* psnedf_schedule(struct task_struct * prev)
8304 */
8305 resched = preempt;
8306
8307 +#ifdef CONFIG_PGMRT_SUPPORT
8308 + if (exists) {
8309 + if (is_pgm_sending(pedf->scheduled)) {
8310 + if (!is_pgm_satisfied(pedf->scheduled)) {
8311 + if (!is_priority_boosted(pedf->scheduled)) {
8312 + TRACE_TASK(pedf->scheduled, "is sending PGM tokens and needs boosting.\n");
8313 + BUG_ON(is_pgm_satisfied(pedf->scheduled));
8314 +
8315 + /* We are either sending tokens or waiting for tokes.
8316 + If waiting: Boost priority so we'll be scheduled
8317 + immediately when needed tokens arrive.
8318 + If sending: Boost priority so no one (specifically, our
8319 + consumers) will preempt us while signalling the token
8320 + transmission.
8321 + */
8322 + tsk_rt(pedf->scheduled)->priority_boosted = 1;
8323 + tsk_rt(pedf->scheduled)->boost_start_time = litmus_clock();
8324 +
8325 + if (likely(!blocks)) {
8326 + requeue(pedf->scheduled, edf);
8327 + /* we may regain the processor */
8328 + if (preempt) {
8329 + preempt = edf_preemption_needed(edf, prev);
8330 + if (!preempt) {
8331 + TRACE_TASK(pedf->scheduled, "blocked preemption by lazy boosting.\n");
8332 + }
8333 + }
8334 + }
8335 + }
8336 + }
8337 + else { /* sending is satisfied */
8338 + tsk_rt(pedf->scheduled)->ctrl_page->pgm_sending = 0;
8339 + tsk_rt(pedf->scheduled)->ctrl_page->pgm_satisfied = 0;
8340 +
8341 + if (is_priority_boosted(pedf->scheduled)) {
8342 + TRACE_TASK(pedf->scheduled,
8343 + "is done sending PGM tokens must relinquish boosting.\n");
8344 + /* clear boosting */
8345 + tsk_rt(pedf->scheduled)->priority_boosted = 0;
8346 + if(likely(!blocks)) {
8347 + /* recheck priority */
8348 + requeue(pedf->scheduled, edf);
8349 + /* we may lose the processor */
8350 + if (!preempt) {
8351 + preempt = edf_preemption_needed(edf, prev);
8352 + if (preempt) {
8353 + TRACE_TASK(pedf->scheduled, "preempted by lazy unboosting.\n");
8354 + }
8355 + }
8356 + }
8357 + }
8358 + }
8359 + }
8360 + }
8361 +#endif
8362 +
8363 /* If a task blocks we have no choice but to reschedule.
8364 */
8365 if (blocks)
8366 @@ -243,7 +303,7 @@ static struct task_struct* psnedf_schedule(struct task_struct * prev)
8367 if (next) {
8368 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
8369 } else {
8370 - TRACE("becoming idle at %llu\n", litmus_clock());
8371 + ; //TRACE("becoming idle at %llu\n", litmus_clock());
8372 }
8373
8374 pedf->scheduled = next;
8375 @@ -644,10 +704,14 @@ static long psnedf_admit_task(struct task_struct* tsk)
8376 /* don't allow tasks on release master CPU */
8377 && task_cpu(tsk) != remote_edf(task_cpu(tsk))->release_master
8378 #endif
8379 - )
8380 + ) {
8381 + TRACE_TASK(tsk, "admitted\n");
8382 return 0;
8383 - else
8384 + }
8385 + else {
8386 + TRACE_TASK(tsk, "not admitted\n");
8387 return -EINVAL;
8388 + }
8389 }
8390
8391 /* Plugin object */
8392 diff --git litmus/uncachedev.c litmus/uncachedev.c
8393 index 06a6a7c..8687581 100644
8394 --- litmus/uncachedev.c
8395 +++ litmus/uncachedev.c
8396 @@ -28,8 +28,8 @@ int litmus_uncache_vm_fault(struct vm_area_struct* vma,
8397 /* modeled after SG DMA video4linux, but without DMA. */
8398 /* (see drivers/media/video/videobuf-dma-sg.c) */
8399 struct page *page;
8400 -
8401 - page = alloc_page(GFP_USER);
8402 +
8403 + page = alloc_page(GFP_USER|GFP_COLOR);
8404 if (!page)
8405 return VM_FAULT_OOM;
8406
8407 diff --git mm/dmapool.c mm/dmapool.c
8408 index fd5fe43..b69dc13 100644
8409 --- mm/dmapool.c
8410 +++ mm/dmapool.c
8411 @@ -333,7 +333,7 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
8412
8413 /* pool_alloc_page() might sleep, so temporarily drop &pool->lock */
8414 spin_unlock_irqrestore(&pool->lock, flags);
8415 -
8416 + printk(KERN_INFO "dma_pool_alloc(): called with %x flags\n", mem_flags);
8417 page = pool_alloc_page(pool, mem_flags);
8418 if (!page)
8419 return NULL;
8420 diff --git mm/migrate.c mm/migrate.c
8421 index f53838f..29b69cd 100644
8422 --- mm/migrate.c
8423 +++ mm/migrate.c
8424 @@ -38,6 +38,8 @@
8425 #include <linux/balloon_compaction.h>
8426 #include <linux/mmu_notifier.h>
8427
8428 +#include <litmus/litmus.h> // for TRACE_TASK
8429 +
8430 #include <asm/tlbflush.h>
8431
8432 #define CREATE_TRACE_POINTS
8433 @@ -391,6 +393,65 @@ int migrate_page_move_mapping(struct address_space *mapping,
8434 }
8435
8436 /*
8437 + * Replace the page in the mapping.
8438 + *
8439 + * The number of remaining references must be:
8440 + * 1 for anonymous pages without a mapping
8441 + * 2 for pages with a mapping
8442 + * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
8443 + */
8444 +int replicate_page_move_mapping(struct address_space *mapping,
8445 + struct page *newpage, struct page *page,
8446 + struct buffer_head *head, enum migrate_mode mode,
8447 + int extra_count)
8448 +{
8449 + int prev_count = page_count(page);
8450 + void **pslot;
8451 +
8452 + BUG_ON(!mapping);
8453 +
8454 + spin_lock_irq(&mapping->tree_lock);
8455 +
8456 + pslot = radix_tree_lookup_slot(&mapping->page_tree, page_index(page));
8457 +
8458 + /*
8459 + * Now we know that no one else is looking at the page.
8460 + */
8461 + get_page(newpage); /* add cache reference */
8462 + if (PageSwapCache(page)) {
8463 + SetPageSwapCache(newpage);
8464 + set_page_private(newpage, page_private(page));
8465 + }
8466 +
8467 + /*
8468 + * Drop cache reference from old page by unfreezing
8469 + * to the previous reference.
8470 + * We know this isn't the last reference.
8471 + */
8472 + page_unfreeze_refs(page, prev_count);
8473 +
8474 + /*
8475 + * If moved to a different zone then also account
8476 + * the page for that zone. Other VM counters will be
8477 + * taken care of when we establish references to the
8478 + * new page and drop references to the old page.
8479 + *
8480 + * Note that anonymous pages are accounted for
8481 + * via NR_FILE_PAGES and NR_ANON_PAGES if they
8482 + * are mapped to swap space.
8483 + */
8484 + __dec_zone_page_state(page, NR_FILE_PAGES);
8485 + __inc_zone_page_state(newpage, NR_FILE_PAGES);
8486 + if (!PageSwapCache(page) && PageSwapBacked(page)) {
8487 + __dec_zone_page_state(page, NR_SHMEM);
8488 + __inc_zone_page_state(newpage, NR_SHMEM);
8489 + }
8490 + spin_unlock_irq(&mapping->tree_lock);
8491 +
8492 + return MIGRATEPAGE_SUCCESS;
8493 +}
8494 +
8495 +/*
8496 * The expected number of remaining references is the same as that
8497 * of migrate_page_move_mapping().
8498 */
8499 @@ -550,6 +611,73 @@ void migrate_page_copy(struct page *newpage, struct page *page)
8500 end_page_writeback(newpage);
8501 }
8502
8503 +/*
8504 + * Copy the page to its new location
8505 + */
8506 +void replicate_page_copy(struct page *newpage, struct page *page)
8507 +{
8508 + if (PageHuge(page) || PageTransHuge(page))
8509 + copy_huge_page(newpage, page);
8510 + else
8511 + copy_highpage(newpage, page);
8512 +
8513 + if (PageError(page))
8514 + SetPageError(newpage);
8515 + if (PageReferenced(page))
8516 + SetPageReferenced(newpage);
8517 + if (PageUptodate(page))
8518 + SetPageUptodate(newpage);
8519 + if (PageActive(page)) {
8520 + VM_BUG_ON_PAGE(PageUnevictable(page), page);
8521 + SetPageActive(newpage);
8522 + } else if (PageUnevictable(page))
8523 + SetPageUnevictable(newpage);
8524 + if (PageChecked(page))
8525 + SetPageChecked(newpage);
8526 + if (PageMappedToDisk(page))
8527 + SetPageMappedToDisk(newpage);
8528 +
8529 + if (PageDirty(page)) {
8530 + clear_page_dirty_for_io(page);
8531 + /*
8532 + * Want to mark the page and the radix tree as dirty, and
8533 + * redo the accounting that clear_page_dirty_for_io undid,
8534 + * but we can't use set_page_dirty because that function
8535 + * is actually a signal that all of the page has become dirty.
8536 + * Whereas only part of our page may be dirty.
8537 + */
8538 + if (PageSwapBacked(page))
8539 + SetPageDirty(newpage);
8540 + else
8541 + __set_page_dirty_nobuffers(newpage);
8542 + }
8543 +
8544 + /*
8545 + * Copy NUMA information to the new page, to prevent over-eager
8546 + * future migrations of this same page.
8547 + */
8548 +#ifdef CONFIG_NUMA_BALANCING
8549 + BUG();
8550 +#endif
8551 +
8552 + if (PageMlocked(page)) {
8553 + unsigned long flags;
8554 + int nr_pages = hpage_nr_pages(page);
8555 +
8556 + local_irq_save(flags);
8557 + SetPageMlocked(newpage);
8558 + __mod_zone_page_state(page_zone(newpage), NR_MLOCK, nr_pages);
8559 + local_irq_restore(flags);
8560 + }
8561 +
8562 + /*
8563 + * If any waiters have accumulated on the new page then
8564 + * wake them up.
8565 + */
8566 + if (PageWriteback(newpage))
8567 + end_page_writeback(newpage);
8568 +}
8569 +
8570 /************************************************************
8571 * Migration functions
8572 ***********************************************************/
8573 @@ -578,6 +706,23 @@ int migrate_page(struct address_space *mapping,
8574 }
8575 EXPORT_SYMBOL(migrate_page);
8576
8577 +int replicate_page(struct address_space *mapping,
8578 + struct page *newpage, struct page *page,
8579 + enum migrate_mode mode, int has_replica)
8580 +{
8581 + int rc, extra_count = 0;
8582 +
8583 + BUG_ON(PageWriteback(page)); /* Writeback must be complete */
8584 +
8585 + rc = replicate_page_move_mapping(mapping, newpage, page, NULL, mode, extra_count);
8586 + if (rc != MIGRATEPAGE_SUCCESS)
8587 + return rc;
8588 +
8589 + if (has_replica == 0)
8590 + replicate_page_copy(newpage, page);
8591 + return MIGRATEPAGE_SUCCESS;
8592 +}
8593 +
8594 #ifdef CONFIG_BLOCK
8595 /*
8596 * Migration function for pages with buffers. This function can only be used
8597 @@ -638,6 +783,8 @@ int buffer_migrate_page(struct address_space *mapping,
8598 EXPORT_SYMBOL(buffer_migrate_page);
8599 #endif
8600
8601 +extern struct list_head shared_lib_pages;
8602 +
8603 /*
8604 * Writeback a page to clean the dirty state
8605 */
8606 @@ -763,6 +910,64 @@ static int move_to_new_page(struct page *newpage, struct page *page,
8607 return rc;
8608 }
8609
8610 +/*
8611 + * Copy a page to a newly allocated page
8612 + * The page is locked and all ptes have been successfully removed.
8613 + *
8614 + * The new page will have replaced the old page if this function
8615 + * is successful.
8616 + *
8617 + * Return value:
8618 + * < 0 - error code
8619 + * MIGRATEPAGE_SUCCESS - success
8620 + */
8621 +static int copy_to_new_page(struct page *newpage, struct page *page,
8622 + int page_was_mapped, enum migrate_mode mode,
8623 + int has_replica)
8624 +{
8625 + struct address_space *mapping;
8626 + int rc;
8627 +
8628 + /*
8629 + * Block others from accessing the page when we get around to
8630 + * establishing additional references. We are the only one
8631 + * holding a reference to the new page at this point.
8632 + */
8633 + if (!trylock_page(newpage))
8634 + BUG();
8635 +
8636 + /* Prepare mapping for the new page.*/
8637 + newpage->index = page->index;
8638 + newpage->mapping = page->mapping;
8639 + if (PageSwapBacked(page))
8640 + SetPageSwapBacked(newpage);
8641 +
8642 + mapping = page_mapping(page);
8643 + if (!mapping) {
8644 + /* a shared library page must have a mapping. */
8645 + BUG();
8646 + }
8647 + else if (mapping->a_ops->migratepage) {
8648 + rc = replicate_page(mapping, newpage, page, mode, has_replica);
8649 + }
8650 + else {
8651 + rc = fallback_migrate_page(mapping, newpage, page, mode);
8652 + }
8653 +
8654 + if (rc != MIGRATEPAGE_SUCCESS) {
8655 + newpage->mapping = NULL;
8656 + } else {
8657 + if (page_was_mapped) {
8658 + remove_migration_ptes(page, newpage);
8659 + }
8660 + }
8661 +
8662 + unlock_page(newpage);
8663 +
8664 + return rc;
8665 +}
8666 +
8667 +
8668 static int __unmap_and_move(struct page *page, struct page *newpage,
8669 int force, enum migrate_mode mode)
8670 {
8671 @@ -901,6 +1106,106 @@ out:
8672 return rc;
8673 }
8674
8675 +static int __unmap_and_copy(struct page *page, struct page *newpage,
8676 + int force, enum migrate_mode mode, int has_replica)
8677 +{
8678 + int rc = -EAGAIN;
8679 + int ttu_ret = SWAP_AGAIN;
8680 + int page_was_mapped = 0;
8681 + struct anon_vma *anon_vma = NULL;
8682 +
8683 + if (!trylock_page(page)) {
8684 + if (!force || mode == MIGRATE_ASYNC)
8685 + goto out;
8686 +
8687 + /*
8688 + * It's not safe for direct compaction to call lock_page.
8689 + * For example, during page readahead pages are added locked
8690 + * to the LRU. Later, when the IO completes the pages are
8691 + * marked uptodate and unlocked. However, the queueing
8692 + * could be merging multiple pages for one bio (e.g.
8693 + * mpage_readpages). If an allocation happens for the
8694 + * second or third page, the process can end up locking
8695 + * the same page twice and deadlocking. Rather than
8696 + * trying to be clever about what pages can be locked,
8697 + * avoid the use of lock_page for direct compaction
8698 + * altogether.
8699 + */
8700 + if (current->flags & PF_MEMALLOC)
8701 + goto out;
8702 +
8703 + lock_page(page);
8704 + }
8705 +
8706 + if (PageWriteback(page)) {
8707 + /*
8708 + * The code of shared library cannot be written.
8709 + */
8710 + BUG();
8711 + }
8712 +
8713 + if (PageAnon(page) && !PageKsm(page)) {
8714 + /* The shared library pages must be backed by a file. */
8715 + BUG();
8716 + }
8717 +
8718 + if (unlikely(isolated_balloon_page(page))) {
8719 + BUG();
8720 + }
8721 +
8722 + /*
8723 + * Corner case handling:
8724 + * 1. When a new swap-cache page is read into, it is added to the LRU
8725 + * and treated as swapcache but it has no rmap yet.
8726 + * Calling try_to_unmap() against a page->mapping==NULL page will
8727 + * trigger a BUG. So handle it here.
8728 + * 2. An orphaned page (see truncate_complete_page) might have
8729 + * fs-private metadata. The page can be picked up due to memory
8730 + * offlining. Everywhere else except page reclaim, the page is
8731 + * invisible to the vm, so the page can not be migrated. So try to
8732 + * free the metadata, so the page can be freed.
8733 + */
8734 + if (!page->mapping) {
8735 + VM_BUG_ON_PAGE(PageAnon(page), page);
8736 + if (page_has_private(page)) {
8737 + try_to_free_buffers(page);
8738 + goto out_unlock;
8739 + }
8740 + goto skip_unmap;
8741 + }
8742 +
8743 + /* Establish migration ptes or remove ptes */
8744 + if (page_mapped(page)) {
8745 + struct rmap_walk_control rwc = {
8746 + .rmap_one = try_to_unmap_one_only,
8747 + .arg = (void *)(TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS),
8748 + };
8749 + ttu_ret = rmap_walk(page, &rwc);
8750 +
8751 + page_was_mapped = 1;
8752 + }
8753 +
8754 +skip_unmap:
8755 + if (ttu_ret == SWAP_SUCCESS) {
8756 + rc = copy_to_new_page(newpage, page, page_was_mapped, mode, has_replica);
8757 + } else if (ttu_ret == SWAP_AGAIN)
8758 + printk(KERN_ERR "rmap_walk returned SWAP_AGAIN\n");
8759 + else
8760 + printk(KERN_ERR "rmap_walk failed\n");
8761 +
8762 + if (rc && page_was_mapped)
8763 + remove_migration_ptes(page, page);
8764 +
8765 + /* Drop an anon_vma reference if we took one */
8766 + if (anon_vma)
8767 + put_anon_vma(anon_vma);
8768 +
8769 +out_unlock:
8770 + unlock_page(page);
8771 +out:
8772 + return rc;
8773 +}
8774 +
8775 /*
8776 * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move(). Work
8777 * around it.
8778 @@ -976,6 +1281,97 @@ out:
8779 }
8780
8781 /*
8782 + * Obtain the lock on page, remove all ptes.
8783 + * 1) If r_pfn == INVALID_PFN, then copy the page to the newly allocated page in newpage.
8784 + * 2) If r_pfn != INVALID_PFN, then unmap and modify ptes.
8785 + */
8786 +#include <litmus/replicate_lib.h>
8787 +
8788 +static ICE_noinline int unmap_and_copy(new_page_t get_new_page,
8789 + free_page_t put_new_page,
8790 + unsigned long private, struct page *page,
8791 + int force, enum migrate_mode mode)
8792 +{
8793 + int rc = 0;
8794 + int *result = NULL;
8795 + struct page *newpage;
8796 + struct shared_lib_page *lib_page;
8797 + int master_exist_in_psl = 0, has_replica = 0, cpu = private/2;
8798 +
8799 + /* check if this page is in the PSL list */
8800 + rcu_read_lock();
8801 + list_for_each_entry(lib_page, &shared_lib_pages, list)
8802 + {
8803 + if (page_to_pfn(page) == lib_page->master_pfn) {
8804 + master_exist_in_psl = 1;
8805 + break;
8806 + }
8807 + }
8808 + rcu_read_unlock();
8809 +
8810 + if (lib_page->r_page[cpu] == NULL) {
8811 + newpage = get_new_page(page, private, &result);
8812 + if (!newpage)
8813 + return -ENOMEM;
8814 + } else {
8815 + newpage = lib_page->r_page[cpu];
8816 + has_replica = 1;
8817 + }
8818 +
8819 + if (page_count(page) == 1) {
8820 + /* page was freed from under us. So we are done. */
8821 + goto out;
8822 + }
8823 +
8824 + if (unlikely(PageTransHuge(page)))
8825 + if (unlikely(split_huge_page(page)))
8826 + goto out;
8827 +
8828 + rc = __unmap_and_copy(page, newpage, force, mode, has_replica);
8829 +
8830 + if (has_replica == 0 && rc == MIGRATEPAGE_SUCCESS) {
8831 + lib_page->r_page[cpu] = newpage;
8832 + lib_page->r_pfn[cpu] = page_to_pfn(newpage);
8833 + }
8834 +
8835 +out:
8836 + if (rc != -EAGAIN) {
8837 + /*
8838 + * A page that has been migrated has all references
8839 + * removed and will be freed. A page that has not been
8840 + * migrated will have kepts its references and be
8841 + * restored.
8842 + */
8843 + list_del(&page->lru);
8844 + dec_zone_page_state(page, NR_ISOLATED_ANON +
8845 + page_is_file_cache(page));
8846 + putback_lru_page(page);
8847 + }
8848 +
8849 + /*
8850 + * If migration was not successful and there's a freeing callback, use
8851 + * it. Otherwise, putback_lru_page() will drop the reference grabbed
8852 + * during isolation.
8853 + */
8854 + if (rc != MIGRATEPAGE_SUCCESS && put_new_page) {
8855 + ClearPageSwapBacked(newpage);
8856 + put_new_page(newpage, private);
8857 + } else if (unlikely(__is_movable_balloon_page(newpage))) {
8858 + /* drop our reference, page already in the balloon */
8859 + put_page(newpage);
8860 + } else
8861 + putback_lru_page(newpage);
8862 +
8863 + if (result) {
8864 + if (rc)
8865 + *result = rc;
8866 + else
8867 + *result = page_to_nid(newpage);
8868 + }
8869 + return rc;
8870 +}
8871 +
8872 +/*
8873 * Counterpart of unmap_and_move_page() for hugepage migration.
8874 *
8875 * This function doesn't wait the completion of hugepage I/O
8876 @@ -1159,6 +1555,85 @@ out:
8877 return rc;
8878 }
8879
8880 +/*
8881 + * replicate_pages - replicate the pages specified in a list
8882 + *
8883 + * @from: The list of pages to be migrated.
8884 + * @get_new_page: The function used to allocate free pages to be used
8885 + * if there is no replicated page.
8886 + * @put_new_page: The function used to free target pages if migration
8887 + * fails, or NULL if no special handling is necessary.
8888 + * @private: Private data to be passed on to get_new_page()
8889 + * @mode: The migration mode that specifies the constraints for
8890 + * page migration, if any.
8891 + * @reason: The reason for page migration.
8892 + *
8893 + * The function returns after 10 attempts or if no pages are movable any more
8894 + * because the list has become empty or no retryable pages exist any more.
8895 + * The caller should call putback_lru_pages() to return pages to the LRU
8896 + * or free list only if ret != 0.
8897 + *
8898 + * Returns the number of pages that were not migrated, or an error code.
8899 + */
8900 +int replicate_pages(struct list_head *from, new_page_t get_new_page,
8901 + free_page_t put_new_page, unsigned long private,
8902 + enum migrate_mode mode, int reason)
8903 +{
8904 + int retry = 1;
8905 + int nr_failed = 0;
8906 + int nr_succeeded = 0;
8907 + int pass = 0;
8908 + struct page *page;
8909 + struct page *page2;
8910 + int swapwrite = current->flags & PF_SWAPWRITE;
8911 + int rc;
8912 +
8913 + if (!swapwrite)
8914 + current->flags |= PF_SWAPWRITE;
8915 +
8916 + for(pass = 0; pass < 10 && retry; pass++) {
8917 + retry = 0;
8918 +
8919 + list_for_each_entry_safe(page, page2, from, lru) {
8920 + cond_resched();
8921 +
8922 + rc = unmap_and_copy(get_new_page, put_new_page, private, page, pass > 2, mode);
8923 +
8924 + switch(rc) {
8925 + case -ENOMEM:
8926 + goto out;
8927 + case -EAGAIN:
8928 + retry++;
8929 + break;
8930 + case MIGRATEPAGE_SUCCESS:
8931 + nr_succeeded++;
8932 + break;
8933 + default:
8934 + /*
8935 + * Permanent failure (-EBUSY, -ENOSYS, etc.):
8936 + * unlike -EAGAIN case, the failed page is
8937 + * removed from migration page list and not
8938 + * retried in the next outer loop.
8939 + */
8940 + nr_failed++;
8941 + break;
8942 + }
8943 + }
8944 + }
8945 + rc = nr_failed + retry;
8946 +out:
8947 + if (nr_succeeded)
8948 + count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
8949 + if (nr_failed)
8950 + count_vm_events(PGMIGRATE_FAIL, nr_failed);
8951 + trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
8952 +
8953 + if (!swapwrite)
8954 + current->flags &= ~PF_SWAPWRITE;
8955 +
8956 + return rc;
8957 +}
8958 +
8959 #ifdef CONFIG_NUMA
8960 /*
8961 * Move a list of individual pages
8962 diff --git mm/page_alloc.c mm/page_alloc.c
8963 index 950c002..70b3a08 100644
8964 --- mm/page_alloc.c
8965 +++ mm/page_alloc.c
8966 @@ -65,11 +65,32 @@
8967 #include <asm/sections.h>
8968
8969 #include <litmus/litmus.h> /* for is_realtime() */
8970 +#include <litmus/page_dev.h> /* for coloring pages */
8971
8972 #include <asm/tlbflush.h>
8973 #include <asm/div64.h>
8974 #include "internal.h"
8975
8976 +// This Address Decoding is used in imx6-sabredsd platform
8977 +#define BANK_MASK 0x38000000
8978 +#define BANK_SHIFT 27
8979 +
8980 +#define CACHE_MASK 0x0000f000
8981 +#define CACHE_SHIFT 12
8982 +#define MAX_COLOR_NODE 128
8983 +
8984 +/* Decoding page color, 0~15 */
8985 +static inline unsigned int page_color(struct page *page)
8986 +{
8987 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
8988 +}
8989 +
8990 +/* Decoding page bank number, 0~7 */
8991 +static inline unsigned int page_bank(struct page *page)
8992 +{
8993 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
8994 +}
8995 +
8996 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
8997 static DEFINE_MUTEX(pcp_batch_high_lock);
8998 #define MIN_PERCPU_PAGELIST_FRACTION (8)
8999 @@ -464,7 +485,7 @@ static inline void set_page_guard(struct zone *zone, struct page *page,
9000 INIT_LIST_HEAD(&page->lru);
9001 set_page_private(page, order);
9002 /* Guard pages are not available for any usage */
9003 - __mod_zone_freepage_state(zone, -(1 << order), migratetype);
9004 + __mod_zone_freepage_state(zone, -(1 << order), migratetype, bank_to_partition(page_bank(page)));
9005 }
9006
9007 static inline void clear_page_guard(struct zone *zone, struct page *page,
9008 @@ -480,7 +501,7 @@ static inline void clear_page_guard(struct zone *zone, struct page *page,
9009
9010 set_page_private(page, 0);
9011 if (!is_migrate_isolate(migratetype))
9012 - __mod_zone_freepage_state(zone, (1 << order), migratetype);
9013 + __mod_zone_freepage_state(zone, (1 << order), migratetype, bank_to_partition(page_bank(page)));
9014 }
9015 #else
9016 struct page_ext_operations debug_guardpage_ops = { NULL, };
9017 @@ -582,76 +603,154 @@ static inline void __free_one_page(struct page *page,
9018 unsigned long combined_idx;
9019 unsigned long uninitialized_var(buddy_idx);
9020 struct page *buddy;
9021 - int max_order = MAX_ORDER;
9022 + int max_order, parti_no;
9023 +
9024 + parti_no = bank_to_partition(page_bank(page));
9025 + BUG_ON(parti_no < 0 || parti_no > NR_CPUS);
9026 +
9027 + //if (parti_no < NR_CPUS)
9028 + //printk(KERN_ALERT "pfn = %lx, part_no = %d order = %d\n", pfn, parti_no, order);
9029 +
9030 + if (parti_no < NR_CPUS) {
9031 + max_order = MAX_PARTITIONED_ORDER;
9032 +
9033 + VM_BUG_ON(!zone_is_initialized(zone));
9034 + VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
9035 +
9036 + VM_BUG_ON(migratetype == -1);
9037 + if (is_migrate_isolate(migratetype)) {
9038 + max_order = min(MAX_PARTITIONED_ORDER, pageblock_order + 1);
9039 + } else {
9040 + __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no);
9041 + }
9042
9043 - VM_BUG_ON(!zone_is_initialized(zone));
9044 - VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
9045 + page_idx = pfn & ((1 << max_order) - 1);
9046
9047 - VM_BUG_ON(migratetype == -1);
9048 - if (is_migrate_isolate(migratetype)) {
9049 - /*
9050 - * We restrict max order of merging to prevent merge
9051 - * between freepages on isolate pageblock and normal
9052 - * pageblock. Without this, pageblock isolation
9053 - * could cause incorrect freepage accounting.
9054 - */
9055 - max_order = min(MAX_ORDER, pageblock_order + 1);
9056 - } else {
9057 - __mod_zone_freepage_state(zone, 1 << order, migratetype);
9058 - }
9059 + VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
9060 + VM_BUG_ON_PAGE(bad_range(zone, page), page);
9061
9062 - page_idx = pfn & ((1 << max_order) - 1);
9063 + while (order < max_order - 1) {
9064 + buddy_idx = __find_buddy_index(page_idx, order);
9065 + buddy = page + (buddy_idx - page_idx);
9066 + if (!page_is_buddy(page, buddy, order))
9067 + break;
9068
9069 - VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
9070 - VM_BUG_ON_PAGE(bad_range(zone, page), page);
9071 + if (page_is_guard(buddy)) {
9072 + clear_page_guard(zone, buddy, order, migratetype);
9073 + } else {
9074 + list_del(&buddy->lru);
9075 + zone->free_area_d[parti_no][order].nr_free--;
9076 + rmv_page_order(buddy);
9077 + }
9078 + combined_idx = buddy_idx & page_idx;
9079 + page = page + (combined_idx - page_idx);
9080 + page_idx = combined_idx;
9081 + order++;
9082 + }
9083 + set_page_order(page, order);
9084 +
9085 + if ((order < MAX_PARTITIONED_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
9086 + struct page *higher_page, *higher_buddy;
9087 + combined_idx = buddy_idx & page_idx;
9088 + higher_page = page + (combined_idx - page_idx);
9089 + buddy_idx = __find_buddy_index(combined_idx, order + 1);
9090 + higher_buddy = higher_page + (buddy_idx - combined_idx);
9091 + if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
9092 + list_add_tail(&page->lru,
9093 + &zone->free_area_d[parti_no][order].free_list[migratetype]);
9094 + zone->free_area_d[parti_no][order].nr_free++;
9095 + return;
9096 + }
9097 + }
9098
9099 - while (order < max_order - 1) {
9100 - buddy_idx = __find_buddy_index(page_idx, order);
9101 - buddy = page + (buddy_idx - page_idx);
9102 - if (!page_is_buddy(page, buddy, order))
9103 - break;
9104 - /*
9105 - * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
9106 - * merge with it and move up one order.
9107 - */
9108 - if (page_is_guard(buddy)) {
9109 - clear_page_guard(zone, buddy, order, migratetype);
9110 + if (order >= MAX_PARTITIONED_ORDER) {
9111 + int n_idx = 0;
9112 + struct page *lower_page;
9113 + for (n_idx = 0 ; n_idx < (1 << (order - MAX_PARTITIONED_ORDER + 1)); n_idx++) {
9114 + lower_page = page + (n_idx << (MAX_PARTITIONED_ORDER - 1));
9115 + if (lower_page->flags & PAGE_FLAGS_CHECK_AT_PREP)
9116 + lower_page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
9117 + set_page_order(lower_page, MAX_PARTITIONED_ORDER-1);
9118 + list_add(&lower_page->lru, &zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].free_list[migratetype]);
9119 + zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].nr_free++;
9120 + }
9121 } else {
9122 - list_del(&buddy->lru);
9123 - zone->free_area[order].nr_free--;
9124 - rmv_page_order(buddy);
9125 + list_add(&page->lru, &zone->free_area_d[parti_no][order].free_list[migratetype]);
9126 + zone->free_area_d[parti_no][order].nr_free++;
9127 }
9128 - combined_idx = buddy_idx & page_idx;
9129 - page = page + (combined_idx - page_idx);
9130 - page_idx = combined_idx;
9131 - order++;
9132 }
9133 - set_page_order(page, order);
9134 + else {
9135 + max_order = MAX_ORDER;
9136
9137 - /*
9138 - * If this is not the largest possible page, check if the buddy
9139 - * of the next-highest order is free. If it is, it's possible
9140 - * that pages are being freed that will coalesce soon. In case,
9141 - * that is happening, add the free page to the tail of the list
9142 - * so it's less likely to be used soon and more likely to be merged
9143 - * as a higher order page
9144 - */
9145 - if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
9146 - struct page *higher_page, *higher_buddy;
9147 - combined_idx = buddy_idx & page_idx;
9148 - higher_page = page + (combined_idx - page_idx);
9149 - buddy_idx = __find_buddy_index(combined_idx, order + 1);
9150 - higher_buddy = higher_page + (buddy_idx - combined_idx);
9151 - if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
9152 - list_add_tail(&page->lru,
9153 - &zone->free_area[order].free_list[migratetype]);
9154 - goto out;
9155 + VM_BUG_ON(!zone_is_initialized(zone));
9156 + VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
9157 +
9158 + VM_BUG_ON(migratetype == -1);
9159 + if (is_migrate_isolate(migratetype)) {
9160 + /*
9161 + * We restrict max order of merging to prevent merge
9162 + * between freepages on isolate pageblock and normal
9163 + * pageblock. Without this, pageblock isolation
9164 + * could cause incorrect freepage accounting.
9165 + */
9166 + max_order = min(MAX_ORDER, pageblock_order + 1);
9167 + } else {
9168 + __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no);
9169 }
9170 - }
9171
9172 - list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
9173 + page_idx = pfn & ((1 << max_order) - 1);
9174 +
9175 + VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
9176 + VM_BUG_ON_PAGE(bad_range(zone, page), page);
9177 +
9178 + while (order < max_order - 1) {
9179 + buddy_idx = __find_buddy_index(page_idx, order);
9180 + buddy = page + (buddy_idx - page_idx);
9181 + if (!page_is_buddy(page, buddy, order))
9182 + break;
9183 + /*
9184 + * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
9185 + * merge with it and move up one order.
9186 + */
9187 + if (page_is_guard(buddy)) {
9188 + clear_page_guard(zone, buddy, order, migratetype);
9189 + } else {
9190 + list_del(&buddy->lru);
9191 + zone->free_area[order].nr_free--;
9192 + rmv_page_order(buddy);
9193 + }
9194 + combined_idx = buddy_idx & page_idx;
9195 + page = page + (combined_idx - page_idx);
9196 + page_idx = combined_idx;
9197 + order++;
9198 + }
9199 + set_page_order(page, order);
9200 +
9201 + /*
9202 + * If this is not the largest possible page, check if the buddy
9203 + * of the next-highest order is free. If it is, it's possible
9204 + * that pages are being freed that will coalesce soon. In case,
9205 + * that is happening, add the free page to the tail of the list
9206 + * so it's less likely to be used soon and more likely to be merged
9207 + * as a higher order page
9208 + */
9209 + if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
9210 + struct page *higher_page, *higher_buddy;
9211 + combined_idx = buddy_idx & page_idx;
9212 + higher_page = page + (combined_idx - page_idx);
9213 + buddy_idx = __find_buddy_index(combined_idx, order + 1);
9214 + higher_buddy = higher_page + (buddy_idx - combined_idx);
9215 + if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
9216 + list_add_tail(&page->lru,
9217 + &zone->free_area[order].free_list[migratetype]);
9218 + goto out;
9219 + }
9220 + }
9221 +
9222 + list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
9223 out:
9224 - zone->free_area[order].nr_free++;
9225 + zone->free_area[order].nr_free++;
9226 + }
9227 }
9228
9229 static inline int free_pages_check(struct page *page)
9230 @@ -829,7 +928,10 @@ static void __free_pages_ok(struct page *page, unsigned int order)
9231
9232 migratetype = get_pfnblock_migratetype(page, pfn);
9233 local_irq_save(flags);
9234 - __count_vm_events(PGFREE, 1 << order);
9235 + if (bank_to_partition(page_bank(page)) == NR_CPUS)
9236 + __count_vm_events(PGFREE, 1 << order);
9237 + else if (bank_to_partition(page_bank(page)) < NR_CPUS)
9238 + __count_vm_events(PGFREE_HC, 1 << order);
9239 set_freepage_migratetype(page, migratetype);
9240 free_one_page(page_zone(page), page, pfn, order, migratetype);
9241 local_irq_restore(flags);
9242 @@ -930,6 +1032,45 @@ static inline void expand(struct zone *zone, struct page *page,
9243 }
9244 }
9245
9246 +static inline int expand_middle(struct zone *zone, struct page *page,
9247 + int offset, int low, int high, struct free_area *area,
9248 + int migratetype)
9249 +{
9250 + unsigned long size = 1 << high;
9251 +
9252 + while ((size>>1) > offset) {
9253 + area--;
9254 + high--;
9255 + size >>= 1;
9256 + VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
9257 +
9258 + list_add(&page[size].lru, &area->free_list[migratetype]);
9259 + area->nr_free++;
9260 + set_page_order(&page[size], high);
9261 + }
9262 + area--;
9263 + high--;
9264 + size >>= 1;
9265 + VM_BUG_ON_PAGE(bad_range(zone, page), page);
9266 + list_add(&page[0].lru, &area->free_list[migratetype]);
9267 + area->nr_free++;
9268 + set_page_order(&page[0], high);
9269 +
9270 + if (offset == size) {
9271 + //printk(KERN_INFO "offset == size %d high = %d\n", offset, high);
9272 + return high;
9273 + }
9274 +
9275 + area--;
9276 + high--;
9277 + VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
9278 + list_add(&page[size].lru, &area->free_list[migratetype]);
9279 + area->nr_free++;
9280 + set_page_order(&page[size], high);
9281 +
9282 + return high;
9283 +}
9284 +
9285 /*
9286 * This page is about to be returned from the page allocator
9287 */
9288 @@ -996,34 +1137,190 @@ static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
9289 return 0;
9290 }
9291
9292 +/* Kernel page coloring */
9293 +
9294 +/* build colored page list */
9295 +#if 0
9296 +static void build_colored_pages(struct zone *zone, struct page *page, int order)
9297 +{
9298 + int i, color, bank;
9299 +
9300 + list_del(&page->lru);
9301 + zone->free_area[order].nr_free--;
9302 +
9303 + /* insert pages to zone->color_list[] */
9304 + for (i = 0; i < (1<<order); i++) {
9305 + int node;
9306 + color = page_color(&page[i]);
9307 + bank = page_bank(&page[i]);
9308 + node = bank*MAX_NUM_COLOR+color;
9309 +
9310 + INIT_LIST_HEAD(&page[i].lru);
9311 + list_add_tail(&page[i].lru, &zone->color_list[node]);
9312 + bitmap_set(zone->color_map, node, 1);
9313 + zone->free_area[0].nr_free++;
9314 + rmv_page_order(&page[i]);
9315 + }
9316 +}
9317 +
9318 +int color_seq_index[9] = {
9319 + 0, /* Core 0, and Level A*/
9320 + 0, /* Core 0, and Level B*/
9321 + 0, /* Core 1, and Level A*/
9322 + 0, /* Core 1, and Level B*/
9323 + 0, /* Core 2, and Level A*/
9324 + 0, /* Core 2, and Level B*/
9325 + 0, /* Core 3, and Level A*/
9326 + 0, /* Core 3, and Level B*/
9327 + 0, /* Level C */
9328 +};
9329 +
9330 +/* return a colored page */
9331 +static inline struct page *get_colored_page(struct zone *zone, unsigned long req_color_map[BITS_TO_LONGS(MAX_COLOR_NODE)], int order, int partition)
9332 +{
9333 + struct page *page;
9334 + unsigned int color, bank, index;
9335 + int i;
9336 + DECLARE_BITMAP(candidate_bit, MAX_COLOR_NODE);
9337 +
9338 + /* if req_color_map does not exist in zone, return NULL */
9339 + if (!bitmap_intersects(zone->color_map, req_color_map, MAX_COLOR_NODE))
9340 + return NULL;
9341 +
9342 + bitmap_and(candidate_bit, zone->color_map, req_color_map, MAX_COLOR_NODE);
9343 + index = color_seq_index[partition];
9344 +
9345 + for_each_set_bit(i, candidate_bit, MAX_COLOR_NODE) {
9346 + if (index-- <= 0)
9347 + break;
9348 + }
9349 +
9350 + BUG_ON(i >= MAX_COLOR_NODE);
9351 + BUG_ON(list_empty(&zone->color_list[i]));
9352 +
9353 + page = list_entry(zone->color_list[i].next, struct page, lru);
9354 +
9355 + list_del(&page->lru);
9356 +
9357 + if (list_empty(&zone->color_list[i]))
9358 + bitmap_clear(zone->color_map, i, 1);
9359 +
9360 + zone->free_area[0].nr_free--;
9361 + color = page_color(page);
9362 + bank = page_bank(page);
9363 + printk(KERN_INFO "color=%d, bank=%d allocated\n", color, bank);
9364 + return page;
9365 +}
9366 +#endif
9367 +
9368 /*
9369 * Go through the free lists for the given migratetype and remove
9370 * the smallest available page from the freelists
9371 */
9372 static inline
9373 struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
9374 - int migratetype)
9375 + int migratetype, int color_req)
9376 {
9377 unsigned int current_order;
9378 struct free_area *area;
9379 struct page *page;
9380 + int cpu = raw_smp_processor_id();
9381 +
9382 + // if (order <= 2 && color_req == 1) {
9383 + /* The max. order of color_req is <= 2 */
9384 + if (color_req != 0) {
9385 + int found = 0;
9386 + int area_index;
9387 + unsigned long s_pfn = zone->zone_start_pfn;
9388 + unsigned long e_pfn = zone_end_pfn(zone);
9389 +
9390 + if (color_req == 2)
9391 + area_index = get_area_index(1);
9392 + else
9393 + area_index = get_area_index(cpu);
9394 +
9395 + //printk(KERN_INFO "CPU%d color_request %d, area_index %d\n", cpu, color_req, area_index);
9396 + //printk(KERN_INFO "COLOR PAGE requested on CPU%d with order = %d migratetype = %d\n", cpu, order, migratetype);
9397 + /* Find a page of the appropriate size in the preferred list */
9398 + for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) {
9399 + int offset = 0;
9400 + area = &(zone->free_area_d[area_index][current_order]);
9401 + if (list_empty(&area->free_list[migratetype])) {
9402 + //printk(KERN_INFO "P%d order %d list empty\n", cpu, current_order);
9403 + continue;
9404 + }
9405 +
9406 + if (order >= MAX_CONTIG_ORDER) { // requested order >= 3 , must be uncacheable.
9407 + page = list_entry(area->free_list[migratetype].next, struct page, lru);
9408 + found = 1;
9409 + } else {
9410 + list_for_each_entry(page, &area->free_list[migratetype], lru) {
9411 + //printk(KERN_INFO "P%d __rmqueue_smallest order [%d] list entry %p color %d pfn:%05lx\n", cpu, current_order, page, page_color(page), page_to_pfn(page));
9412 + if (current_order < MAX_CONTIG_ORDER) {
9413 + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) {
9414 + offset = 0;
9415 + found = 1;
9416 + break;
9417 + }
9418 + } else {
9419 + int size = 1 << current_order;
9420 + for (offset = 0; offset < size; offset++) {
9421 + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) {
9422 + found = 1;
9423 + break;
9424 + }
9425 + }
9426 + if (found)
9427 + break;
9428 + }
9429 + }
9430 + }
9431 +
9432 + //printk(KERN_INFO "P%d __rmqueue_smallest LAST list entry %p\n", cpu, page);
9433 +
9434 + if (!found)
9435 + continue;
9436 + //printk(KERN_INFO "P%d __rmqueue_smallest LAST list entry %p, order %d current_order %d offset %d\n", cpu, page, order, current_order, offset);
9437 +
9438 + list_del(&page->lru);
9439 + rmv_page_order(page);
9440 + area->nr_free--;
9441 +
9442 + if (offset == 0) {
9443 + expand(zone, page, order, current_order, area, migratetype);
9444 + } else {
9445 + int frac = expand_middle(zone, page, offset, order, current_order, area, migratetype);
9446 + page = &page[offset];
9447 + //list_del(&page->lru);
9448 + //rmv_page_order(page);
9449 + area = &(zone->free_area_d[area_index][frac]);
9450 + //area->nr_free--;
9451 + expand(zone, page, order, frac, area, migratetype);
9452 + }
9453 +
9454 + set_freepage_migratetype(page, migratetype);
9455 + //printk(KERN_INFO "__rmqueue_smallest(): CPU%d COLOR %d BANK %d page return %p pfn:%05lx\n", cpu, page_color(page), page_bank(page), page, page_to_pfn(page));
9456 + return page;
9457 + }
9458 + } else {
9459 + /* Buddy allocator */
9460 + /* Find a page of the appropriate size in the preferred list */
9461 + for (current_order = order; current_order < MAX_ORDER; ++current_order) {
9462 + area = &(zone->free_area[current_order]);
9463 + if (list_empty(&area->free_list[migratetype]))
9464 + continue;
9465
9466 - /* Find a page of the appropriate size in the preferred list */
9467 - for (current_order = order; current_order < MAX_ORDER; ++current_order) {
9468 - area = &(zone->free_area[current_order]);
9469 - if (list_empty(&area->free_list[migratetype]))
9470 - continue;
9471 -
9472 - page = list_entry(area->free_list[migratetype].next,
9473 - struct page, lru);
9474 - list_del(&page->lru);
9475 - rmv_page_order(page);
9476 - area->nr_free--;
9477 - expand(zone, page, order, current_order, area, migratetype);
9478 - set_freepage_migratetype(page, migratetype);
9479 - return page;
9480 + page = list_entry(area->free_list[migratetype].next,
9481 + struct page, lru);
9482 + list_del(&page->lru);
9483 + rmv_page_order(page);
9484 + area->nr_free--;
9485 + expand(zone, page, order, current_order, area, migratetype);
9486 + set_freepage_migratetype(page, migratetype);
9487 + return page;
9488 + }
9489 }
9490 -
9491 +
9492 return NULL;
9493 }
9494
9495 @@ -1049,7 +1346,7 @@ static int fallbacks[MIGRATE_TYPES][4] = {
9496 static struct page *__rmqueue_cma_fallback(struct zone *zone,
9497 unsigned int order)
9498 {
9499 - return __rmqueue_smallest(zone, order, MIGRATE_CMA);
9500 + return __rmqueue_smallest(zone, order, MIGRATE_CMA, 0);
9501 }
9502 #else
9503 static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
9504 @@ -1236,7 +1533,7 @@ int find_suitable_fallback(struct free_area *area, unsigned int order,
9505
9506 /* Remove an element from the buddy allocator from the fallback list */
9507 static inline struct page *
9508 -__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
9509 +__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, int color_req)
9510 {
9511 struct free_area *area;
9512 unsigned int current_order;
9513 @@ -1244,44 +1541,138 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
9514 int fallback_mt;
9515 bool can_steal;
9516
9517 - /* Find the largest possible block of pages in the other list */
9518 - for (current_order = MAX_ORDER-1;
9519 - current_order >= order && current_order <= MAX_ORDER-1;
9520 - --current_order) {
9521 - area = &(zone->free_area[current_order]);
9522 - fallback_mt = find_suitable_fallback(area, current_order,
9523 - start_migratetype, false, &can_steal);
9524 - if (fallback_mt == -1)
9525 - continue;
9526 + if (color_req != 0) {
9527 + int cpu = raw_smp_processor_id();
9528 + int found = 0;
9529 + int area_index;
9530 + unsigned long s_pfn = zone->zone_start_pfn;
9531 + unsigned long e_pfn = zone_end_pfn(zone);
9532 +
9533 + if (color_req == 2)
9534 + area_index = get_area_index(1);
9535 + else
9536 + area_index = get_area_index(cpu);
9537 +
9538 + /* Find the largest possible block of pages in the other list */
9539 + for (current_order = MAX_PARTITIONED_ORDER-1;
9540 + current_order >= order && current_order <= MAX_PARTITIONED_ORDER-1;
9541 + --current_order) {
9542 + int offset = 0;
9543 + area = &(zone->free_area_d[area_index][current_order]);
9544 + fallback_mt = find_suitable_fallback(area, current_order,
9545 + start_migratetype, false, &can_steal);
9546 + if (fallback_mt == -1)
9547 + continue;
9548 +
9549 + if (order >= MAX_CONTIG_ORDER) {
9550 + page = list_entry(area->free_list[fallback_mt].next, struct page, lru);
9551 + found = 1;
9552 + //printk(KERN_INFO "__rmqueue_fallback order >= MAX_CONTIG_ORDER page = %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page));
9553 + } else {
9554 + list_for_each_entry(page, &area->free_list[fallback_mt], lru) {
9555 + //printk(KERN_INFO "__rmqueue_fallback list entry %p color %d pfn:%05lx\n", page, page_color(page), page_to_pfn(page));
9556 + if (current_order < MAX_CONTIG_ORDER) {
9557 + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) {
9558 + found = 1;
9559 + offset = 0;
9560 + break;
9561 + }
9562 + } else {
9563 + int size = 1 << current_order;
9564 + for (offset = 0; offset < size; offset++) {
9565 + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) {
9566 + found = 1;
9567 + break;
9568 + }
9569 + }
9570 + if (found)
9571 + break;
9572 + }
9573 + }
9574 + }
9575 + //printk(KERN_INFO "__rmqueue_fallback LAST list entry %p\n", page);
9576
9577 - page = list_entry(area->free_list[fallback_mt].next,
9578 - struct page, lru);
9579 - if (can_steal)
9580 - steal_suitable_fallback(zone, page, start_migratetype);
9581 + if (!found)
9582 + continue;
9583 +
9584 + if (can_steal)
9585 + steal_suitable_fallback(zone, page, start_migratetype);
9586
9587 - /* Remove the page from the freelists */
9588 - area->nr_free--;
9589 - list_del(&page->lru);
9590 - rmv_page_order(page);
9591 + /* Remove the page from the freelists */
9592 + area->nr_free--;
9593 + list_del(&page->lru);
9594 + rmv_page_order(page);
9595 +
9596 + if (offset == 0)
9597 + expand(zone, page, order, current_order, area, start_migratetype);
9598 + else {
9599 + int frac = expand_middle(zone, page, offset, order, current_order, area, start_migratetype);
9600 + page = &page[offset];
9601 + //list_del(&page->lru);
9602 + //rmv_page_order(page);
9603 + area = &(zone->free_area_d[area_index][frac]);
9604 + //area->nr_free--;
9605 + expand(zone, page, order, frac, area, start_migratetype);
9606 +
9607 + }
9608 +
9609
9610 - expand(zone, page, order, current_order, area,
9611 - start_migratetype);
9612 - /*
9613 - * The freepage_migratetype may differ from pageblock's
9614 - * migratetype depending on the decisions in
9615 - * try_to_steal_freepages(). This is OK as long as it
9616 - * does not differ for MIGRATE_CMA pageblocks. For CMA
9617 - * we need to make sure unallocated pages flushed from
9618 - * pcp lists are returned to the correct freelist.
9619 - */
9620 - set_freepage_migratetype(page, start_migratetype);
9621 + /*
9622 + * The freepage_migratetype may differ from pageblock's
9623 + * migratetype depending on the decisions in
9624 + * try_to_steal_freepages(). This is OK as long as it
9625 + * does not differ for MIGRATE_CMA pageblocks. For CMA
9626 + * we need to make sure unallocated pages flushed from
9627 + * pcp lists are returned to the correct freelist.
9628 + */
9629 + set_freepage_migratetype(page, start_migratetype);
9630
9631 - trace_mm_page_alloc_extfrag(page, order, current_order,
9632 - start_migratetype, fallback_mt);
9633 + trace_mm_page_alloc_extfrag(page, order, current_order,
9634 + start_migratetype, fallback_mt);
9635 +
9636 + //printk(KERN_INFO "__rmqueue_fallback(): CPU%d COLOR %d BANK %d page return %p pfn:%05lx\n", cpu, page_color(page), page_bank(page), page, page_to_pfn(page));
9637 + return page;
9638 + }
9639 + } else {
9640 + /* Find the largest possible block of pages in the other list */
9641 + for (current_order = MAX_ORDER-1;
9642 + current_order >= order && current_order <= MAX_ORDER-1;
9643 + --current_order) {
9644 + area = &(zone->free_area[current_order]);
9645 + fallback_mt = find_suitable_fallback(area, current_order,
9646 + start_migratetype, false, &can_steal);
9647 + if (fallback_mt == -1)
9648 + continue;
9649
9650 - return page;
9651 - }
9652 + page = list_entry(area->free_list[fallback_mt].next,
9653 + struct page, lru);
9654 + if (can_steal)
9655 + steal_suitable_fallback(zone, page, start_migratetype);
9656 +
9657 + /* Remove the page from the freelists */
9658 + area->nr_free--;
9659 + list_del(&page->lru);
9660 + rmv_page_order(page);
9661
9662 + expand(zone, page, order, current_order, area,
9663 + start_migratetype);
9664 + /*
9665 + * The freepage_migratetype may differ from pageblock's
9666 + * migratetype depending on the decisions in
9667 + * try_to_steal_freepages(). This is OK as long as it
9668 + * does not differ for MIGRATE_CMA pageblocks. For CMA
9669 + * we need to make sure unallocated pages flushed from
9670 + * pcp lists are returned to the correct freelist.
9671 + */
9672 + set_freepage_migratetype(page, start_migratetype);
9673 +
9674 + trace_mm_page_alloc_extfrag(page, order, current_order,
9675 + start_migratetype, fallback_mt);
9676 +
9677 + return page;
9678 + }
9679 + }
9680 +
9681 return NULL;
9682 }
9683
9684 @@ -1290,26 +1681,32 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
9685 * Call me with the zone->lock already held.
9686 */
9687 static struct page *__rmqueue(struct zone *zone, unsigned int order,
9688 - int migratetype)
9689 + int migratetype, int color_req)
9690 {
9691 struct page *page;
9692
9693 retry_reserve:
9694 - page = __rmqueue_smallest(zone, order, migratetype);
9695 + page = __rmqueue_smallest(zone, order, migratetype, color_req);
9696
9697 if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
9698 if (migratetype == MIGRATE_MOVABLE)
9699 page = __rmqueue_cma_fallback(zone, order);
9700
9701 - if (!page)
9702 - page = __rmqueue_fallback(zone, order, migratetype);
9703 + if (!page) {
9704 + page = __rmqueue_fallback(zone, order, migratetype, color_req);
9705 +#if 0
9706 +//#ifdef CONFIG_SCHED_DEBUG_TRACE
9707 + if (color_req)
9708 + printk(KERN_INFO "page received from __rmqueue_fallback()");
9709 +#endif
9710 + }
9711
9712 /*
9713 * Use MIGRATE_RESERVE rather than fail an allocation. goto
9714 * is used because __rmqueue_smallest is an inline function
9715 * and we want just one call site
9716 */
9717 - if (!page) {
9718 + if (!page && !color_req) {
9719 migratetype = MIGRATE_RESERVE;
9720 goto retry_reserve;
9721 }
9722 @@ -1332,7 +1729,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
9723
9724 spin_lock(&zone->lock);
9725 for (i = 0; i < count; ++i) {
9726 - struct page *page = __rmqueue(zone, order, migratetype);
9727 + struct page *page = __rmqueue(zone, order, migratetype, 0);
9728 if (unlikely(page == NULL))
9729 break;
9730
9731 @@ -1542,6 +1939,8 @@ void free_hot_cold_page(struct page *page, bool cold)
9732 unsigned long flags;
9733 unsigned long pfn = page_to_pfn(page);
9734 int migratetype;
9735 + //unsigned int part_no;
9736 + //int is_local, is_hc_page;
9737
9738 if (!free_pages_prepare(page, 0))
9739 return;
9740 @@ -1549,8 +1948,16 @@ void free_hot_cold_page(struct page *page, bool cold)
9741 migratetype = get_pfnblock_migratetype(page, pfn);
9742 set_freepage_migratetype(page, migratetype);
9743 local_irq_save(flags);
9744 - __count_vm_event(PGFREE);
9745 -
9746 +
9747 +
9748 + if (bank_to_partition(page_bank(page)) == NR_CPUS)
9749 + __count_vm_event(PGFREE);
9750 + else if (bank_to_partition(page_bank(page)) < NR_CPUS) {
9751 + __count_vm_event(PGFREE_HC);
9752 + free_one_page(zone, page, pfn, 0, migratetype);
9753 + goto out;
9754 + }
9755 +
9756 /*
9757 * We only track unmovable, reclaimable and movable on pcp lists.
9758 * Free ISOLATE pages back to the allocator because they are being
9759 @@ -1565,19 +1972,33 @@ void free_hot_cold_page(struct page *page, bool cold)
9760 }
9761 migratetype = MIGRATE_MOVABLE;
9762 }
9763 -
9764 - pcp = &this_cpu_ptr(zone->pageset)->pcp;
9765 - if (!cold)
9766 - list_add(&page->lru, &pcp->lists[migratetype]);
9767 - else
9768 - list_add_tail(&page->lru, &pcp->lists[migratetype]);
9769 - pcp->count++;
9770 - if (pcp->count >= pcp->high) {
9771 - unsigned long batch = READ_ONCE(pcp->batch);
9772 - free_pcppages_bulk(zone, batch, pcp);
9773 - pcp->count -= batch;
9774 - }
9775 -
9776 +
9777 + //part_no = bank_to_partition(page_bank(page));
9778 + //BUG_ON(part_no<0);
9779 +
9780 + //if (part_no == smp_processor_id())
9781 + // is_local = 1;
9782 + //else
9783 + // is_local = 0;
9784 +
9785 + //is_hc_page = is_in_llc_partition(page, smp_processor_id());
9786 + //if (part_no != NR_CPUS)
9787 + // printk(KERN_ALERT "CPU%d Free order-0 page bank = %d, color = %d, is_local %d is_hc_page %d\n", smp_processor_id(), page_bank(page), page_color(page), is_local, is_hc_page);
9788 + //if (!is_local || !is_hc_page) {
9789 + pcp = &this_cpu_ptr(zone->pageset)->pcp;
9790 + if (!cold)
9791 + list_add(&page->lru, &pcp->lists[migratetype]);
9792 + else
9793 + list_add_tail(&page->lru, &pcp->lists[migratetype]);
9794 + pcp->count++;
9795 + if (pcp->count >= pcp->high) {
9796 + unsigned long batch = READ_ONCE(pcp->batch);
9797 + free_pcppages_bulk(zone, batch, pcp);
9798 + pcp->count -= batch;
9799 + }
9800 +// } else {
9801 +// __free_page(page);
9802 +// }
9803 out:
9804 local_irq_restore(flags);
9805 }
9806 @@ -1590,8 +2011,12 @@ void free_hot_cold_page_list(struct list_head *list, bool cold)
9807 struct page *page, *next;
9808
9809 list_for_each_entry_safe(page, next, list, lru) {
9810 + int parti_no = bank_to_partition(page_bank(page));
9811 trace_mm_page_free_batched(page, cold);
9812 - free_hot_cold_page(page, cold);
9813 + if (parti_no == NR_CPUS)
9814 + free_hot_cold_page(page, cold);
9815 + else
9816 + __free_pages_ok(page, 0);
9817 }
9818 }
9819
9820 @@ -1644,7 +2069,7 @@ int __isolate_free_page(struct page *page, unsigned int order)
9821 if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
9822 return 0;
9823
9824 - __mod_zone_freepage_state(zone, -(1UL << order), mt);
9825 + __mod_zone_freepage_state(zone, -(1UL << order), mt, bank_to_partition(page_bank(page)));
9826 }
9827
9828 /* Remove page from free list */
9829 @@ -1705,8 +2130,14 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
9830 unsigned long flags;
9831 struct page *page;
9832 bool cold = ((gfp_flags & __GFP_COLD) != 0);
9833 -
9834 - if (likely(order == 0)) {
9835 + int colored_req = ((gfp_flags & __GFP_COLOR) != 0);
9836 +
9837 +#ifdef CONFIG_SCHED_DEBUG_TRACE
9838 + if (colored_req)
9839 + printk(KERN_INFO "buffered_rmqueue(): colored_req %d received\n", colored_req);
9840 +#endif
9841 +
9842 + if (likely(order == 0) && !colored_req) {
9843 struct per_cpu_pages *pcp;
9844 struct list_head *list;
9845
9846 @@ -1743,12 +2174,12 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
9847 WARN_ON_ONCE(order > 1);
9848 }
9849 spin_lock_irqsave(&zone->lock, flags);
9850 - page = __rmqueue(zone, order, migratetype);
9851 + page = __rmqueue(zone, order, migratetype, colored_req);
9852 spin_unlock(&zone->lock);
9853 if (!page)
9854 goto failed;
9855 __mod_zone_freepage_state(zone, -(1 << order),
9856 - get_freepage_migratetype(page));
9857 + get_freepage_migratetype(page), bank_to_partition(page_bank(page)));
9858 }
9859
9860 __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
9861 @@ -2856,6 +3287,12 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
9862 .migratetype = gfpflags_to_migratetype(gfp_mask),
9863 };
9864
9865 +//#ifdef CONFIG_SCHED_DEBUG_TRACE
9866 +#if 0
9867 + if (gfp_mask&GFP_COLOR)
9868 + printk(KERN_INFO "__alloc_pages_nodemask(): called gfp %08x gfp_allowed_mask %08x mt = %d\n", gfp_mask, gfp_allowed_mask, ac.migratetype);
9869 +#endif
9870 +
9871 gfp_mask &= gfp_allowed_mask;
9872
9873 lockdep_trace_alloc(gfp_mask);
9874 @@ -2950,8 +3387,9 @@ EXPORT_SYMBOL(get_zeroed_page);
9875
9876 void __free_pages(struct page *page, unsigned int order)
9877 {
9878 + int parti_no = bank_to_partition(page_bank(page));
9879 if (put_page_testzero(page)) {
9880 - if (order == 0)
9881 + if (order == 0 && parti_no == NR_CPUS)
9882 free_hot_cold_page(page, false);
9883 else
9884 __free_pages_ok(page, order);
9885 @@ -4230,10 +4668,23 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
9886 static void __meminit zone_init_free_lists(struct zone *zone)
9887 {
9888 unsigned int order, t;
9889 + int cpu;
9890 +
9891 for_each_migratetype_order(order, t) {
9892 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
9893 zone->free_area[order].nr_free = 0;
9894 }
9895 +
9896 + /* Initialize per-partition free_area data structures */
9897 + for (cpu = 0; cpu < NR_CPUS; cpu++) {
9898 + for (order = 0; order < MAX_PARTITIONED_ORDER; order++) {
9899 + for (t = 0; t < MIGRATE_TYPES; t++) {
9900 + INIT_LIST_HEAD(&zone->free_area_d[cpu][order].free_list[t]);
9901 + zone->free_area_d[cpu][order].nr_free = 0;
9902 + printk(KERN_ALERT "free_area_d[%d][%d].free_list[%d] init.\n", cpu, order, t);
9903 + }
9904 + }
9905 + }
9906 }
9907
9908 #ifndef __HAVE_ARCH_MEMMAP_INIT
9909 diff --git mm/page_isolation.c mm/page_isolation.c
9910 index 303c908..ed05910 100644
9911 --- mm/page_isolation.c
9912 +++ mm/page_isolation.c
9913 @@ -58,12 +58,13 @@ out:
9914 if (!ret) {
9915 unsigned long nr_pages;
9916 int migratetype = get_pageblock_migratetype(page);
9917 + int partno = bank_to_partition(page_bank(page));
9918
9919 set_pageblock_migratetype(page, MIGRATE_ISOLATE);
9920 zone->nr_isolate_pageblock++;
9921 nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE);
9922
9923 - __mod_zone_freepage_state(zone, -nr_pages, migratetype);
9924 + __mod_zone_freepage_state(zone, -nr_pages, migratetype, partno);
9925 }
9926
9927 spin_unlock_irqrestore(&zone->lock, flags);
9928 @@ -117,8 +118,9 @@ void unset_migratetype_isolate(struct page *page, unsigned migratetype)
9929 * pageblock scanning for freepage moving.
9930 */
9931 if (!isolated_page) {
9932 + int partno = bank_to_partition(page_bank(page));
9933 nr_pages = move_freepages_block(zone, page, migratetype);
9934 - __mod_zone_freepage_state(zone, nr_pages, migratetype);
9935 + __mod_zone_freepage_state(zone, nr_pages, migratetype, partno);
9936 }
9937 set_pageblock_migratetype(page, migratetype);
9938 zone->nr_isolate_pageblock--;
9939 diff --git mm/rmap.c mm/rmap.c
9940 index 24dd3f9..0613210 100644
9941 --- mm/rmap.c
9942 +++ mm/rmap.c
9943 @@ -1317,6 +1317,170 @@ out_mlock:
9944 return ret;
9945 }
9946
9947 +/*
9948 + * @arg: enum ttu_flags will be passed to this argument
9949 + */
9950 +static int try_to_unmap_one_entry(struct page *page, struct vm_area_struct *vma,
9951 + unsigned long address, void *arg)
9952 +{
9953 + struct mm_struct *mm = vma->vm_mm;
9954 + pte_t *pte;
9955 + pte_t pteval;
9956 + spinlock_t *ptl;
9957 + int ret = SWAP_AGAIN;
9958 + enum ttu_flags flags = (enum ttu_flags)arg;
9959 +
9960 + pte = page_check_address(page, mm, address, &ptl, 0);
9961 + if (!pte)
9962 + goto out;
9963 +
9964 + /*
9965 + * If the page is mlock()d, we cannot swap it out.
9966 + * If it's recently referenced (perhaps page_referenced
9967 + * skipped over this mm) then we should reactivate it.
9968 + */
9969 + if (!(flags & TTU_IGNORE_MLOCK)) {
9970 + if (vma->vm_flags & VM_LOCKED)
9971 + goto out_mlock;
9972 +
9973 + if (flags & TTU_MUNLOCK)
9974 + goto out_unmap;
9975 + }
9976 + if (!(flags & TTU_IGNORE_ACCESS)) {
9977 + if (ptep_clear_flush_young_notify(vma, address, pte)) {
9978 + ret = SWAP_FAIL;
9979 + goto out_unmap;
9980 + }
9981 + }
9982 +
9983 + /* Nuke the page table entry. */
9984 + flush_cache_page(vma, address, page_to_pfn(page));
9985 + pteval = ptep_clear_flush(vma, address, pte);
9986 +
9987 + /* Move the dirty bit to the physical page now the pte is gone. */
9988 + if (pte_dirty(pteval))
9989 + set_page_dirty(page);
9990 +
9991 + /* Update high watermark before we lower rss */
9992 + update_hiwater_rss(mm);
9993 +
9994 + if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
9995 + if (!PageHuge(page)) {
9996 + if (PageAnon(page))
9997 + dec_mm_counter(mm, MM_ANONPAGES);
9998 + else
9999 + dec_mm_counter(mm, MM_FILEPAGES);
10000 + }
10001 + set_pte_at(mm, address, pte,
10002 + swp_entry_to_pte(make_hwpoison_entry(page)));
10003 + } else if (pte_unused(pteval)) {
10004 + /*
10005 + * The guest indicated that the page content is of no
10006 + * interest anymore. Simply discard the pte, vmscan
10007 + * will take care of the rest.
10008 + */
10009 + if (PageAnon(page))
10010 + dec_mm_counter(mm, MM_ANONPAGES);
10011 + else
10012 + dec_mm_counter(mm, MM_FILEPAGES);
10013 + } else if (PageAnon(page)) {
10014 + swp_entry_t entry = { .val = page_private(page) };
10015 + pte_t swp_pte;
10016 +
10017 + if (PageSwapCache(page)) {
10018 + /*
10019 + * Store the swap location in the pte.
10020 + * See handle_pte_fault() ...
10021 + */
10022 + if (swap_duplicate(entry) < 0) {
10023 + set_pte_at(mm, address, pte, pteval);
10024 + ret = SWAP_FAIL;
10025 + goto out_unmap;
10026 + }
10027 + if (list_empty(&mm->mmlist)) {
10028 + spin_lock(&mmlist_lock);
10029 + if (list_empty(&mm->mmlist))
10030 + list_add(&mm->mmlist, &init_mm.mmlist);
10031 + spin_unlock(&mmlist_lock);
10032 + }
10033 + dec_mm_counter(mm, MM_ANONPAGES);
10034 + inc_mm_counter(mm, MM_SWAPENTS);
10035 + } else if (IS_ENABLED(CONFIG_MIGRATION)) {
10036 + /*
10037 + * Store the pfn of the page in a special migration
10038 + * pte. do_swap_page() will wait until the migration
10039 + * pte is removed and then restart fault handling.
10040 + */
10041 + BUG_ON(!(flags & TTU_MIGRATION));
10042 + entry = make_migration_entry(page, pte_write(pteval));
10043 + }
10044 + swp_pte = swp_entry_to_pte(entry);
10045 + if (pte_soft_dirty(pteval))
10046 + swp_pte = pte_swp_mksoft_dirty(swp_pte);
10047 + set_pte_at(mm, address, pte, swp_pte);
10048 + } else if (IS_ENABLED(CONFIG_MIGRATION) &&
10049 + (flags & TTU_MIGRATION)) {
10050 + /* Establish migration entry for a file page */
10051 + swp_entry_t entry;
10052 + entry = make_migration_entry(page, pte_write(pteval));
10053 + set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
10054 + } else
10055 + dec_mm_counter(mm, MM_FILEPAGES);
10056 +
10057 + page_remove_rmap(page);
10058 + page_cache_release(page);
10059 +
10060 +out_unmap:
10061 + pte_unmap_unlock(pte, ptl);
10062 + if (ret != SWAP_FAIL && !(flags & TTU_MUNLOCK)) {
10063 + mmu_notifier_invalidate_page(mm, address);
10064 + ret = SWAP_SUCCESS;
10065 + }
10066 +out:
10067 + return ret;
10068 +
10069 +out_mlock:
10070 + pte_unmap_unlock(pte, ptl);
10071 +
10072 +
10073 + /*
10074 + * We need mmap_sem locking, Otherwise VM_LOCKED check makes
10075 + * unstable result and race. Plus, We can't wait here because
10076 + * we now hold anon_vma->rwsem or mapping->i_mmap_rwsem.
10077 + * if trylock failed, the page remain in evictable lru and later
10078 + * vmscan could retry to move the page to unevictable lru if the
10079 + * page is actually mlocked.
10080 + */
10081 + if (down_read_trylock(&vma->vm_mm->mmap_sem)) {
10082 + if (vma->vm_flags & VM_LOCKED) {
10083 + mlock_vma_page(page);
10084 + ret = SWAP_MLOCK;
10085 + }
10086 + up_read(&vma->vm_mm->mmap_sem);
10087 + }
10088 + return ret;
10089 +}
10090 +
10091 +int try_to_unmap_one_only(struct page *page, struct vm_area_struct *vma,
10092 + unsigned long address, void *arg)
10093 +{
10094 + struct mm_struct *mm = vma->vm_mm;
10095 + struct mm_struct *current_mm;
10096 +
10097 + rcu_read_lock();
10098 + get_task_struct(current);
10099 + rcu_read_unlock();
10100 + current_mm = get_task_mm(current);
10101 + put_task_struct(current);
10102 + if (!current_mm)
10103 + BUG();
10104 +
10105 + if (mm == current_mm) {
10106 + return try_to_unmap_one_entry(page, vma, address, arg);
10107 + }
10108 + return SWAP_AGAIN;
10109 +}
10110 +
10111 bool is_vma_temporary_stack(struct vm_area_struct *vma)
10112 {
10113 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
10114 diff --git mm/slab.h mm/slab.h
10115 index 4c3ac12..48be14c 100644
10116 --- mm/slab.h
10117 +++ mm/slab.h
10118 @@ -114,7 +114,7 @@ static inline unsigned long kmem_cache_flags(unsigned long object_size,
10119
10120 /* Legal flag mask for kmem_cache_create(), for various configurations */
10121 #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
10122 - SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )
10123 + SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS | SLAB_NO_MERGE)
10124
10125 #if defined(CONFIG_DEBUG_SLAB)
10126 #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
10127 diff --git mm/slab_common.c mm/slab_common.c
10128 index 999bb34..3f896c8 100644
10129 --- mm/slab_common.c
10130 +++ mm/slab_common.c
10131 @@ -35,7 +35,7 @@ struct kmem_cache *kmem_cache;
10132 */
10133 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
10134 SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \
10135 - SLAB_FAILSLAB)
10136 + SLAB_FAILSLAB | SLAB_NO_MERGE)
10137
10138 #define SLAB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
10139 SLAB_CACHE_DMA | SLAB_NOTRACK)
10140 @@ -703,7 +703,9 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
10141 panic("Out of memory when creating slab %s\n", name);
10142
10143 create_boot_cache(s, name, size, flags);
10144 +
10145 list_add(&s->list, &slab_caches);
10146 +
10147 s->refcount = 1;
10148 return s;
10149 }
10150 @@ -711,6 +713,11 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
10151 struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
10152 EXPORT_SYMBOL(kmalloc_caches);
10153
10154 +/* for per-cpu kmalloc objects */
10155 +struct kmem_cache *hc_kmalloc_caches[NR_CPUS][KMALLOC_SHIFT_HIGH + 1];
10156 +//struct kmem_cache *hc_kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
10157 +EXPORT_SYMBOL(hc_kmalloc_caches);
10158 +
10159 #ifdef CONFIG_ZONE_DMA
10160 struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
10161 EXPORT_SYMBOL(kmalloc_dma_caches);
10162 @@ -780,7 +787,14 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
10163 return kmalloc_dma_caches[index];
10164
10165 #endif
10166 - return kmalloc_caches[index];
10167 +
10168 + if (flags & GFP_COLOR) {
10169 + int cpu = raw_smp_processor_id();
10170 + TRACE("in kmalloc_slab index %d\n", index);
10171 + return hc_kmalloc_caches[cpu][index];
10172 + }
10173 + else
10174 + return kmalloc_caches[index];
10175 }
10176
10177 /*
10178 @@ -790,7 +804,7 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
10179 */
10180 void __init create_kmalloc_caches(unsigned long flags)
10181 {
10182 - int i;
10183 + int i, cpu = 0;
10184
10185 /*
10186 * Patch up the size_index table if we have strange large alignment
10187 @@ -844,11 +858,36 @@ void __init create_kmalloc_caches(unsigned long flags)
10188 * These have to be created immediately after the
10189 * earlier power of two caches
10190 */
10191 - if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6)
10192 + if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6) {
10193 kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
10194 + }
10195
10196 - if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7)
10197 + if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7) {
10198 kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
10199 + }
10200 + }
10201 +
10202 +/* per-cpu kmalloc caches */
10203 + for (cpu = 0; cpu < NR_CPUS; cpu++) {
10204 + for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
10205 + char *n;
10206 + n = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(i));
10207 +
10208 + hc_kmalloc_caches[cpu][i] = create_kmalloc_cache(n, 1 << i, SLAB_NO_MERGE|flags);
10209 + hc_kmalloc_caches[cpu][i]->cpu_id = cpu;
10210 +
10211 + if (KMALLOC_MIN_SIZE <= 32 && !hc_kmalloc_caches[cpu][1] && i == 6) {
10212 + char *nm;
10213 + nm = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(1));
10214 + hc_kmalloc_caches[cpu][1] = create_kmalloc_cache(nm, 96, SLAB_NO_MERGE|flags);
10215 + }
10216 +
10217 + if (KMALLOC_MIN_SIZE <= 64 && !hc_kmalloc_caches[cpu][2] && i == 7) {
10218 + char *nm;
10219 + nm = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(2));
10220 + hc_kmalloc_caches[cpu][2] = create_kmalloc_cache(nm, 192, SLAB_NO_MERGE|flags);
10221 + }
10222 + }
10223 }
10224
10225 /* Kmalloc array is now usable */
10226 diff --git mm/slub.c mm/slub.c
10227 index 54c0876..1fc9971 100644
10228 --- mm/slub.c
10229 +++ mm/slub.c
10230 @@ -115,6 +115,26 @@
10231 * the fast path and disables lockless freelists.
10232 */
10233
10234 +// This Address Decoding is used in imx6-sabredsd platform
10235 +#define BANK_MASK 0x38000000
10236 +#define BANK_SHIFT 27
10237 +
10238 +#define CACHE_MASK 0x0000f000
10239 +#define CACHE_SHIFT 12
10240 +#define MAX_COLOR_NODE 128
10241 +
10242 +/* Decoding page color, 0~15 */
10243 +static inline unsigned int page_color(struct page *page)
10244 +{
10245 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
10246 +}
10247 +
10248 +/* Decoding page bank number, 0~7 */
10249 +static inline unsigned int page_bank(struct page *page)
10250 +{
10251 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
10252 +}
10253 +
10254 static inline int kmem_cache_debug(struct kmem_cache *s)
10255 {
10256 #ifdef CONFIG_SLUB_DEBUG
10257 @@ -1320,8 +1340,13 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s,
10258 if (memcg_charge_slab(s, flags, order))
10259 return NULL;
10260
10261 - if (node == NUMA_NO_NODE)
10262 + if (node == NUMA_NO_NODE) {
10263 +#ifdef CONFIG_SCHED_DEBUG_TRACE
10264 +// if (flags&GFP_COLOR)
10265 +// printk(KERN_INFO "alloc_pages calls with GFP_COLOR order = %d\n", order);
10266 +#endif
10267 page = alloc_pages(flags, order);
10268 + }
10269 else
10270 page = alloc_pages_exact_node(node, flags, order);
10271
10272 @@ -1337,6 +1362,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
10273 struct kmem_cache_order_objects oo = s->oo;
10274 gfp_t alloc_gfp;
10275
10276 +#ifdef CONFIG_SCHED_DEBUG_TRACE
10277 +// if (flags&GFP_COLOR)
10278 +// printk(KERN_INFO "gfp_allowed_mask = %08x\n", gfp_allowed_mask);
10279 +#endif
10280 +
10281 flags &= gfp_allowed_mask;
10282
10283 if (flags & __GFP_WAIT)
10284 @@ -1350,6 +1380,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
10285 */
10286 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
10287
10288 +#ifdef CONFIG_SCHED_DEBUG_TRACE
10289 +// if (flags&__GFP_COLOR)
10290 +// printk(KERN_INFO "allocate_slab with GFP_COLOR alloc_gfp = %08x\n", alloc_gfp);
10291 +#endif
10292 +
10293 page = alloc_slab_page(s, alloc_gfp, node, oo);
10294 if (unlikely(!page)) {
10295 oo = s->min;
10296 @@ -1419,7 +1454,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
10297 }
10298
10299 page = allocate_slab(s,
10300 - flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
10301 + flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK | GFP_COLOR), node);
10302 if (!page)
10303 goto out;
10304
10305 @@ -2223,6 +2258,12 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
10306 return freelist;
10307
10308 page = new_slab(s, flags, node);
10309 +
10310 +#ifdef CONFIG_SCHED_DEBUG_TRACE
10311 +// if (flags&GFP_COLOR)
10312 +// printk(KERN_INFO "new_slab_objects(): gets page %p color %d, bank %d\n", page, page_color(page), page_bank(page));
10313 +#endif
10314 +
10315 if (page) {
10316 c = raw_cpu_ptr(s->cpu_slab);
10317 if (c->page)
10318 @@ -2309,6 +2350,11 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
10319 struct page *page;
10320 unsigned long flags;
10321
10322 +#ifdef CONFIG_SCHED_DEBUG_TRACE
10323 +// if (gfpflags&GFP_COLOR)
10324 +// printk(KERN_INFO "__slab_alloc slow_path\n");
10325 +#endif
10326 +
10327 local_irq_save(flags);
10328 #ifdef CONFIG_PREEMPT
10329 /*
10330 @@ -2319,6 +2365,11 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
10331 c = this_cpu_ptr(s->cpu_slab);
10332 #endif
10333
10334 +#ifdef CONFIG_SCHED_DEBUG_TRACE
10335 +// if (gfpflags&GFP_COLOR)
10336 +// printk(KERN_INFO "__slab_alloc : page %p, partial %p\n", c->page, c->partial);
10337 +#endif
10338 +
10339 page = c->page;
10340 if (!page)
10341 goto new_slab;
10342 @@ -2770,7 +2821,8 @@ EXPORT_SYMBOL(kmem_cache_free);
10343 * take the list_lock.
10344 */
10345 static int slub_min_order;
10346 -static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
10347 +//static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
10348 +static int slub_max_order = 2;
10349 static int slub_min_objects;
10350
10351 /*
10352 @@ -3307,14 +3359,29 @@ void *__kmalloc(size_t size, gfp_t flags)
10353 struct kmem_cache *s;
10354 void *ret;
10355
10356 +#ifdef CONFIG_SCHED_DEBUG_TRACE
10357 +// if (flags & GFP_COLOR)
10358 +// printk(KERN_INFO "kmalloc size %d\n", size);
10359 +#endif
10360 +
10361 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
10362 return kmalloc_large(size, flags);
10363
10364 s = kmalloc_slab(size, flags);
10365 +
10366 +#ifdef CONFIG_SCHED_DEBUG_TRACE
10367 +// if (flags & GFP_COLOR)
10368 +// printk(KERN_INFO "kmalloc_slab %p\n", s);
10369 +#endif
10370
10371 if (unlikely(ZERO_OR_NULL_PTR(s)))
10372 return s;
10373
10374 +#ifdef CONFIG_SCHED_DEBUG_TRACE
10375 +// if (flags & GFP_COLOR)
10376 +// printk(KERN_INFO "slab_alloc calls!!\n");
10377 +#endif
10378 +
10379 ret = slab_alloc(s, flags, _RET_IP_);
10380
10381 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
10382 @@ -5139,6 +5206,10 @@ static char *create_unique_id(struct kmem_cache *s)
10383 * are matched during merging to guarantee that the id is
10384 * unique.
10385 */
10386 + if (s->flags & SLAB_NO_MERGE) {
10387 + *p++ = 'n';
10388 + p += sprintf(p, "%01d", s->cpu_id);
10389 + }
10390 if (s->flags & SLAB_CACHE_DMA)
10391 *p++ = 'd';
10392 if (s->flags & SLAB_RECLAIM_ACCOUNT)
10393 diff --git mm/vmalloc.c mm/vmalloc.c
10394 index 2faaa29..5b1c46b 100644
10395 --- mm/vmalloc.c
10396 +++ mm/vmalloc.c
10397 @@ -1746,6 +1746,22 @@ void *vmalloc(unsigned long size)
10398 EXPORT_SYMBOL(vmalloc);
10399
10400 /**
10401 + * vmalloc_color - allocate virtually contiguous memory from a dedicated free_area
10402 + * @size: allocation size
10403 + * Allocate enough pages to cover @size from the page level
10404 + * allocator and map them into contiguous kernel virtual space.
10405 + *
10406 + * For tight control over page level allocator and protection flags
10407 + * use __vmalloc() instead.
10408 + */
10409 +void *vmalloc_color(unsigned long size)
10410 +{
10411 + return __vmalloc_node_flags(size, NUMA_NO_NODE,
10412 + GFP_KERNEL | __GFP_HIGHMEM | GFP_COLOR);
10413 +}
10414 +EXPORT_SYMBOL(vmalloc_color);
10415 +
10416 +/**
10417 * vzalloc - allocate virtually contiguous memory with zero fill
10418 * @size: allocation size
10419 * Allocate enough pages to cover @size from the page level
10420 @@ -1787,6 +1803,30 @@ void *vmalloc_user(unsigned long size)
10421 EXPORT_SYMBOL(vmalloc_user);
10422
10423 /**
10424 + * vmalloc_color_user - allocate zeroed virtually contiguous memory for userspace
10425 + * @size: allocation size
10426 + *
10427 + * The resulting memory area is zeroed so it can be mapped to userspace
10428 + * without leaking data.
10429 + */
10430 +void *vmalloc_color_user(unsigned long size)
10431 +{
10432 + struct vm_struct *area;
10433 + void *ret;
10434 +
10435 + ret = __vmalloc_node(size, SHMLBA,
10436 + GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | GFP_COLOR,
10437 + PAGE_KERNEL, NUMA_NO_NODE,
10438 + __builtin_return_address(0));
10439 + if (ret) {
10440 + area = find_vm_area(ret);
10441 + area->flags |= VM_USERMAP;
10442 + }
10443 + return ret;
10444 +}
10445 +EXPORT_SYMBOL(vmalloc_color_user);
10446 +
10447 +/**
10448 * vmalloc_node - allocate memory on a specific node
10449 * @size: allocation size
10450 * @node: numa node
10451 diff --git mm/vmstat.c mm/vmstat.c
10452 index 4f5cd97..5df6edb 100644
10453 --- mm/vmstat.c
10454 +++ mm/vmstat.c
10455 @@ -739,6 +739,7 @@ const char * const vmstat_text[] = {
10456 "workingset_nodereclaim",
10457 "nr_anon_transparent_hugepages",
10458 "nr_free_cma",
10459 + "nr_free_hc_pages",
10460
10461 /* enum writeback_stat_item counters */
10462 "nr_dirty_threshold",
10463 @@ -754,6 +755,7 @@ const char * const vmstat_text[] = {
10464 TEXTS_FOR_ZONES("pgalloc")
10465
10466 "pgfree",
10467 + "pgfree_hc",
10468 "pgactivate",
10469 "pgdeactivate",
10470
10471 @@ -956,6 +958,67 @@ static void pagetypeinfo_showfree_print(struct seq_file *m,
10472 }
10473 }
10474
10475 +static void pagetypeinfo_showpartitionfree_print(struct seq_file *m,
10476 + pg_data_t *pgdat, struct zone *zone, int cpu)
10477 +{
10478 + int order, mtype;
10479 +
10480 + for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) {
10481 + seq_printf(m, "Node %4d, zone %8s, type %12s ",
10482 + pgdat->node_id,
10483 + zone->name,
10484 + migratetype_names[mtype]);
10485 + for (order = 0; order < MAX_PARTITIONED_ORDER; ++order) {
10486 + unsigned long freecount = 0;
10487 + struct free_area *area;
10488 + struct list_head *curr;
10489 +
10490 + area = &(zone->free_area_d[cpu][order]);
10491 +
10492 + list_for_each(curr, &area->free_list[mtype])
10493 + freecount++;
10494 + seq_printf(m, "%6lu ", freecount);
10495 + }
10496 + seq_putc(m, '\n');
10497 + }
10498 +}
10499 +
10500 +static void walk_zones_in_node_in_partition(struct seq_file *m, pg_data_t *pgdat,
10501 + int cpu, void (*print)(struct seq_file *m, pg_data_t *, struct zone *, int))
10502 +{
10503 + struct zone *zone;
10504 + struct zone *node_zones = pgdat->node_zones;
10505 + unsigned long flags;
10506 +
10507 + for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
10508 + if (!populated_zone(zone))
10509 + continue;
10510 +
10511 + spin_lock_irqsave(&zone->lock, flags);
10512 + print(m, pgdat, zone, cpu);
10513 + spin_unlock_irqrestore(&zone->lock, flags);
10514 + }
10515 +}
10516 +
10517 +/* Print out the free pages at each order for each migatetype and partition */
10518 +static int pagetypeinfo_showpartitioned(struct seq_file *m, void *arg)
10519 +{
10520 + int order, cpu;
10521 + pg_data_t *pgdat = (pg_data_t *)arg;
10522 +
10523 + for_each_online_cpu(cpu) {
10524 + /* Print header */
10525 + seq_putc(m, '\n');
10526 + seq_printf(m, "CPU%d %-43s ", cpu, "free pages count per migrate type at order");
10527 + for (order = 0; order < MAX_PARTITIONED_ORDER; ++order)
10528 + seq_printf(m, "%6d ", order);
10529 + seq_putc(m, '\n');
10530 +
10531 + walk_zones_in_node_in_partition(m, pgdat, cpu, pagetypeinfo_showpartitionfree_print);
10532 + }
10533 + return 0;
10534 +}
10535 +
10536 /* Print out the free pages at each order for each migatetype */
10537 static int pagetypeinfo_showfree(struct seq_file *m, void *arg)
10538 {
10539 @@ -1138,7 +1201,7 @@ static int pagetypeinfo_show(struct seq_file *m, void *arg)
10540 pagetypeinfo_showfree(m, pgdat);
10541 pagetypeinfo_showblockcount(m, pgdat);
10542 pagetypeinfo_showmixedcount(m, pgdat);
10543 -
10544 + pagetypeinfo_showpartitioned(m, pgdat);
10545 return 0;
10546 }
10547
10548 @@ -1180,10 +1243,27 @@ static const struct file_operations pagetypeinfo_file_ops = {
10549 .release = seq_release,
10550 };
10551
10552 +#define BANK_MASK 0x38000000
10553 +#define BANK_SHIFT 27
10554 +#define CACHE_MASK 0x0000f000
10555 +#define CACHE_SHIFT 12
10556 +/* Decoding page bank number, 0~7 */
10557 +static inline unsigned int page_bank(struct page *page)
10558 +{
10559 + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT);
10560 +}
10561 +/* Decoding page color, 0~15 */
10562 +static inline unsigned int page_color(struct page *page)
10563 +{
10564 + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT);
10565 +}
10566 +
10567 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
10568 struct zone *zone)
10569 {
10570 int i;
10571 + int mtype;
10572 +
10573 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
10574 seq_printf(m,
10575 "\n pages free %lu"
10576 @@ -1232,6 +1312,16 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
10577 seq_printf(m, "\n vm stats threshold: %d",
10578 pageset->stat_threshold);
10579 #endif
10580 + /* pcp test */
10581 +/* seq_printf(m, "\n");
10582 + for (mtype = 0; mtype < MIGRATE_PCPTYPES; mtype++) {
10583 + struct page *p;
10584 + list_for_each_entry(p, &pageset->pcp.lists[mtype], lru) {
10585 + if (p)
10586 + seq_printf(m, "page bank = %d color = %d\n", page_bank(p), page_color(p));
10587 + }
10588 + }
10589 +*/
10590 }
10591 seq_printf(m,
10592 "\n all_unreclaimable: %u"
10593 diff --git net/core/dev.c net/core/dev.c
10594 index aa82f9a..f02d0c5 100644
10595 --- net/core/dev.c
10596 +++ net/core/dev.c
10597 @@ -4629,6 +4629,7 @@ static int napi_poll(struct napi_struct *n, struct list_head *repoll)
10598 */
10599 work = 0;
10600 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
10601 + // fec_enet_rx_napi() is called
10602 work = n->poll(n, weight);
10603 trace_napi_poll(n);
10604 }
10605 diff --git net/core/skbuff.c net/core/skbuff.c
10606 index 41ec022..5bf0a73 100644
10607 --- net/core/skbuff.c
10608 +++ net/core/skbuff.c
10609 @@ -77,6 +77,13 @@
10610 #include <linux/capability.h>
10611 #include <linux/user_namespace.h>
10612
10613 +#define USE_LEVEL_A_BANK 1
10614 +#ifdef USE_LEVEL_A_BANK
10615 +#define SKB_FLAG (GFP_COLOR)
10616 +#else
10617 +#define SKB_FLAG (0)
10618 +#endif
10619 +
10620 struct kmem_cache *skbuff_head_cache __read_mostly;
10621 static struct kmem_cache *skbuff_fclone_cache __read_mostly;
10622
10623 @@ -133,14 +140,14 @@ static void *__kmalloc_reserve(size_t size, gfp_t flags, int node,
10624 * to the reserves, fail.
10625 */
10626 obj = kmalloc_node_track_caller(size,
10627 - flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
10628 + flags | __GFP_NOMEMALLOC | __GFP_NOWARN | SKB_FLAG,
10629 node);
10630 if (obj || !(gfp_pfmemalloc_allowed(flags)))
10631 goto out;
10632
10633 /* Try again but now we are using pfmemalloc reserves */
10634 ret_pfmemalloc = true;
10635 - obj = kmalloc_node_track_caller(size, flags, node);
10636 + obj = kmalloc_node_track_caller(size, flags | SKB_FLAG, node);
10637
10638 out:
10639 if (pfmemalloc)
10640 @@ -161,7 +168,7 @@ struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node)
10641
10642 /* Get the HEAD */
10643 skb = kmem_cache_alloc_node(skbuff_head_cache,
10644 - gfp_mask & ~__GFP_DMA, node);
10645 + (gfp_mask & ~__GFP_DMA) | SKB_FLAG, node);
10646 if (!skb)
10647 goto out;
10648
10649 @@ -213,7 +220,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
10650 gfp_mask |= __GFP_MEMALLOC;
10651
10652 /* Get the HEAD */
10653 - skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
10654 + skb = kmem_cache_alloc_node(cache, (gfp_mask & ~__GFP_DMA) | SKB_FLAG, node);
10655 if (!skb)
10656 goto out;
10657 prefetchw(skb);
10658 @@ -225,7 +232,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
10659 */
10660 size = SKB_DATA_ALIGN(size);
10661 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
10662 - data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
10663 + data = kmalloc_reserve(size, gfp_mask | SKB_FLAG, node, &pfmemalloc);
10664 if (!data)
10665 goto nodata;
10666 /* kmalloc(size) might give us more room than requested.
10667 @@ -304,7 +311,7 @@ struct sk_buff *__build_skb(void *data, unsigned int frag_size)
10668 struct sk_buff *skb;
10669 unsigned int size = frag_size ? : ksize(data);
10670
10671 - skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
10672 + skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC | SKB_FLAG);
10673 if (!skb)
10674 return NULL;
10675
10676 @@ -367,12 +374,12 @@ static struct page *__page_frag_refill(struct netdev_alloc_cache *nc,
10677 if (order) {
10678 gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
10679 __GFP_NOMEMALLOC;
10680 - page = alloc_pages_node(NUMA_NO_NODE, gfp_mask, order);
10681 + page = alloc_pages_node(NUMA_NO_NODE, gfp_mask | SKB_FLAG, order);
10682 nc->frag.size = PAGE_SIZE << (page ? order : 0);
10683 }
10684
10685 if (unlikely(!page))
10686 - page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
10687 + page = alloc_pages_node(NUMA_NO_NODE, gfp | SKB_FLAG, 0);
10688
10689 nc->frag.page = page;
10690
10691 @@ -389,7 +396,7 @@ static void *__alloc_page_frag(struct netdev_alloc_cache __percpu *cache,
10692
10693 if (unlikely(!page)) {
10694 refill:
10695 - page = __page_frag_refill(nc, gfp_mask);
10696 + page = __page_frag_refill(nc, gfp_mask | SKB_FLAG);
10697 if (!page)
10698 return NULL;
10699
10700 @@ -434,7 +441,7 @@ static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
10701 void *data;
10702
10703 local_irq_save(flags);
10704 - data = __alloc_page_frag(&netdev_alloc_cache, fragsz, gfp_mask);
10705 + data = __alloc_page_frag(&netdev_alloc_cache, fragsz, gfp_mask | SKB_FLAG);
10706 local_irq_restore(flags);
10707 return data;
10708 }
10709 @@ -448,18 +455,18 @@ static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
10710 */
10711 void *netdev_alloc_frag(unsigned int fragsz)
10712 {
10713 - return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
10714 + return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD | SKB_FLAG);
10715 }
10716 EXPORT_SYMBOL(netdev_alloc_frag);
10717
10718 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
10719 {
10720 - return __alloc_page_frag(&napi_alloc_cache, fragsz, gfp_mask);
10721 + return __alloc_page_frag(&napi_alloc_cache, fragsz, gfp_mask | SKB_FLAG);
10722 }
10723
10724 void *napi_alloc_frag(unsigned int fragsz)
10725 {
10726 - return __napi_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD);
10727 + return __napi_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD | SKB_FLAG);
10728 }
10729 EXPORT_SYMBOL(napi_alloc_frag);
10730
10731 @@ -493,8 +500,8 @@ static struct sk_buff *__alloc_rx_skb(unsigned int length, gfp_t gfp_mask,
10732 gfp_mask |= __GFP_MEMALLOC;
10733
10734 data = (flags & SKB_ALLOC_NAPI) ?
10735 - __napi_alloc_frag(fragsz, gfp_mask) :
10736 - __netdev_alloc_frag(fragsz, gfp_mask);
10737 + __napi_alloc_frag(fragsz, gfp_mask | SKB_FLAG) :
10738 + __netdev_alloc_frag(fragsz, gfp_mask | SKB_FLAG);
10739
10740 if (likely(data)) {
10741 skb = build_skb(data, fragsz);
10742 @@ -502,7 +509,7 @@ static struct sk_buff *__alloc_rx_skb(unsigned int length, gfp_t gfp_mask,
10743 put_page(virt_to_head_page(data));
10744 }
10745 } else {
10746 - skb = __alloc_skb(length, gfp_mask,
10747 + skb = __alloc_skb(length, gfp_mask | SKB_FLAG,
10748 SKB_ALLOC_RX, NUMA_NO_NODE);
10749 }
10750 return skb;
10751 @@ -527,7 +534,8 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
10752 struct sk_buff *skb;
10753
10754 length += NET_SKB_PAD;
10755 - skb = __alloc_rx_skb(length, gfp_mask, 0);
10756 +
10757 + skb = __alloc_rx_skb(length, gfp_mask | SKB_FLAG, 0);
10758
10759 if (likely(skb)) {
10760 skb_reserve(skb, NET_SKB_PAD);
10761 @@ -557,7 +565,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
10762 struct sk_buff *skb;
10763
10764 length += NET_SKB_PAD + NET_IP_ALIGN;
10765 - skb = __alloc_rx_skb(length, gfp_mask, SKB_ALLOC_NAPI);
10766 + skb = __alloc_rx_skb(length, gfp_mask | SKB_FLAG, SKB_ALLOC_NAPI);
10767
10768 if (likely(skb)) {
10769 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN);
10770 @@ -932,7 +940,7 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask)
10771 u8 *vaddr;
10772 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
10773
10774 - page = alloc_page(gfp_mask);
10775 + page = alloc_page(gfp_mask | SKB_FLAG);
10776 if (!page) {
10777 while (head) {
10778 struct page *next = (struct page *)page_private(head);
10779 @@ -988,7 +996,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
10780 skb1);
10781 struct sk_buff *n;
10782
10783 - if (skb_orphan_frags(skb, gfp_mask))
10784 + if (skb_orphan_frags(skb, gfp_mask | SKB_FLAG))
10785 return NULL;
10786
10787 if (skb->fclone == SKB_FCLONE_ORIG &&
10788 @@ -999,7 +1007,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
10789 if (skb_pfmemalloc(skb))
10790 gfp_mask |= __GFP_MEMALLOC;
10791
10792 - n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
10793 + n = kmem_cache_alloc(skbuff_head_cache, gfp_mask | SKB_FLAG);
10794 if (!n)
10795 return NULL;
10796
10797 @@ -1063,7 +1071,7 @@ struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
10798 {
10799 int headerlen = skb_headroom(skb);
10800 unsigned int size = skb_end_offset(skb) + skb->data_len;
10801 - struct sk_buff *n = __alloc_skb(size, gfp_mask,
10802 + struct sk_buff *n = __alloc_skb(size, gfp_mask | SKB_FLAG,
10803 skb_alloc_rx_flag(skb), NUMA_NO_NODE);
10804
10805 if (!n)
10806 @@ -1104,7 +1112,7 @@ struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
10807 {
10808 unsigned int size = skb_headlen(skb) + headroom;
10809 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0);
10810 - struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE);
10811 + struct sk_buff *n = __alloc_skb(size, gfp_mask | SKB_FLAG, flags, NUMA_NO_NODE);
10812
10813 if (!n)
10814 goto out;
10815 @@ -1123,7 +1131,7 @@ struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom,
10816 if (skb_shinfo(skb)->nr_frags) {
10817 int i;
10818
10819 - if (skb_orphan_frags(skb, gfp_mask)) {
10820 + if (skb_orphan_frags(skb, gfp_mask | SKB_FLAG)) {
10821 kfree_skb(n);
10822 n = NULL;
10823 goto out;
10824 @@ -1180,7 +1188,7 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
10825 if (skb_pfmemalloc(skb))
10826 gfp_mask |= __GFP_MEMALLOC;
10827 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
10828 - gfp_mask, NUMA_NO_NODE, NULL);
10829 + gfp_mask | SKB_FLAG, NUMA_NO_NODE, NULL);
10830 if (!data)
10831 goto nodata;
10832 size = SKB_WITH_OVERHEAD(ksize(data));
10833 @@ -1201,7 +1209,7 @@ int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
10834 */
10835 if (skb_cloned(skb)) {
10836 /* copy this zero copy skb frags */
10837 - if (skb_orphan_frags(skb, gfp_mask))
10838 + if (skb_orphan_frags(skb, gfp_mask | SKB_FLAG))
10839 goto nofrags;
10840 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
10841 skb_frag_ref(skb, i);
10842 @@ -1286,7 +1294,7 @@ struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
10843 * Allocate the copy buffer
10844 */
10845 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom,
10846 - gfp_mask, skb_alloc_rx_flag(skb),
10847 + gfp_mask | SKB_FLAG, skb_alloc_rx_flag(skb),
10848 NUMA_NO_NODE);
10849 int oldheadroom = skb_headroom(skb);
10850 int head_copy_len, head_copy_off;
10851 @@ -4387,7 +4395,7 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
10852 gfp_head |= __GFP_REPEAT;
10853
10854 *errcode = -ENOBUFS;
10855 - skb = alloc_skb(header_len, gfp_head);
10856 + skb = alloc_skb(header_len, gfp_head | SKB_FLAG);
10857 if (!skb)
10858 return NULL;
10859
10860 @@ -4401,7 +4409,7 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
10861 page = alloc_pages((gfp_mask & ~__GFP_WAIT) |
10862 __GFP_COMP |
10863 __GFP_NOWARN |
10864 - __GFP_NORETRY,
10865 + __GFP_NORETRY | SKB_FLAG,
10866 order);
10867 if (page)
10868 goto fill_page;
10869 @@ -4411,7 +4419,7 @@ struct sk_buff *alloc_skb_with_frags(unsigned long header_len,
10870 }
10871 order--;
10872 }
10873 - page = alloc_page(gfp_mask);
10874 + page = alloc_page(gfp_mask | SKB_FLAG);
10875 if (!page)
10876 goto failure;
10877 fill_page:
Attached Files
To refer to attachments on a page, use attachment:filename, as shown below in the list of files. Do NOT use the URL of the [get] link, since this is subject to change and can break easily.You are not allowed to attach a file to this page.