diff --git arch/arm/boot/compressed/Makefile arch/arm/boot/compressed/Makefile index 6e1fb2b..e2284fe 100644 --- arch/arm/boot/compressed/Makefile +++ arch/arm/boot/compressed/Makefile @@ -107,6 +107,7 @@ ifeq ($(CONFIG_FUNCTION_TRACER),y) ORIG_CFLAGS := $(KBUILD_CFLAGS) KBUILD_CFLAGS = $(subst -pg, , $(ORIG_CFLAGS)) endif +KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING ccflags-y := -fpic -mno-single-pic-base -fno-builtin -I$(obj) asflags-y := -DZIMAGE diff --git arch/arm/include/asm/unistd.h arch/arm/include/asm/unistd.h index 0c462a9..7197bbe 100644 --- arch/arm/include/asm/unistd.h +++ arch/arm/include/asm/unistd.h @@ -19,7 +19,8 @@ * This may need to be greater than __NR_last_syscall+1 in order to * account for the padding in the syscall table */ -#define __NR_syscalls (388 + NR_litmus_syscalls) +#define __NR_syscalls (388 + NR_litmus_syscalls + 3) + /* * *NOTE*: This is a ghost syscall private to the kernel. Only the diff --git arch/arm/kernel/calls.S arch/arm/kernel/calls.S index f4738a8..04cc83b 100644 --- arch/arm/kernel/calls.S +++ arch/arm/kernel/calls.S @@ -409,6 +409,15 @@ CALL(sys_wait_for_ts_release) CALL(sys_release_ts) CALL(sys_null_call) +/* 400 */ CALL(sys_get_current_budget) + CALL(sys_reservation_create) + CALL(sys_reservation_destroy) + CALL(sys_set_mc2_task_param) + CALL(sys_set_page_color) +/* 405 */ CALL(sys_test_call) + CALL(sys_run_test) + CALL(sys_lock_buffer) + CALL(sys_recolor_mem) #ifndef syscalls_counted .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls diff --git arch/arm/kernel/irq.c arch/arm/kernel/irq.c index 350f188..a9ba6e5 100644 --- arch/arm/kernel/irq.c +++ arch/arm/kernel/irq.c @@ -44,6 +44,9 @@ #include #include +#include +#include + unsigned long irq_err_count; int arch_show_interrupts(struct seq_file *p, int prec) @@ -66,7 +69,9 @@ int arch_show_interrupts(struct seq_file *p, int prec) */ void handle_IRQ(unsigned int irq, struct pt_regs *regs) { + enter_irq_mode(); __handle_domain_irq(NULL, irq, false, regs); + exit_irq_mode(); } /* diff --git arch/arm/mm/cache-l2x0.c arch/arm/mm/cache-l2x0.c index e309c8f..969da4a 100644 --- arch/arm/mm/cache-l2x0.c +++ arch/arm/mm/cache-l2x0.c @@ -33,6 +33,8 @@ #include "cache-tauros3.h" #include "cache-aurora-l2.h" +#include + struct l2c_init_data { const char *type; unsigned way_size_0; @@ -651,6 +653,11 @@ static void __init l2c310_enable(void __iomem *base, u32 aux, unsigned num_lock) */ aux |= L310_AUX_CTRL_NS_LOCKDOWN; + /* + * Always enable non-secure interrupt access control registers + */ + aux |= L220_AUX_CTRL_NS_INT_CTRL; + l2c_enable(base, aux, num_lock); /* Read back resulting AUX_CTRL value as it could have been altered. */ @@ -726,7 +733,6 @@ static void __init l2c310_fixup(void __iomem *base, u32 cache_id, if (n) { unsigned i; - pr_info("L2C-310 errat%s", n > 1 ? "a" : "um"); for (i = 0; i < n; i++) pr_cont(" %s", errata[i]); @@ -774,6 +780,11 @@ static const struct l2c_init_data l2c310_init_fns __initconst = { }, }; +void l2c310_flush_all(void) +{ + l2c210_flush_all(); +}; + static int __init __l2c_init(const struct l2c_init_data *data, u32 aux_val, u32 aux_mask, u32 cache_id) { @@ -876,6 +887,8 @@ static int __init __l2c_init(const struct l2c_init_data *data, pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n", data->type, cache_id, aux); + litmus_setup_lockdown(l2x0_base, cache_id); + return 0; } diff --git arch/x86/syscalls/syscall_32.tbl arch/x86/syscalls/syscall_32.tbl index 34680a5..6b80d98 100644 --- arch/x86/syscalls/syscall_32.tbl +++ arch/x86/syscalls/syscall_32.tbl @@ -377,3 +377,12 @@ 368 i386 wait_for_ts_release sys_wait_for_ts_release 369 i386 release_ts sys_release_ts 370 i386 null_call sys_null_call +371 i386 get_current_budget sys_get_current_budget +372 i386 reservation_create sys_reservation_create +373 i386 reservation_destroy sys_reservation_destroy +374 i386 set_mc2_task_param sys_set_mc2_task_param +375 i386 set_page_color sys_set_page_color +376 i386 test_call sys_test_call +377 i386 run_test sys_run_test +378 i386 lock_buffer sys_lock_buffer +379 i386 recolor_mem sys_recolor_mem diff --git arch/x86/syscalls/syscall_64.tbl arch/x86/syscalls/syscall_64.tbl index cbd1b6b..05aa913 100644 --- arch/x86/syscalls/syscall_64.tbl +++ arch/x86/syscalls/syscall_64.tbl @@ -342,6 +342,15 @@ 360 common wait_for_ts_release sys_wait_for_ts_release 361 common release_ts sys_release_ts 362 common null_call sys_null_call +363 common get_current_budget sys_get_current_budget +364 common reservation_create sys_reservation_create +365 common reservation_destroy sys_reservation_destroy +366 common set_mc2_task_param sys_set_mc2_task_param +367 common set_page_color sys_set_page_color +368 common test_call sys_test_call +369 common run_test sys_run_test +370 common lock_buffer sys_lock_buffer +371 common recolor_mem sys_recolor_mem # # x32-specific system call numbers start at 512 to avoid cache impact diff --git drivers/media/usb/uvc/uvc_video.c drivers/media/usb/uvc/uvc_video.c index 20ccc9d..9dd652c 100644 --- drivers/media/usb/uvc/uvc_video.c +++ drivers/media/usb/uvc/uvc_video.c @@ -26,6 +26,16 @@ #include "uvcvideo.h" +#include +#include + +// default behavior is using Level-C/OS +#ifdef CONFIG_UVC_ALLOC_LEVEL_AB +#define UVC_FLAG (GFP_COLOR) +#else +#define UVC_FLAG (0) +#endif + /* ------------------------------------------------------------------------ * UVC Controls */ @@ -167,7 +177,7 @@ static int uvc_get_video_ctrl(struct uvc_streaming *stream, query == UVC_GET_DEF) return -EIO; - data = kmalloc(size, GFP_KERNEL); + data = kmalloc(size, GFP_KERNEL|UVC_FLAG); if (data == NULL) return -ENOMEM; @@ -251,7 +261,7 @@ static int uvc_set_video_ctrl(struct uvc_streaming *stream, int ret; size = stream->dev->uvc_version >= 0x0110 ? 34 : 26; - data = kzalloc(size, GFP_KERNEL); + data = kzalloc(size, GFP_KERNEL|UVC_FLAG); if (data == NULL) return -ENOMEM; @@ -494,7 +504,7 @@ static int uvc_video_clock_init(struct uvc_streaming *stream) clock->size = 32; clock->samples = kmalloc(clock->size * sizeof(*clock->samples), - GFP_KERNEL); + GFP_KERNEL|UVC_FLAG); if (clock->samples == NULL) return -ENOMEM; @@ -1341,9 +1351,9 @@ static void uvc_video_complete(struct urb *urb) queue); spin_unlock_irqrestore(&queue->irqlock, flags); - stream->decode(urb, stream, buf); + stream->decode(urb, stream, buf); //uvc_video_decode_isoc() - if ((ret = usb_submit_urb(urb, GFP_ATOMIC)) < 0) { + if ((ret = usb_submit_urb(urb, GFP_ATOMIC|UVC_FLAG)) < 0) { uvc_printk(KERN_ERR, "Failed to resubmit video URB (%d).\n", ret); } @@ -1406,10 +1416,10 @@ static int uvc_alloc_urb_buffers(struct uvc_streaming *stream, #ifndef CONFIG_DMA_NONCOHERENT stream->urb_buffer[i] = usb_alloc_coherent( stream->dev->udev, stream->urb_size, - gfp_flags | __GFP_NOWARN, &stream->urb_dma[i]); + gfp_flags | __GFP_NOWARN | UVC_FLAG, &stream->urb_dma[i]); #else stream->urb_buffer[i] = - kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN); + kmalloc(stream->urb_size, gfp_flags | __GFP_NOWARN | UVC_FLAG; #endif if (!stream->urb_buffer[i]) { uvc_free_urb_buffers(stream); @@ -1492,14 +1502,14 @@ static int uvc_init_video_isoc(struct uvc_streaming *stream, psize = uvc_endpoint_max_bpi(stream->dev->udev, ep); size = stream->ctrl.dwMaxVideoFrameSize; - npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags); + npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags|UVC_FLAG); if (npackets == 0) return -ENOMEM; size = npackets * psize; for (i = 0; i < UVC_URBS; ++i) { - urb = usb_alloc_urb(npackets, gfp_flags); + urb = usb_alloc_urb(npackets, gfp_flags|UVC_FLAG); if (urb == NULL) { uvc_uninit_video(stream, 1); return -ENOMEM; @@ -1548,7 +1558,7 @@ static int uvc_init_video_bulk(struct uvc_streaming *stream, size = stream->ctrl.dwMaxPayloadTransferSize; stream->bulk.max_payload_size = size; - npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags); + npackets = uvc_alloc_urb_buffers(stream, size, psize, gfp_flags|UVC_FLAG); if (npackets == 0) return -ENOMEM; @@ -1565,7 +1575,7 @@ static int uvc_init_video_bulk(struct uvc_streaming *stream, size = 0; for (i = 0; i < UVC_URBS; ++i) { - urb = usb_alloc_urb(0, gfp_flags); + urb = usb_alloc_urb(0, gfp_flags|UVC_FLAG); if (urb == NULL) { uvc_uninit_video(stream, 1); return -ENOMEM; @@ -1649,12 +1659,11 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags) uvc_trace(UVC_TRACE_VIDEO, "Selecting alternate setting %u " "(%u B/frame bandwidth).\n", altsetting, best_psize); - ret = usb_set_interface(stream->dev->udev, intfnum, altsetting); if (ret < 0) return ret; - ret = uvc_init_video_isoc(stream, best_ep, gfp_flags); + ret = uvc_init_video_isoc(stream, best_ep, gfp_flags|UVC_FLAG); } else { /* Bulk endpoint, proceed to URB initialization. */ ep = uvc_find_endpoint(&intf->altsetting[0], @@ -1662,7 +1671,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags) if (ep == NULL) return -EIO; - ret = uvc_init_video_bulk(stream, ep, gfp_flags); + ret = uvc_init_video_bulk(stream, ep, gfp_flags|UVC_FLAG); } if (ret < 0) @@ -1670,7 +1679,7 @@ static int uvc_init_video(struct uvc_streaming *stream, gfp_t gfp_flags) /* Submit the URBs. */ for (i = 0; i < UVC_URBS; ++i) { - ret = usb_submit_urb(stream->urb[i], gfp_flags); + ret = usb_submit_urb(stream->urb[i], gfp_flags|UVC_FLAG); if (ret < 0) { uvc_printk(KERN_ERR, "Failed to submit URB %u " "(%d).\n", i, ret); @@ -1741,7 +1750,7 @@ int uvc_video_resume(struct uvc_streaming *stream, int reset) if (ret < 0) return ret; - return uvc_init_video(stream, GFP_NOIO); + return uvc_init_video(stream, GFP_NOIO|UVC_FLAG); } /* ------------------------------------------------------------------------ @@ -1892,7 +1901,7 @@ int uvc_video_enable(struct uvc_streaming *stream, int enable) if (ret < 0) goto error_commit; - ret = uvc_init_video(stream, GFP_KERNEL); + ret = uvc_init_video(stream, GFP_KERNEL|UVC_FLAG); if (ret < 0) goto error_video; diff --git drivers/media/v4l2-core/videobuf2-core.c drivers/media/v4l2-core/videobuf2-core.c index 66ada01..0a5b616 100644 --- drivers/media/v4l2-core/videobuf2-core.c +++ drivers/media/v4l2-core/videobuf2-core.c @@ -30,6 +30,12 @@ #include #include +#ifdef UVC_ALLOC_LEVEL_AB +#define VB2_CORE_FLAG (GFP_COLOR) +#else +#define VB2_CORE_FLAG (0) +#endif + static int debug; module_param(debug, int, 0644); @@ -200,9 +206,8 @@ static int __vb2_buf_mem_alloc(struct vb2_buffer *vb) */ for (plane = 0; plane < vb->num_planes; ++plane) { unsigned long size = PAGE_ALIGN(q->plane_sizes[plane]); - mem_priv = call_ptr_memop(vb, alloc, q->alloc_ctx[plane], - size, dma_dir, q->gfp_flags); + size, dma_dir, q->gfp_flags|VB2_CORE_FLAG); if (IS_ERR_OR_NULL(mem_priv)) goto free; @@ -352,7 +357,7 @@ static int __vb2_queue_alloc(struct vb2_queue *q, enum v4l2_memory memory, for (buffer = 0; buffer < num_buffers; ++buffer) { /* Allocate videobuf buffer structures */ - vb = kzalloc(q->buf_struct_size, GFP_KERNEL); + vb = kzalloc(q->buf_struct_size, GFP_KERNEL|VB2_CORE_FLAG); if (!vb) { dprintk(1, "memory alloc for buffer struct failed\n"); break; @@ -2830,7 +2835,7 @@ static int __vb2_init_fileio(struct vb2_queue *q, int read) (read) ? "read" : "write", count, q->fileio_read_once, q->fileio_write_immediately); - fileio = kzalloc(sizeof(struct vb2_fileio_data), GFP_KERNEL); + fileio = kzalloc(sizeof(struct vb2_fileio_data), GFP_KERNEL|VB2_CORE_FLAG); if (fileio == NULL) return -ENOMEM; @@ -3223,7 +3228,7 @@ int vb2_thread_start(struct vb2_queue *q, vb2_thread_fnc fnc, void *priv, if (WARN_ON(q->fileio)) return -EBUSY; - threadio = kzalloc(sizeof(*threadio), GFP_KERNEL); + threadio = kzalloc(sizeof(*threadio), GFP_KERNEL|VB2_CORE_FLAG); if (threadio == NULL) return -ENOMEM; threadio->fnc = fnc; diff --git drivers/media/v4l2-core/videobuf2-vmalloc.c drivers/media/v4l2-core/videobuf2-vmalloc.c index 657ab30..dab3772 100644 --- drivers/media/v4l2-core/videobuf2-vmalloc.c +++ drivers/media/v4l2-core/videobuf2-vmalloc.c @@ -21,6 +21,12 @@ #include #include +#ifdef UVC_ALLOC_LEVEL_AB +#define VB2_FLAG (GFP_COLOR) +#else +#define VB2_FLAG (0) +#endif + struct vb2_vmalloc_buf { void *vaddr; struct page **pages; @@ -40,12 +46,16 @@ static void *vb2_vmalloc_alloc(void *alloc_ctx, unsigned long size, { struct vb2_vmalloc_buf *buf; - buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags); + buf = kzalloc(sizeof(*buf), GFP_KERNEL | gfp_flags | VB2_FLAG); if (!buf) return NULL; buf->size = size; +#ifdef UVC_ALLOC_LEVEL_AB + buf->vaddr = vmalloc_color_user(buf->size); +#else buf->vaddr = vmalloc_user(buf->size); +#endif buf->dma_dir = dma_dir; buf->handler.refcount = &buf->refcount; buf->handler.put = vb2_vmalloc_put; @@ -81,7 +91,7 @@ static void *vb2_vmalloc_get_userptr(void *alloc_ctx, unsigned long vaddr, struct vm_area_struct *vma; dma_addr_t physp; - buf = kzalloc(sizeof(*buf), GFP_KERNEL); + buf = kzalloc(sizeof(*buf), GFP_KERNEL | VB2_FLAG); if (!buf) return NULL; @@ -103,7 +113,7 @@ static void *vb2_vmalloc_get_userptr(void *alloc_ctx, unsigned long vaddr, last = (vaddr + size - 1) >> PAGE_SHIFT; buf->n_pages = last - first + 1; buf->pages = kzalloc(buf->n_pages * sizeof(struct page *), - GFP_KERNEL); + GFP_KERNEL | VB2_FLAG); if (!buf->pages) goto fail_pages_array_alloc; @@ -233,12 +243,12 @@ static int vb2_vmalloc_dmabuf_ops_attach(struct dma_buf *dbuf, struct device *de int ret; int i; - attach = kzalloc(sizeof(*attach), GFP_KERNEL); + attach = kzalloc(sizeof(*attach), GFP_KERNEL | VB2_FLAG); if (!attach) return -ENOMEM; sgt = &attach->sgt; - ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL); + ret = sg_alloc_table(sgt, num_pages, GFP_KERNEL | VB2_FLAG); if (ret) { kfree(attach); return ret; @@ -429,7 +439,7 @@ static void *vb2_vmalloc_attach_dmabuf(void *alloc_ctx, struct dma_buf *dbuf, if (dbuf->size < size) return ERR_PTR(-EFAULT); - buf = kzalloc(sizeof(*buf), GFP_KERNEL); + buf = kzalloc(sizeof(*buf), GFP_KERNEL | VB2_FLAG); if (!buf) return ERR_PTR(-ENOMEM); diff --git drivers/net/ethernet/freescale/fec_main.c drivers/net/ethernet/freescale/fec_main.c index 66d47e4..7dd6f26 100644 --- drivers/net/ethernet/freescale/fec_main.c +++ drivers/net/ethernet/freescale/fec_main.c @@ -60,6 +60,7 @@ #include #include +#include #include "fec.h" @@ -1587,6 +1588,8 @@ fec_enet_interrupt(int irq, void *dev_id) writel(int_events, fep->hwp + FEC_IEVENT); fec_enet_collect_events(fep, int_events); + enter_irq_mode(); + if ((fep->work_tx || fep->work_rx) && fep->link) { ret = IRQ_HANDLED; @@ -1605,6 +1608,7 @@ fec_enet_interrupt(int irq, void *dev_id) if (fep->ptp_clock) fec_ptp_check_pps_event(fep); + exit_irq_mode(); return ret; } diff --git include/linux/gfp.h include/linux/gfp.h index 15928f0..92643b8 100644 --- include/linux/gfp.h +++ include/linux/gfp.h @@ -35,6 +35,7 @@ struct vm_area_struct; #define ___GFP_NO_KSWAPD 0x400000u #define ___GFP_OTHER_NODE 0x800000u #define ___GFP_WRITE 0x1000000u +#define ___GFP_COLOR 0x2000000u /* If the above are modified, __GFP_BITS_SHIFT may need updating */ /* @@ -94,6 +95,7 @@ struct vm_area_struct; #define __GFP_NO_KSWAPD ((__force gfp_t)___GFP_NO_KSWAPD) #define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */ #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) /* Allocator intends to dirty page */ +#define __GFP_COLOR ((__force gfp_t)___GFP_COLOR) /* Colored page request */ /* * This may seem redundant, but it's a way of annotating false positives vs. @@ -101,7 +103,7 @@ struct vm_area_struct; */ #define __GFP_NOTRACK_FALSE_POSITIVE (__GFP_NOTRACK) -#define __GFP_BITS_SHIFT 25 /* Room for N __GFP_FOO bits */ +#define __GFP_BITS_SHIFT 26 /* Room for N __GFP_FOO bits */ #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /* This equals 0, but use constants in case they ever change */ @@ -127,7 +129,7 @@ struct vm_area_struct; /* Control page allocator reclaim behavior */ #define GFP_RECLAIM_MASK (__GFP_WAIT|__GFP_HIGH|__GFP_IO|__GFP_FS|\ __GFP_NOWARN|__GFP_REPEAT|__GFP_NOFAIL|\ - __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC) + __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|__GFP_COLOR) /* Control slab gfp mask during early boot */ #define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_WAIT|__GFP_IO|__GFP_FS)) @@ -146,6 +148,9 @@ struct vm_area_struct; /* 4GB DMA on some platforms */ #define GFP_DMA32 __GFP_DMA32 +/* Colored page requests */ +#define GFP_COLOR __GFP_COLOR + /* Convert GFP flags to their corresponding migrate type */ static inline int gfpflags_to_migratetype(const gfp_t gfp_flags) { diff --git include/linux/migrate.h include/linux/migrate.h index cac1c09..b16047b 100644 --- include/linux/migrate.h +++ include/linux/migrate.h @@ -33,6 +33,8 @@ extern int migrate_page(struct address_space *, struct page *, struct page *, enum migrate_mode); extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free, unsigned long private, enum migrate_mode mode, int reason); +extern int replicate_pages(struct list_head *l, new_page_t new, free_page_t free, + unsigned long private, enum migrate_mode mode, int reason); extern int migrate_prep(void); extern int migrate_prep_local(void); @@ -50,7 +52,11 @@ static inline int migrate_pages(struct list_head *l, new_page_t new, free_page_t free, unsigned long private, enum migrate_mode mode, int reason) { return -ENOSYS; } - +static inline int replicate_pages(struct list_head *l, new_page_t new, + free_page_t free, unsigned long private, enum migrate_mode mode, + int reason) + { return -ENOSYS; } + static inline int migrate_prep(void) { return -ENOSYS; } static inline int migrate_prep_local(void) { return -ENOSYS; } diff --git include/linux/mmzone.h include/linux/mmzone.h index 54d74f6..3d3f9ac 100644 --- include/linux/mmzone.h +++ include/linux/mmzone.h @@ -35,6 +35,16 @@ */ #define PAGE_ALLOC_COSTLY_ORDER 3 +/* For page coloring - This address decoding is used in imx6-sabresd + * platform without bank interleaving . + */ +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 +#define MAX_PARTITIONED_ORDER 11 +#define MAX_CONTIG_ORDER 11 + enum { MIGRATE_UNMOVABLE, MIGRATE_RECLAIMABLE, @@ -157,6 +167,7 @@ enum zone_stat_item { WORKINGSET_NODERECLAIM, NR_ANON_TRANSPARENT_HUGEPAGES, NR_FREE_CMA_PAGES, + NR_FREE_HC_PAGES, NR_VM_ZONE_STAT_ITEMS }; /* @@ -476,7 +487,8 @@ struct zone { ZONE_PADDING(_pad1_) /* free areas of different sizes */ struct free_area free_area[MAX_ORDER]; - + struct free_area free_area_d[NR_CPUS][MAX_PARTITIONED_ORDER]; + /* zone flags, see below */ unsigned long flags; @@ -523,7 +535,7 @@ struct zone { /* Set to true when the PG_migrate_skip bits should be cleared */ bool compact_blockskip_flush; #endif - + ZONE_PADDING(_pad3_) /* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; diff --git include/linux/msg.h include/linux/msg.h index f3f302f..9bf01c9 100644 --- include/linux/msg.h +++ include/linux/msg.h @@ -12,6 +12,8 @@ struct msg_msg { struct msg_msgseg *next; void *security; /* the actual message follows immediately */ + dma_addr_t handle; + size_t alloc_len; }; /* one msq_queue structure for each present queue on the system */ diff --git include/linux/rmap.h include/linux/rmap.h index c89c53a..7c90e02 100644 --- include/linux/rmap.h +++ include/linux/rmap.h @@ -188,7 +188,8 @@ int page_referenced(struct page *, int is_locked, #define TTU_ACTION(x) ((x) & TTU_ACTION_MASK) int try_to_unmap(struct page *, enum ttu_flags flags); - +int try_to_unmap_one_only(struct page *page, struct vm_area_struct *vma, + unsigned long address, void *arg); /* * Used by uprobes to replace a userspace page safely */ diff --git include/linux/slab.h include/linux/slab.h index ffd24c8..a899dda 100644 --- include/linux/slab.h +++ include/linux/slab.h @@ -87,6 +87,8 @@ # define SLAB_FAILSLAB 0x00000000UL #endif +#define SLAB_NO_MERGE 0x04000000UL /* Do not merge with existing slab */ + /* The following flags affect the page allocator grouping pages by mobility */ #define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */ #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ diff --git include/linux/slub_def.h include/linux/slub_def.h index 3388511..9400aa1 100644 --- include/linux/slub_def.h +++ include/linux/slub_def.h @@ -98,6 +98,8 @@ struct kmem_cache { */ int remote_node_defrag_ratio; #endif + /* cpu id for higher-criticality slabs */ + int cpu_id; struct kmem_cache_node *node[MAX_NUMNODES]; }; diff --git include/linux/vm_event_item.h include/linux/vm_event_item.h index 9246d32..3f5a9da 100644 --- include/linux/vm_event_item.h +++ include/linux/vm_event_item.h @@ -23,7 +23,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, FOR_ALL_ZONES(PGALLOC), - PGFREE, PGACTIVATE, PGDEACTIVATE, + PGFREE, PGFREE_HC, PGACTIVATE, PGDEACTIVATE, PGFAULT, PGMAJFAULT, FOR_ALL_ZONES(PGREFILL), FOR_ALL_ZONES(PGSTEAL_KSWAPD), diff --git include/linux/vmalloc.h include/linux/vmalloc.h index 0ec5983..c647658 100644 --- include/linux/vmalloc.h +++ include/linux/vmalloc.h @@ -67,8 +67,10 @@ static inline void vmalloc_init(void) #endif extern void *vmalloc(unsigned long size); +extern void *vmalloc_color(unsigned long size); extern void *vzalloc(unsigned long size); extern void *vmalloc_user(unsigned long size); +extern void *vmalloc_color_user(unsigned long size); extern void *vmalloc_node(unsigned long size, int node); extern void *vzalloc_node(unsigned long size, int node); extern void *vmalloc_exec(unsigned long size); diff --git include/linux/vmstat.h include/linux/vmstat.h index 82e7db7..b6410f7 100644 --- include/linux/vmstat.h +++ include/linux/vmstat.h @@ -278,9 +278,12 @@ static inline void drain_zonestat(struct zone *zone, #endif /* CONFIG_SMP */ static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages, - int migratetype) + int migratetype, int part_no) { - __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages); + if (part_no == NR_CPUS) + __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages); + else + __mod_zone_page_state(zone, NR_FREE_HC_PAGES, nr_pages); if (is_migrate_cma(migratetype)) __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages); } diff --git include/litmus/budget.h include/litmus/budget.h index bd2d5c9..60eb814 100644 --- include/litmus/budget.h +++ include/litmus/budget.h @@ -33,4 +33,6 @@ static inline int requeue_preempted_job(struct task_struct* t) (!budget_exhausted(t) || !budget_enforced(t)); } +void litmus_current_budget(lt_t *used_so_far, lt_t *remaining); + #endif diff --git include/litmus/cache_proc.h include/litmus/cache_proc.h new file mode 100644 index 0000000..1d9d443 --- /dev/null +++ include/litmus/cache_proc.h @@ -0,0 +1,21 @@ +#ifndef LITMUS_CACHE_PROC_H +#define LITMUS_CACHE_PROC_H + +#ifdef __KERNEL__ + +void litmus_setup_lockdown(void __iomem*, u32); +void enter_irq_mode(void); +void exit_irq_mode(void); +void flush_cache(int all); +void lock_cache(int cpu, u32 val); +void cache_lockdown(u32 lock_val, int cpu); + +extern struct page *new_alloc_page_color(unsigned long color); + +u32 color_read_in_mem_lock(u32 lock_val, u32 unlock_val, void *start, void *end); +u32 color_read_in_mem(u32 lock_val, u32 unlock_val, void *start, void *end); + +#endif + +#endif + diff --git include/litmus/litmus.h include/litmus/litmus.h index a6eb534..441210c 100644 --- include/litmus/litmus.h +++ include/litmus/litmus.h @@ -113,6 +113,13 @@ static inline lt_t litmus_clock(void) ((current)->state == TASK_RUNNING || \ preempt_count() & PREEMPT_ACTIVE) +#define is_running(t) \ + ((t)->state == TASK_RUNNING || \ + task_thread_info(t)->preempt_count & PREEMPT_ACTIVE) + +#define is_blocked(t) \ + (!is_running(t)) + #define is_released(t, now) \ (lt_before_eq(get_release(t), now)) #define is_tardy(t, now) \ diff --git include/litmus/mc2_common.h include/litmus/mc2_common.h new file mode 100644 index 0000000..e3c0af2 --- /dev/null +++ include/litmus/mc2_common.h @@ -0,0 +1,31 @@ +/* + * MC^2 common data structures + */ + +#ifndef __UNC_MC2_COMMON_H__ +#define __UNC_MC2_COMMON_H__ + +enum crit_level { + CRIT_LEVEL_A = 0, + CRIT_LEVEL_B = 1, + CRIT_LEVEL_C = 2, + NUM_CRIT_LEVELS = 3, +}; + +struct mc2_task { + enum crit_level crit; + unsigned int res_id; +}; + +#ifdef __KERNEL__ + +#include + +#define tsk_mc2_data(t) (tsk_rt(t)->mc2_data) + +long mc2_task_client_init(struct task_client *tc, struct mc2_task *mc2_param, struct task_struct *tsk, + struct reservation *res); + +#endif /* __KERNEL__ */ + +#endif \ No newline at end of file diff --git include/litmus/page_dev.h include/litmus/page_dev.h new file mode 100644 index 0000000..c42ed05 --- /dev/null +++ include/litmus/page_dev.h @@ -0,0 +1,23 @@ +#ifndef _LITMUS_PAGE_DEV_H +#define _LITMUS_PAGE_DEV_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +int llc_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +int dram_partition_handler(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); +int bank_to_partition(unsigned int bank); +int get_area_index(int cpu); +int is_in_correct_bank(struct page* page, int cpu); +int is_in_llc_partition(struct page* page, int cpu); + +#endif /* _LITMUS_PAGE_DEV_H */ \ No newline at end of file diff --git include/litmus/polling_reservations.h include/litmus/polling_reservations.h new file mode 100644 index 0000000..66c9b1e --- /dev/null +++ include/litmus/polling_reservations.h @@ -0,0 +1,36 @@ +#ifndef LITMUS_POLLING_RESERVATIONS_H +#define LITMUS_POLLING_RESERVATIONS_H + +#include + +struct polling_reservation { + /* extend basic reservation */ + struct reservation res; + + lt_t max_budget; + lt_t period; + lt_t deadline; + lt_t offset; +}; + +void polling_reservation_init(struct polling_reservation *pres, int use_edf_prio, + int use_periodic_polling, lt_t budget, lt_t period, lt_t deadline, lt_t offset); + +struct table_driven_reservation { + /* extend basic reservation */ + struct reservation res; + + lt_t major_cycle; + unsigned int next_interval; + unsigned int num_intervals; + struct lt_interval *intervals; + + /* info about current scheduling slot */ + struct lt_interval cur_interval; + lt_t major_cycle_start; +}; + +void table_driven_reservation_init(struct table_driven_reservation *tdres, + lt_t major_cycle, struct lt_interval *intervals, unsigned int num_intervals); + +#endif diff --git include/litmus/replicate_lib.h include/litmus/replicate_lib.h new file mode 100644 index 0000000..186837b --- /dev/null +++ include/litmus/replicate_lib.h @@ -0,0 +1,19 @@ +#ifndef LITMUS_REPLICATE_LIB_H +#define LITMUS_REPLICATE_LIB_H + +#include +#include +#include + +/* Data structure for the "master" list */ +struct shared_lib_page { + struct page *master_page; + struct page *r_page[NR_CPUS+1]; + unsigned long int master_pfn; + unsigned long int r_pfn[NR_CPUS+1]; + struct list_head list; +}; + +extern struct list_head shared_lib_pages; + +#endif diff --git include/litmus/reservation.h include/litmus/reservation.h new file mode 100644 index 0000000..7e022b3 --- /dev/null +++ include/litmus/reservation.h @@ -0,0 +1,256 @@ +#ifndef LITMUS_RESERVATION_H +#define LITMUS_RESERVATION_H + +#include +#include + +struct reservation_client; +struct reservation_environment; +struct reservation; + +typedef enum { + /* reservation has no clients, is not consuming budget */ + RESERVATION_INACTIVE = 0, + + /* reservation has clients, consumes budget when scheduled */ + RESERVATION_ACTIVE, + + /* reservation has no clients, but may be consuming budget */ + RESERVATION_ACTIVE_IDLE, + + /* Reservation has no budget and waits for + * replenishment. May or may not have clients. */ + RESERVATION_DEPLETED, +} reservation_state_t; + + +/* ************************************************************************** */ + +/* Select which task to dispatch. If NULL is returned, it means there is nothing + * to schedule right now and background work can be scheduled. */ +typedef struct task_struct * (*dispatch_t) ( + struct reservation_client *client +); + +/* Something that can be managed in a reservation and that can yield + * a process for dispatching. Contains a pointer to the reservation + * to which it "belongs". */ +struct reservation_client { + struct list_head list; + struct reservation* reservation; + dispatch_t dispatch; +}; + + +/* ************************************************************************** */ + +/* Called by reservations to request state change. */ +typedef void (*reservation_change_state_t) ( + struct reservation_environment* env, + struct reservation *res, + reservation_state_t new_state +); + +/* The framework within wich reservations operate. */ +struct reservation_environment { + lt_t time_zero; + lt_t current_time; + + /* services invoked by reservations */ + reservation_change_state_t change_state; +}; + + +/* ************************************************************************** */ + +/* A new client is added or an existing client resumes. */ +typedef void (*client_arrives_t) ( + struct reservation *reservation, + struct reservation_client *client +); + +/* A client suspends or terminates. */ +typedef void (*client_departs_t) ( + struct reservation *reservation, + struct reservation_client *client, + int did_signal_job_completion +); + +/* A previously requested replenishment has occurred. */ +typedef void (*on_replenishment_timer_t) ( + struct reservation *reservation +); + +/* Update the reservation's budget to reflect execution or idling. */ +typedef void (*drain_budget_t) ( + struct reservation *reservation, + lt_t how_much +); + +/* Select a ready task from one of the clients for scheduling. */ +typedef struct task_struct* (*dispatch_client_t) ( + struct reservation *reservation, + lt_t *time_slice /* May be used to force rescheduling after + some amount of time. 0 => no limit */ +); + + +struct reservation_ops { + dispatch_client_t dispatch_client; + + client_arrives_t client_arrives; + client_departs_t client_departs; + + on_replenishment_timer_t replenish; + drain_budget_t drain_budget; +}; + +struct reservation { + /* used to queue in environment */ + struct list_head list; + + reservation_state_t state; + unsigned int id; + + /* exact meaning defined by impl. */ + lt_t priority; + lt_t cur_budget; + lt_t next_replenishment; + + /* budget stats */ + lt_t budget_consumed; /* how much budget consumed in this allocation cycle? */ + lt_t budget_consumed_total; + + /* interaction with framework */ + struct reservation_environment *env; + struct reservation_ops *ops; + + struct list_head clients; + + /* for global env. */ + int scheduled_on; + int event_added; + /* for blocked by ghost. Do not charge budget when ACTIVE */ + int blocked_by_ghost; + /* ghost_job. If it is clear, do not charge budget when ACTIVE_IDLE */ + int is_ghost; +}; + +void reservation_init(struct reservation *res); + +/* Default implementations */ + +/* simply select the first client in the list, set *for_at_most to zero */ +struct task_struct* default_dispatch_client( + struct reservation *res, + lt_t *for_at_most +); + +/* "connector" reservation client to hook up tasks with reservations */ +struct task_client { + struct reservation_client client; + struct task_struct *task; +}; + +void task_client_init(struct task_client *tc, struct task_struct *task, + struct reservation *reservation); + +#define SUP_RESCHEDULE_NOW (0) +#define SUP_NO_SCHEDULER_UPDATE (ULLONG_MAX) + +/* A simple uniprocessor (SUP) flat (i.e., non-hierarchical) reservation + * environment. + */ +struct sup_reservation_environment { + struct reservation_environment env; + + /* ordered by priority */ + struct list_head active_reservations; + + /* ordered by next_replenishment */ + struct list_head depleted_reservations; + + /* unordered */ + struct list_head inactive_reservations; + + /* - SUP_RESCHEDULE_NOW means call sup_dispatch() now + * - SUP_NO_SCHEDULER_UPDATE means nothing to do + * any other value means program a timer for the given time + */ + lt_t next_scheduler_update; + /* set to true if a call to sup_dispatch() is imminent */ + bool will_schedule; +}; + +/* Contract: + * - before calling into sup_ code, or any reservation methods, + * update the time with sup_update_time(); and + * - after calling into sup_ code, or any reservation methods, + * check next_scheduler_update and program timer or trigger + * scheduler invocation accordingly. + */ + +void sup_init(struct sup_reservation_environment* sup_env); +void sup_add_new_reservation(struct sup_reservation_environment* sup_env, + struct reservation* new_res); +void sup_scheduler_update_after(struct sup_reservation_environment* sup_env, + lt_t timeout); +void sup_update_time(struct sup_reservation_environment* sup_env, lt_t now); +struct task_struct* sup_dispatch(struct sup_reservation_environment* sup_env); + +struct reservation* sup_find_by_id(struct sup_reservation_environment* sup_env, + unsigned int id); + +/* A global multiprocessor reservation environment. */ + +typedef enum { + EVENT_REPLENISH = 0, + EVENT_DRAIN, + EVENT_OTHERS, +} event_type_t; + + +struct next_timer_event { + lt_t next_update; + int timer_armed_on; + unsigned int id; + event_type_t type; + struct list_head list; +}; + +struct gmp_reservation_environment { + raw_spinlock_t lock; + struct reservation_environment env; + + /* ordered by priority */ + struct list_head active_reservations; + + /* ordered by next_replenishment */ + struct list_head depleted_reservations; + + /* unordered */ + struct list_head inactive_reservations; + + /* timer event ordered by next_update */ + struct list_head next_events; + + /* (schedule_now == true) means call gmp_dispatch() now */ + int schedule_now; + /* set to true if a call to gmp_dispatch() is imminent */ + bool will_schedule; +}; + +void gmp_init(struct gmp_reservation_environment* gmp_env); +void gmp_add_new_reservation(struct gmp_reservation_environment* gmp_env, + struct reservation* new_res); +void gmp_add_event_after(struct gmp_reservation_environment* gmp_env, + lt_t timeout, unsigned int id, event_type_t type); +void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now); +int gmp_update_time(struct gmp_reservation_environment* gmp_env, lt_t now); +struct task_struct* gmp_dispatch(struct gmp_reservation_environment* gmp_env); +struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env, unsigned int id); +struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env, lt_t when); +struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env, + unsigned int id); + +#endif diff --git include/litmus/rt_param.h include/litmus/rt_param.h index 7b9a909..2044327 100644 --- include/litmus/rt_param.h +++ include/litmus/rt_param.h @@ -62,6 +62,7 @@ typedef enum { #define LITMUS_MAX_PRIORITY 512 #define LITMUS_HIGHEST_PRIORITY 1 #define LITMUS_LOWEST_PRIORITY (LITMUS_MAX_PRIORITY - 1) +#define LITMUS_NO_PRIORITY UINT_MAX /* Provide generic comparison macros for userspace, * in case that we change this later. */ @@ -71,6 +72,46 @@ typedef enum { ((p) >= LITMUS_HIGHEST_PRIORITY && \ (p) <= LITMUS_LOWEST_PRIORITY) +/* reservation support */ + +typedef enum { + PERIODIC_POLLING, + SPORADIC_POLLING, + TABLE_DRIVEN, +} reservation_type_t; + +struct lt_interval { + lt_t start; + lt_t end; +}; + +#ifndef __KERNEL__ +#define __user +#endif + +struct reservation_config { + unsigned int id; + lt_t priority; + int cpu; + + union { + struct { + lt_t period; + lt_t budget; + lt_t relative_deadline; + lt_t offset; + } polling_params; + + struct { + lt_t major_cycle_length; + unsigned int num_intervals; + struct lt_interval __user *intervals; + } table_driven_params; + }; +}; + +/* regular sporadic task support */ + struct rt_task { lt_t exec_cost; lt_t period; @@ -120,7 +161,6 @@ struct control_page { uint64_t ts_syscall_start; /* Feather-Trace cycles */ uint64_t irq_syscall_start; /* Snapshot of irq_count when the syscall * started. */ - /* to be extended */ }; @@ -165,6 +205,7 @@ struct rt_job { }; struct pfair_param; +struct mc2_task; /* RT task parameters for scheduling extensions * These parameters are inherited during clone and therefore must @@ -246,7 +287,10 @@ struct rt_param { volatile int linked_on; /* PFAIR/PD^2 state. Allocated on demand. */ - struct pfair_param* pfair; + union { + void *plugin_state; + struct pfair_param *pfair; + }; /* Fields saved before BE->RT transition. */ @@ -275,6 +319,10 @@ struct rt_param { /* Pointer to the page shared between userspace and kernel. */ struct control_page * ctrl_page; + + /* Mixed-criticality specific data */ + struct mc2_task* mc2_data; + unsigned long addr_ctrl_page; }; #endif diff --git include/litmus/sched_plugin.h include/litmus/sched_plugin.h index 0ccccd6..4c8aaa6 100644 --- include/litmus/sched_plugin.h +++ include/litmus/sched_plugin.h @@ -77,6 +77,17 @@ typedef long (*wait_for_release_at_t)(lt_t release_time); /* Informs the plugin when a synchronous release takes place. */ typedef void (*synchronous_release_at_t)(lt_t time_zero); +/* How much budget has the current task consumed so far, and how much + * has it left? The default implementation ties into the per-task + * budget enforcement code. Plugins can override this to report + * reservation-specific values. */ +typedef void (*current_budget_t)(lt_t *used_so_far, lt_t *remaining); + +/* Reservation creation/removal backends. Meaning of reservation_type and + * reservation_id are entirely plugin-specific. */ +typedef long (*reservation_create_t)(int reservation_type, void* __user config); +typedef long (*reservation_destroy_t)(unsigned int reservation_id, int cpu); + /************************ misc routines ***********************/ @@ -109,6 +120,12 @@ struct sched_plugin { task_exit_t task_exit; task_cleanup_t task_cleanup; + current_budget_t current_budget; + + /* Reservation support */ + reservation_create_t reservation_create; + reservation_destroy_t reservation_destroy; + #ifdef CONFIG_LITMUS_LOCKING /* locking protocols */ allocate_lock_t allocate_lock; diff --git include/litmus/sched_trace.h include/litmus/sched_trace.h index 82bde82..1b03778 100644 --- include/litmus/sched_trace.h +++ include/litmus/sched_trace.h @@ -23,8 +23,7 @@ struct st_param_data { /* regular params */ u32 period; u32 phase; u8 partition; - u8 class; - u8 __unused[2]; + u8 __unused[3]; }; struct st_release_data { /* A job is was/is going to be released. */ @@ -35,14 +34,12 @@ struct st_release_data { /* A job is was/is going to be released. */ struct st_assigned_data { /* A job was asigned to a CPU. */ u64 when; u8 target; /* Where should it execute? */ - u8 __unused[7]; + u8 __unused[3]; }; struct st_switch_to_data { /* A process was switched to on a given CPU. */ u64 when; /* When did this occur? */ u32 exec_time; /* Time the current job has executed. */ - u8 __unused[4]; - }; struct st_switch_away_data { /* A process was switched away from on a given CPU. */ @@ -52,11 +49,10 @@ struct st_switch_away_data { /* A process was switched away from on a given CPU. struct st_completion_data { /* A job completed. */ u64 when; - u8 forced:1; /* Set to 1 if job overran and kernel advanced to the + u64 forced:1; /* Set to 1 if job overran and kernel advanced to the * next task automatically; set to 0 otherwise. */ - u8 __uflags:7; - u8 __unused[7]; + u64 exec_time:63; }; struct st_block_data { /* A task blocks. */ @@ -69,6 +65,16 @@ struct st_resume_data { /* A task resumes. */ u64 __unused; }; +struct st_np_enter_data { /* A task becomes non-preemptable. */ + u64 when; + u64 __unused; +}; + +struct st_np_exit_data { /* A task becomes preemptable again. */ + u64 when; + u64 __unused; +}; + struct st_action_data { u64 when; u8 action; @@ -94,7 +100,10 @@ typedef enum { ST_BLOCK, ST_RESUME, ST_ACTION, - ST_SYS_RELEASE + ST_SYS_RELEASE, + ST_NP_ENTER, + ST_NP_EXIT, + ST_INVALID } st_event_record_type_t; struct st_event_record { @@ -113,6 +122,8 @@ struct st_event_record { DATA(resume); DATA(action); DATA(sys_release); + DATA(np_enter); + DATA(np_exit); } data; }; diff --git include/litmus/trace.h include/litmus/trace.h index 6017872..24ca412 100644 --- include/litmus/trace.h +++ include/litmus/trace.h @@ -118,6 +118,9 @@ feather_callback void save_cpu_task_latency(unsigned long event, unsigned long w #define TS_TICK_START(t) CPU_TTIMESTAMP(110, t) #define TS_TICK_END(t) CPU_TTIMESTAMP(111, t) +#define TS_RELEASE_C_START CPU_DTIMESTAMP(108, TSK_RT) +#define TS_RELEASE_C_END CPU_DTIMESTAMP(109, TSK_RT) + #define TS_QUANTUM_BOUNDARY_START CPU_TIMESTAMP_CUR(112) #define TS_QUANTUM_BOUNDARY_END CPU_TIMESTAMP_CUR(113) @@ -137,6 +140,17 @@ feather_callback void save_cpu_task_latency(unsigned long event, unsigned long w #define TS_SEND_RESCHED_START(c) MSG_TIMESTAMP_SENT(190, c) #define TS_SEND_RESCHED_END MSG_TIMESTAMP_RECEIVED(191) -#define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when)) +#define TS_ISR_START CPU_TIMESTAMP_CUR(192) +#define TS_ISR_END CPU_TIMESTAMP_CUR(193) + +#define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when)) +#define TS_RELEASE_LATENCY_A(when) CPU_LTIMESTAMP(209, &(when)) +#define TS_RELEASE_LATENCY_B(when) CPU_LTIMESTAMP(210, &(when)) +#define TS_RELEASE_LATENCY_C(when) CPU_LTIMESTAMP(211, &(when)) + +#define TS_SCHED_A_START CPU_DTIMESTAMP(212, TSK_UNKNOWN) +#define TS_SCHED_A_END(t) CPU_TTIMESTAMP(213, t) +#define TS_SCHED_C_START CPU_DTIMESTAMP(214, TSK_UNKNOWN) +#define TS_SCHED_C_END(t) CPU_TTIMESTAMP(215, t) #endif /* !_SYS_TRACE_H_ */ diff --git include/litmus/unistd_32.h include/litmus/unistd_32.h index 94264c2..664f73f 100644 --- include/litmus/unistd_32.h +++ include/litmus/unistd_32.h @@ -17,5 +17,14 @@ #define __NR_wait_for_ts_release __LSC(9) #define __NR_release_ts __LSC(10) #define __NR_null_call __LSC(11) +#define __NR_get_current_budget __LSC(12) +#define __NR_reservation_create __LSC(13) +#define __NR_reservation_destroy __LSC(14) +#define __NR_set_mc2_task_param __LSC(15) +#define __NR_set_page_color __LSC(16) +#define __NR_test_call __LSC(17) +#define __NR_run_test __LSC(18) +#define __NR_lock_buffer __LSC(19) +#define __NR_recolor_mem __LSC(20) -#define NR_litmus_syscalls 12 +#define NR_litmus_syscalls 21 diff --git include/litmus/unistd_64.h include/litmus/unistd_64.h index d5ced0d..994e8a5 100644 --- include/litmus/unistd_64.h +++ include/litmus/unistd_64.h @@ -29,5 +29,23 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release) __SYSCALL(__NR_release_ts, sys_release_ts) #define __NR_null_call __LSC(11) __SYSCALL(__NR_null_call, sys_null_call) +#define __NR_get_current_budget __LSC(12) +__SYSCALL(__NR_get_current_budget, sys_get_current_budget) +#define __NR_reservation_create __LSC(13) +__SYSCALL(__NR_reservation_create, sys_reservation_create) +#define __NR_reservation_destroy __LSC(14) +__SYSCALL(__NR_reservation_destroy, sys_reservation_destroy) +#define __NR_set_mc2_task_param __LSC(15) +__SYSCALL(__NR_set_mc2_task_param, sys_set_mc2_task_param) +#define __NR_set_page_color __LSC(16) +__SYSCALL(__NR_set_page_color, sys_set_page_color) +#define __NR_test_call __LSC(17) +__SYSCALL(__NR_test_call, sys_test_call) +#define __NR_run_test __LSC(18) +__SYSCALL(__NR_run_test, sys_run_test) +#define __NR_lock_buffer __LSC(19) +__SYSCALL(__NR_lock_buffer, sys_lock_buffer) +#define __NR_recolor_mem __LSC(20) +__SYSCALL(__NR_recolor_mem, sys_recolor_mem) -#define NR_litmus_syscalls 12 +#define NR_litmus_syscalls 21 diff --git ipc/mqueue.c ipc/mqueue.c index 3aaea7f..4cb1b7f 100644 --- ipc/mqueue.c +++ ipc/mqueue.c @@ -1004,7 +1004,7 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr, /* First try to allocate memory, before doing anything with * existing queues. */ - msg_ptr = load_msg(u_msg_ptr, msg_len); + msg_ptr = load_msg(u_msg_ptr, msg_len, 0); if (IS_ERR(msg_ptr)) { ret = PTR_ERR(msg_ptr); goto out_fput; diff --git ipc/msg.c ipc/msg.c index 2b6fdbb..da2b3e9 100644 --- ipc/msg.c +++ ipc/msg.c @@ -38,6 +38,10 @@ #include #include +#include +#include +#include + #include #include #include "util.h" @@ -619,7 +623,7 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext, if (mtype < 1) return -EINVAL; - msg = load_msg(mtext, msgsz); + msg = load_msg(mtext, msgsz, mtype); if (IS_ERR(msg)) return PTR_ERR(msg); @@ -752,13 +756,13 @@ static long do_msg_fill(void __user *dest, struct msg_msg *msg, size_t bufsz) { struct msgbuf __user *msgp = dest; size_t msgsz; - if (put_user(msg->m_type, &msgp->mtype)) return -EFAULT; - msgsz = (bufsz > msg->m_ts) ? msg->m_ts : bufsz; + if (store_msg(msgp->mtext, msg, msgsz)) return -EFAULT; + return msgsz; } @@ -976,7 +980,6 @@ out_unlock1: free_copy(copy); return PTR_ERR(msg); } - bufsz = msg_handler(buf, msg, bufsz); free_msg(msg); diff --git ipc/msgutil.c ipc/msgutil.c index 2b49159..1ffc1b2 100644 --- ipc/msgutil.c +++ ipc/msgutil.c @@ -18,6 +18,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include "util.h" @@ -42,20 +48,35 @@ atomic_t nr_ipc_ns = ATOMIC_INIT(1); struct msg_msgseg { struct msg_msgseg *next; /* the next part of the message follows immediately */ + dma_addr_t seg_handle; + size_t seg_len; }; #define DATALEN_MSG ((size_t)PAGE_SIZE-sizeof(struct msg_msg)) #define DATALEN_SEG ((size_t)PAGE_SIZE-sizeof(struct msg_msgseg)) +static dma_addr_t handle; -static struct msg_msg *alloc_msg(size_t len) +mempool_t *msgpool; +extern void *msgvaddr; + +static struct msg_msg *alloc_msg(size_t len, long mtype) { struct msg_msg *msg; struct msg_msgseg **pseg; size_t alen; + int n_seg = 0; alen = min(len, DATALEN_MSG); - msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL); + if (mtype == 1) { + msg = dma_alloc_coherent(NULL, sizeof(*msg) + alen, &handle, GFP_KERNEL|GFP_COLOR); + msg->handle = handle; + msg->alloc_len = sizeof(*msg) + alen; + } else if (mtype == 2) { + msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL|GFP_COLOR); + } else { + msg = msgvaddr; + } if (msg == NULL) return NULL; @@ -67,7 +88,16 @@ static struct msg_msg *alloc_msg(size_t len) while (len > 0) { struct msg_msgseg *seg; alen = min(len, DATALEN_SEG); - seg = kmalloc(sizeof(*seg) + alen, GFP_KERNEL); + if (mtype == 1) { + seg = dma_alloc_coherent(NULL, sizeof(*seg) + alen, &handle, GFP_KERNEL|GFP_COLOR); + seg->seg_handle = handle; + seg->seg_len = alen; + } else if (mtype == 2) { + seg = kmalloc(sizeof(*seg) + alen, GFP_KERNEL|GFP_COLOR); + } else { + n_seg++; + seg = msgvaddr + PAGE_SIZE*n_seg; + } if (seg == NULL) goto out_err; *pseg = seg; @@ -83,18 +113,19 @@ out_err: return NULL; } -struct msg_msg *load_msg(const void __user *src, size_t len) +struct msg_msg *load_msg(const void __user *src, size_t len, long mtype) { struct msg_msg *msg; struct msg_msgseg *seg; int err = -EFAULT; size_t alen; - msg = alloc_msg(len); + msg = alloc_msg(len, mtype); if (msg == NULL) return ERR_PTR(-ENOMEM); alen = min(len, DATALEN_MSG); + if (copy_from_user(msg + 1, src, alen)) goto out_err; @@ -105,6 +136,9 @@ struct msg_msg *load_msg(const void __user *src, size_t len) if (copy_from_user(seg + 1, src, alen)) goto out_err; } + if (mtype == 3) { + cache_lockdown(0xFFFF8000, smp_processor_id()); + } err = security_msg_msg_alloc(msg); if (err) @@ -172,14 +206,22 @@ int store_msg(void __user *dest, struct msg_msg *msg, size_t len) void free_msg(struct msg_msg *msg) { struct msg_msgseg *seg; - + long mtype = msg->m_type; + security_msg_msg_free(msg); seg = msg->next; - kfree(msg); + if (mtype == 1) { + dma_free_coherent(NULL, msg->alloc_len, msg, msg->handle); + } else if (mtype != 3) { + kfree(msg); + } while (seg != NULL) { struct msg_msgseg *tmp = seg->next; - kfree(seg); + if (mtype == 1) { + dma_free_coherent(NULL, sizeof(*seg)+(seg->seg_len), seg, seg->seg_handle); + } else if (mtype != 3) + kfree(seg); seg = tmp; } } diff --git ipc/util.h ipc/util.h index 1a5a0fc..9a29983 100644 --- ipc/util.h +++ ipc/util.h @@ -149,7 +149,7 @@ int ipc_parse_version(int *cmd); #endif extern void free_msg(struct msg_msg *msg); -extern struct msg_msg *load_msg(const void __user *src, size_t len); +extern struct msg_msg *load_msg(const void __user *src, size_t len, long mtype); extern struct msg_msg *copy_msg(struct msg_msg *src, struct msg_msg *dst); extern int store_msg(void __user *dest, struct msg_msg *msg, size_t len); diff --git kernel/sched/litmus.c kernel/sched/litmus.c index 9d58690..cd36358 100644 --- kernel/sched/litmus.c +++ kernel/sched/litmus.c @@ -20,8 +20,9 @@ static void update_time_litmus(struct rq *rq, struct task_struct *p) /* task counter */ p->se.sum_exec_runtime += delta; if (delta) { - TRACE_TASK(p, "charged %llu exec time (total:%llu, rem:%llu)\n", - delta, p->rt_param.job_params.exec_time, budget_remaining(p)); + //TRACE_TASK(p, "charged %llu exec time (total:%llu, rem:%llu)\n", + //delta, p->rt_param.job_params.exec_time, budget_remaining(p)); + ; } /* sched_clock() */ p->se.exec_start = rq->clock; diff --git litmus/Kconfig litmus/Kconfig index babb43d..8f38bc3 100644 --- litmus/Kconfig +++ litmus/Kconfig @@ -372,4 +372,26 @@ config PREEMPT_STATE_TRACE endmenu +menu "Memory Management" + +choice + prompt "UVC buffer allocation" + default UVC_ALLOC_LEVEL_C + help + Determine the location of frame buffers used by UVC (USB Video Class) devices. Other data structures such as URB (USB Request Block) and V4L2 (Video for Linux Version 2) use Level-C/OS banks. + + config UVC_ALLOC_LEVEL_C + bool "Level-C/OS" + help + Allocate frame buffers in Level-C/OS banks. + + config UVC_ALLOC_LEVEL_AB + bool "Level-A/B" + help + Allocate frame buffers in the task's Level-A/B bank. If this option is selected, the task that uses a camera must be a Level-A or -B task. + +endchoice + +endmenu + endmenu diff --git litmus/Makefile litmus/Makefile index 7970cd5..29ae4b0 100644 --- litmus/Makefile +++ litmus/Makefile @@ -11,6 +11,7 @@ obj-y = sched_plugin.o litmus.o \ sync.o \ rt_domain.o \ edf_common.o \ + mc2_common.o \ fp_common.o \ fdso.o \ locking.o \ @@ -19,13 +20,20 @@ obj-y = sched_plugin.o litmus.o \ binheap.o \ ctrldev.o \ uncachedev.o \ + reservation.o \ + polling_reservations.o \ sched_gsn_edf.o \ sched_psn_edf.o \ - sched_pfp.o + sched_pfp.o \ + sched_mc2.o \ + bank_proc.o \ + color_shm.o \ + replicate_lib.o \ + cache_proc.o \ + page_dev.o obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o - obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o obj-$(CONFIG_SCHED_DEBUG_TRACE) += sched_trace.o diff --git litmus/bank_proc.c litmus/bank_proc.c new file mode 100644 index 0000000..3c4f703 --- /dev/null +++ litmus/bank_proc.c @@ -0,0 +1,775 @@ +/* + * bank_proc.c -- Implementation of the page coloring for cache and bank partition. + * The file will keep a pool of colored pages. Users can require pages with + * specific color or bank number. + * Part of the code is modified from Jonathan Herman's code + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define LITMUS_LOCKDEP_NAME_MAX_LEN 50 + +// This Address Decoding is used in imx6-sabredsd platform +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 + +#define PAGES_PER_COLOR 2000 +#define NUM_BANKS 8 +#define NUM_COLORS 16 + +unsigned int NUM_PAGE_LIST; //8*16 + +unsigned int number_banks; +unsigned int number_cachecolors; + +unsigned int set_partition_max = 0x0000ffff; +unsigned int set_partition_min = 0; +unsigned int bank_partition_max = 0x000000ff; +unsigned int bank_partition_min = 0; + +int show_page_pool = 0; +int refill_page_pool = 0; +spinlock_t reclaim_lock; + +unsigned int set_partition[9] = { + 0x00000003, /* Core 0, and Level A*/ + 0x00000003, /* Core 0, and Level B*/ + 0x0000000C, /* Core 1, and Level A*/ + 0x0000000C, /* Core 1, and Level B*/ + 0x00000030, /* Core 2, and Level A*/ + 0x00000030, /* Core 2, and Level B*/ + 0x000000C0, /* Core 3, and Level A*/ + 0x000000C0, /* Core 3, and Level B*/ + 0x0000ff00, /* Level C */ +}; + +unsigned int bank_partition[9] = { + 0x00000010, /* Core 0, and Level A*/ + 0x00000010, /* Core 0, and Level B*/ + 0x00000020, /* Core 1, and Level A*/ + 0x00000020, /* Core 1, and Level B*/ + 0x00000040, /* Core 2, and Level A*/ + 0x00000040, /* Core 2, and Level B*/ + 0x00000080, /* Core 3, and Level A*/ + 0x00000080, /* Core 3, and Level B*/ + 0x0000000c, /* Level C */ +}; + +unsigned int set_index[9] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +unsigned int bank_index[9] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +int node_index[9] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1 +}; + +struct mutex void_lockdown_proc; + +/* + * Every page list should contain a lock, a list, and a number recording how many pages it store + */ +struct color_group { + spinlock_t lock; + char _lock_name[LITMUS_LOCKDEP_NAME_MAX_LEN]; + struct list_head list; + atomic_t nr_pages; +}; + + +static struct color_group *color_groups; + +/* + * Naive function to count the number of 1's + */ +unsigned int counting_one_set(unsigned int v) +{ + unsigned int c; // c accumulates the total bits set in v + + for (c = 0; v; v >>= 1) + { + c += v & 1; + } + return c; +} + +unsigned int two_exp(unsigned int e) +{ + unsigned int v = 1; + for (; e>0; e-- ) + { + v=v*2; + } + return v; +} + +/* helper functions to find the next colored pool index */ +static inline unsigned int first_index(unsigned long node) +{ + unsigned int bank_no = 0, color_no = 0; + + while(bank_no < NUM_BANKS) { + if ((bank_partition[node]>>bank_no) & 0x1) + break; + bank_no++; + } + while(color_no < NUM_COLORS) { + if ((set_partition[node]>>color_no) & 0x1) + break; + color_no++; + } + return NUM_COLORS*bank_no + color_no; +} + +static inline unsigned int last_index(unsigned long node) +{ + unsigned int bank_no = NUM_BANKS-1, color_no = NUM_COLORS-1; + + while(bank_no >= 0) { + if ((bank_partition[node]>>bank_no) & 0x1) + break; + bank_no--; + } + while(color_no >= 0) { + if ((set_partition[node]>>color_no) & 0x1) + break; + color_no--; + } + return NUM_COLORS*bank_no + color_no; +} + +static inline unsigned int next_color(unsigned long node, unsigned int current_color) +{ + int try = 0, ret = 0; + current_color++; + if (current_color == NUM_COLORS) { + current_color = 0; + ret = 1; + } + + while (try < NUM_COLORS) { + if ((set_partition[node]>>current_color)&0x1) + break; + current_color++; + if (current_color == NUM_COLORS) { + current_color = 0; + ret = 1; + } + try++; + } + if (!ret) + return current_color; + else + return current_color + NUM_COLORS; +} + +static inline unsigned int next_bank(unsigned long node, unsigned int current_bank) +{ + int try = 0; + current_bank++; + if (current_bank == NUM_BANKS) { + current_bank = 0; + } + + while (try < NUM_BANKS) { + if ((bank_partition[node]>>current_bank)&0x1) + break; + current_bank++; + if (current_bank == NUM_BANKS) { + current_bank = 0; + } + try++; + } + return current_bank; +} + +static inline unsigned int get_next_index(unsigned long node, unsigned int current_index) +{ + unsigned int bank_no, color_no, color_ret, bank_ret; + bank_no = current_index>>4; // 2^4 = 16 colors + color_no = current_index - bank_no*NUM_COLORS; + bank_ret = bank_no; + color_ret = next_color(node, color_no); + if (color_ret >= NUM_COLORS) { + // next bank + color_ret -= NUM_COLORS; + bank_ret = next_bank(node, bank_no); + } + + return bank_ret * NUM_COLORS + color_ret; +} + +/* Decoding page color, 0~15 */ +static inline unsigned int page_color(struct page *page) +{ + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT); +} + +/* Decoding page bank number, 0~7 */ +static inline unsigned int page_bank(struct page *page) +{ + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT); +} + +static inline unsigned int page_list_index(struct page *page) +{ + unsigned int idx; + idx = (page_color(page) + page_bank(page)*(number_cachecolors)); + + return idx; +} + + + +/* + * It is used to determine the smallest number of page lists. + */ +static unsigned long smallest_nr_pages(void) +{ + unsigned long i, min_pages; + struct color_group *cgroup; + cgroup = &color_groups[16*2]; + min_pages =atomic_read(&cgroup->nr_pages); + for (i = 16*2; i < NUM_PAGE_LIST; ++i) { + cgroup = &color_groups[i]; + if (atomic_read(&cgroup->nr_pages) < min_pages) + min_pages = atomic_read(&cgroup->nr_pages); + } + return min_pages; +} + +static void show_nr_pages(void) +{ + unsigned long i; + struct color_group *cgroup; + printk("show nr pages***************************************\n"); + for (i = 0; i < NUM_PAGE_LIST; ++i) { + cgroup = &color_groups[i]; + printk("(%03ld) = %03d, ", i, atomic_read(&cgroup->nr_pages)); + if((i % 8) ==7) { + printk("\n"); + } + } +} + +/* + * Add a page to current pool. + */ +void add_page_to_color_list(struct page *page) +{ + const unsigned long color = page_list_index(page); + struct color_group *cgroup = &color_groups[color]; + BUG_ON(in_list(&page->lru) || PageLRU(page)); + BUG_ON(page_count(page) > 1); + spin_lock(&cgroup->lock); + list_add_tail(&page->lru, &cgroup->list); + atomic_inc(&cgroup->nr_pages); + SetPageLRU(page); + spin_unlock(&cgroup->lock); +} + +/* + * Replenish the page pool. + * If the newly allocate page is what we want, it will be pushed to the correct page list + * otherwise, it will be freed. + * A user needs to invoke this function until the page pool has enough pages. + */ +static int do_add_pages(void) +{ + struct page *page, *page_tmp; + LIST_HEAD(free_later); + unsigned long color; + int ret = 0; + int i = 0; + int free_counter = 0; + unsigned long counter[128]= {0}; + + // until all the page lists contain enough pages + for (i=0; i< 1024*20;i++) { + page = alloc_page(GFP_HIGHUSER_MOVABLE); + + if (unlikely(!page)) { + printk(KERN_WARNING "Could not allocate pages.\n"); + ret = -ENOMEM; + goto out; + } + color = page_list_index(page); + counter[color]++; + if (atomic_read(&color_groups[color].nr_pages) < PAGES_PER_COLOR && color>=0) { + add_page_to_color_list(page); + } else { + // Pages here will be freed later + list_add_tail(&page->lru, &free_later); + free_counter++; + } + } + + // Free the unwanted pages + list_for_each_entry_safe(page, page_tmp, &free_later, lru) { + list_del(&page->lru); + __free_page(page); + } +out: + return ret; +} + +/* + * Provide pages for replacement according cache color + * This should be the only implementation here + * This function should not be accessed by others directly. + * + */ +static struct page *new_alloc_page_color( unsigned long color) +{ +// printk("allocate new page color = %d\n", color); + struct color_group *cgroup; + struct page *rPage = NULL; + + if( (color <0) || (color)>(number_cachecolors*number_banks -1)) { + TRACE_CUR("Wrong color %lu\n", color); + goto out; + } + + + cgroup = &color_groups[color]; + spin_lock(&cgroup->lock); + if (unlikely(!atomic_read(&cgroup->nr_pages))) { + TRACE_CUR("No free %lu colored pages.\n", color); + goto out_unlock; + } + rPage = list_first_entry(&cgroup->list, struct page, lru); + BUG_ON(page_count(rPage) > 1); + //get_page(rPage); + list_del(&rPage->lru); + atomic_dec(&cgroup->nr_pages); + ClearPageLRU(rPage); +out_unlock: + spin_unlock(&cgroup->lock); +out: + return rPage; +} + +struct page* get_colored_page(unsigned long color) +{ + return new_alloc_page_color(color); +} + +/* + * provide pages for replacement according to + * node = 0 for Level A tasks in Cpu 0 + * node = 1 for Level B tasks in Cpu 0 + * node = 2 for Level A tasks in Cpu 1 + * node = 3 for Level B tasks in Cpu 1 + * node = 4 for Level A tasks in Cpu 2 + * node = 5 for Level B tasks in Cpu 2 + * node = 6 for Level A tasks in Cpu 3 + * node = 7 for Level B tasks in Cpu 3 + * node = 8 for Level C tasks + */ +struct page *new_alloc_page(struct page *page, unsigned long node, int **x) +{ + struct page *rPage = NULL; + int try = 0; + unsigned int idx; + + if (node_index[node] == -1) + idx = first_index(node); + else + idx = node_index[node]; + + BUG_ON(idx<0 || idx>127); + rPage = new_alloc_page_color(idx); + if (node_index[node] == last_index(node)) + node_index[node] = first_index(node); + else + node_index[node]++; + + while (!rPage) { + try++; + if (try>=256) + break; + idx = get_next_index(node, idx); + printk(KERN_ALERT "try = %d out of page! requesting node = %ld, idx = %d\n", try, node, idx); + BUG_ON(idx<0 || idx>127); + rPage = new_alloc_page_color(idx); + } + node_index[node] = idx; + return rPage; +} + + +/* + * Reclaim pages. + */ +void reclaim_page(struct page *page) +{ + const unsigned long color = page_list_index(page); + spin_lock(&reclaim_lock); + put_page(page); + add_page_to_color_list(page); + + spin_unlock(&reclaim_lock); + printk("Reclaimed page(%ld) = color %x, bank %x, [color] =%d \n", color, page_color(page), page_bank(page), atomic_read(&color_groups[color].nr_pages)); +} + + +/* + * Initialize the numbers of banks and cache colors + */ +static void __init init_variables(void) +{ + number_banks = counting_one_set(BANK_MASK); + number_banks = two_exp(number_banks); + + number_cachecolors = counting_one_set(CACHE_MASK); + number_cachecolors = two_exp(number_cachecolors); + NUM_PAGE_LIST = number_banks * number_cachecolors; + printk(KERN_WARNING "number of banks = %d, number of cachecolors=%d\n", number_banks, number_cachecolors); + mutex_init(&void_lockdown_proc); + spin_lock_init(&reclaim_lock); + +} + + +/* + * Initialize the page pool + */ +static int __init init_color_groups(void) +{ + struct color_group *cgroup; + unsigned long i; + int err = 0; + + printk("NUM_PAGE_LIST = %d\n", NUM_PAGE_LIST); + color_groups = kmalloc(NUM_PAGE_LIST *sizeof(struct color_group), GFP_KERNEL); + + if (!color_groups) { + printk(KERN_WARNING "Could not allocate color groups.\n"); + err = -ENOMEM; + }else{ + + for (i = 0; i < NUM_PAGE_LIST; ++i) { + cgroup = &color_groups[i]; + atomic_set(&cgroup->nr_pages, 0); + INIT_LIST_HEAD(&cgroup->list); + spin_lock_init(&cgroup->lock); + } + } + return err; +} + +int set_partition_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i = 0; + mutex_lock(&void_lockdown_proc); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + if (write) { + printk("New set Partition : \n"); + for(i =0;i <9;i++) + { + set_index[i] = 0; + printk("set[%d] = %x \n", i, set_partition[i]); + } + } +out: + mutex_unlock(&void_lockdown_proc); + return ret; +} + +int bank_partition_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i = 0; + mutex_lock(&void_lockdown_proc); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + if (write) { + for(i =0;i <9;i++) + { + bank_index[i] = 0; + } + } +out: + mutex_unlock(&void_lockdown_proc); + return ret; +} + +int show_page_pool_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0; + mutex_lock(&void_lockdown_proc); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + if (write) { + show_nr_pages(); + } +out: + mutex_unlock(&void_lockdown_proc); + return ret; +} + +int refill_page_pool_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0; + mutex_lock(&void_lockdown_proc); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + if (write) { + do_add_pages(); + show_nr_pages(); + } +out: + mutex_unlock(&void_lockdown_proc); + return ret; +} + +/* +static struct ctl_table cache_table[] = +{ + + { + .procname = "C0_LA_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[0], + .maxlen = sizeof(set_partition[0]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C0_LB_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[1], + .maxlen = sizeof(set_partition[1]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C1_LA_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[2], + .maxlen = sizeof(set_partition[2]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C1_LB_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[3], + .maxlen = sizeof(set_partition[3]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C2_LA_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[4], + .maxlen = sizeof(set_partition[4]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C2_LB_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[5], + .maxlen = sizeof(set_partition[5]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C3_LA_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[6], + .maxlen = sizeof(set_partition[6]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C3_LB_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[7], + .maxlen = sizeof(set_partition[7]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "Call_LC_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[8], + .maxlen = sizeof(set_partition[8]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C0_LA_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[0], + .maxlen = sizeof(set_partition[0]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C0_LB_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[1], + .maxlen = sizeof(set_partition[1]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C1_LA_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[2], + .maxlen = sizeof(set_partition[2]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C1_LB_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[3], + .maxlen = sizeof(set_partition[3]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C2_LA_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[4], + .maxlen = sizeof(set_partition[4]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C2_LB_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[5], + .maxlen = sizeof(set_partition[5]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C3_LA_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[6], + .maxlen = sizeof(set_partition[6]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C3_LB_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[7], + .maxlen = sizeof(set_partition[7]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "Call_LC_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[8], + .maxlen = sizeof(set_partition[8]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "show_page_pool", + .mode = 0666, + .proc_handler = show_page_pool_handler, + .data = &show_page_pool, + .maxlen = sizeof(show_page_pool), + }, { + .procname = "refill_page_pool", + .mode = 0666, + .proc_handler = refill_page_pool_handler, + .data = &refill_page_pool, + .maxlen = sizeof(refill_page_pool), + }, + { } +}; +*/ +/* +static struct ctl_table litmus_dir_table[] = { + { + .procname = "litmus", + .mode = 0555, + .child = cache_table, + }, + { } +}; +*/ + +//static struct ctl_table_header *litmus_sysctls; + + +/* + * Initialzie this proc + */ +static int __init litmus_color_init(void) +{ + int err=0; + printk("Init bankproc.c\n"); +/* + init_variables(); + + printk(KERN_INFO "Registering LITMUS^RT proc color sysctl.\n"); + + litmus_sysctls = register_sysctl_table(litmus_dir_table); + if (!litmus_sysctls) { + printk(KERN_WARNING "Could not register LITMUS^RT color sysctl.\n"); + err = -EFAULT; + goto out; + } + + init_color_groups(); + do_add_pages(); +*/ + printk(KERN_INFO "Registering LITMUS^RT color and bank proc.\n"); +out: + return err; +} + +module_init(litmus_color_init); + diff --git litmus/budget.c litmus/budget.c index 47bf78a..d67f4b3 100644 --- litmus/budget.c +++ litmus/budget.c @@ -1,9 +1,11 @@ #include #include #include +#include #include #include +#include #include @@ -113,4 +115,54 @@ static int __init init_budget_enforcement(void) return 0; } +void litmus_current_budget(lt_t *used_so_far, lt_t *remaining) +{ + struct task_struct *t = current; + unsigned long flags; + s64 delta; + + local_irq_save(flags); + + delta = sched_clock_cpu(smp_processor_id()) - t->se.exec_start; + if (delta < 0) + delta = 0; + + TRACE_CUR("current_budget: sc:%llu start:%llu lt_t:%llu delta:%lld exec-time:%llu rem:%llu\n", + sched_clock_cpu(smp_processor_id()), t->se.exec_start, + litmus_clock(), delta, + tsk_rt(t)->job_params.exec_time, + budget_remaining(t)); + + if (used_so_far) + *used_so_far = tsk_rt(t)->job_params.exec_time + delta; + + if (remaining) { + *remaining = budget_remaining(t); + if (*remaining > delta) + *remaining -= delta; + else + *remaining = 0; + } + + local_irq_restore(flags); +} + +asmlinkage long sys_get_current_budget( + lt_t __user * _expended, + lt_t __user *_remaining) +{ + lt_t expended = 0, remaining = 0; + + if (is_realtime(current)) + litmus->current_budget(&expended, &remaining); + + if (_expended && put_user(expended, _expended)) + return -EFAULT; + + if (_remaining && put_user(remaining, _remaining)) + return -EFAULT; + + return 0; +} + module_init(init_budget_enforcement); diff --git litmus/cache_proc.c litmus/cache_proc.c new file mode 100644 index 0000000..51d0c3d --- /dev/null +++ litmus/cache_proc.c @@ -0,0 +1,1426 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#define UNLOCK_ALL 0x00000000 /* allocation in any way */ +#define LOCK_ALL (~UNLOCK_ALL) +#define MAX_NR_WAYS 16 +#define MAX_NR_COLORS 16 +#define CACHELINE_SIZE 32 +#define INTS_IN_CACHELINE (CACHELINE_SIZE/sizeof(int)) +#define CACHELINES_IN_1KB (1024 / sizeof(cacheline_t)) + +typedef struct cacheline +{ + int line[INTS_IN_CACHELINE]; +} __attribute__((aligned(CACHELINE_SIZE))) cacheline_t; + +void mem_lock(u32 lock_val, int cpu); + +/* + * unlocked_way[i] : allocation can occur in way i + * + * 0 = allocation can occur in the corresponding way + * 1 = allocation cannot occur in the corresponding way + */ +u32 unlocked_way[MAX_NR_WAYS] = { + 0xFFFFFFFE, /* way 0 unlocked */ + 0xFFFFFFFD, + 0xFFFFFFFB, + 0xFFFFFFF7, + 0xFFFFFFEF, /* way 4 unlocked */ + 0xFFFFFFDF, + 0xFFFFFFBF, + 0xFFFFFF7F, + 0xFFFFFEFF, /* way 8 unlocked */ + 0xFFFFFDFF, + 0xFFFFFBFF, + 0xFFFFF7FF, + 0xFFFFEFFF, /* way 12 unlocked */ + 0xFFFFDFFF, + 0xFFFFBFFF, + 0xFFFF7FFF, +}; + +u32 nr_unlocked_way[MAX_NR_WAYS+1] = { + 0x0000FFFF, /* all ways are locked. usable = 0*/ + 0x0000FFFE, /* way ~0 unlocked. usable = 1 */ + 0x0000FFFC, + 0x0000FFF8, + 0x0000FFF0, + 0x0000FFE0, + 0x0000FFC0, + 0x0000FF80, + 0x0000FF00, + 0x0000FE00, + 0x0000FC00, + 0x0000F800, + 0x0000F000, + 0x0000E000, + 0x0000C000, + 0x00008000, + 0x00000000, /* way ~15 unlocked. usable = 16 */ +}; + +u32 way_partition[4] = { + 0xfffffff0, /* cpu0 */ + 0xffffff0f, /* cpu1 */ + 0xfffff0ff, /* cpu2 */ + 0xffff0fff, /* cpu3 */ +}; + +u32 way_partitions[9] = { + 0xffff00ff, /* cpu0 A */ + 0xffff00ff, /* cpu0 B */ + 0xffff00ff, /* cpu1 A */ + 0xffff00ff, /* cpu1 B */ + 0xffff00ff, /* cpu2 A */ + 0xffff00ff, /* cpu2 B */ + 0xffff00ff, /* cpu3 A */ + 0xffff00ff, /* cpu3 B */ + 0xffffff00, /* lv C */ +}; + +u32 prev_lockdown_d_reg[5] = { + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x000000FF, /* share with level-C */ +}; + +u32 prev_lockdown_i_reg[5] = { + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x000000FF, /* share with level-C */ +}; + +u32 prev_lbm_i_reg[8] = { + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + +u32 prev_lbm_d_reg[8] = { + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + +static void __iomem *cache_base; +static void __iomem *lockreg_d; +static void __iomem *lockreg_i; + +static u32 cache_id; + +struct mutex actlr_mutex; +struct mutex l2x0_prefetch_mutex; +struct mutex lockdown_proc; +static u32 way_partition_min; +static u32 way_partition_max; + +static int zero = 0; +static int one = 1; + +static int l1_prefetch_proc; +static int l2_prefetch_hint_proc; +static int l2_double_linefill_proc; +static int l2_data_prefetch_proc; +static int os_isolation; +static int use_part; + +static u32 debug_test_val; +struct mutex debug_mutex; + +u32 lockdown_reg[9] = { + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + + +#define ld_d_reg(cpu) ({ int __cpu = cpu; \ + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_D_BASE + \ + __cpu * L2X0_LOCKDOWN_STRIDE; __v; }) +#define ld_i_reg(cpu) ({ int __cpu = cpu; \ + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_I_BASE + \ + __cpu * L2X0_LOCKDOWN_STRIDE; __v; }) + +int lock_all; +int nr_lockregs; +static raw_spinlock_t cache_lock; +static raw_spinlock_t prefetch_lock; +static void ***flusher_pages = NULL; + +extern void l2c310_flush_all(void); + +static inline void cache_wait_way(void __iomem *reg, unsigned long mask) +{ + /* wait for cache operation by line or way to complete */ + while (readl_relaxed(reg) & mask) + cpu_relax(); +} + +#ifdef CONFIG_CACHE_L2X0 +static inline void cache_wait(void __iomem *reg, unsigned long mask) +{ + /* cache operations by line are atomic on PL310 */ +} +#else +#define cache_wait cache_wait_way +#endif + +static inline void cache_sync(void) +{ + void __iomem *base = cache_base; + + writel_relaxed(0, base + L2X0_CACHE_SYNC); + cache_wait(base + L2X0_CACHE_SYNC, 1); +} + +static void print_lockdown_registers(int cpu) +{ + int i; + //for (i = 0; i < nr_lockregs; i++) { + for (i = 0; i < 4; i++) { + printk("P%d Lockdown Data CPU %2d: 0x%04x\n", cpu, + i, readl_relaxed(ld_d_reg(i))); + printk("P%d Lockdown Inst CPU %2d: 0x%04x\n", cpu, + i, readl_relaxed(ld_i_reg(i))); + } +} + +static void test_lockdown(void *ignore) +{ + int i, cpu; + + cpu = smp_processor_id(); + printk("Start lockdown test on CPU %d.\n", cpu); + + for (i = 0; i < nr_lockregs; i++) { + printk("CPU %2d data reg: 0x%8p\n", i, ld_d_reg(i)); + printk("CPU %2d inst reg: 0x%8p\n", i, ld_i_reg(i)); + } + + printk("Lockdown initial state:\n"); + print_lockdown_registers(cpu); + printk("---\n"); + + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(1, ld_d_reg(i)); + writel_relaxed(2, ld_i_reg(i)); + } + printk("Lockdown all data=1 instr=2:\n"); + print_lockdown_registers(cpu); + printk("---\n"); + + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed((1 << i), ld_d_reg(i)); + writel_relaxed(((1 << 8) >> i), ld_i_reg(i)); + } + printk("Lockdown varies:\n"); + print_lockdown_registers(cpu); + printk("---\n"); + + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(UNLOCK_ALL, ld_d_reg(i)); + writel_relaxed(UNLOCK_ALL, ld_i_reg(i)); + } + printk("Lockdown all zero:\n"); + print_lockdown_registers(cpu); + + printk("End lockdown test.\n"); +} + +void litmus_setup_lockdown(void __iomem *base, u32 id) +{ + cache_base = base; + cache_id = id; + lockreg_d = cache_base + L2X0_LOCKDOWN_WAY_D_BASE; + lockreg_i = cache_base + L2X0_LOCKDOWN_WAY_I_BASE; + + if (L2X0_CACHE_ID_PART_L310 == (cache_id & L2X0_CACHE_ID_PART_MASK)) { + nr_lockregs = 8; + } else { + printk("Unknown cache ID!\n"); + nr_lockregs = 1; + } + + mutex_init(&actlr_mutex); + mutex_init(&l2x0_prefetch_mutex); + mutex_init(&lockdown_proc); + mutex_init(&debug_mutex); + raw_spin_lock_init(&cache_lock); + raw_spin_lock_init(&prefetch_lock); + + test_lockdown(NULL); +} + +int way_partition_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + unsigned long flags; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write) { + printk("Way-partition settings:\n"); + for (i = 0; i < 9; i++) { + printk("0x%08X\n", way_partitions[i]); + } + for (i = 0; i < 4; i++) { + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + } + + local_irq_save(flags); + print_lockdown_registers(smp_processor_id()); + l2c310_flush_all(); + local_irq_restore(flags); +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int lock_all_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + unsigned long flags; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write && lock_all == 1) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } +/* + for (i = 0; i < nr_lockregs; i++) { + barrier(); + mem_lock(LOCK_ALL, i); + barrier(); + //writel_relaxed(nr_unlocked_way[0], ld_d_reg(i)); + //writel_relaxed(nr_unlocked_way[0], ld_i_reg(i)); + } +*/ + } + if (write && lock_all == 0) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + + } + + local_irq_save(flags); + print_lockdown_registers(smp_processor_id()); + l2c310_flush_all(); + local_irq_restore(flags); +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +void cache_lockdown(u32 lock_val, int cpu) +{ + __asm__ __volatile__ ( +" str %[lockval], [%[dcachereg]]\n" +" str %[lockval], [%[icachereg]]\n" + : + : [dcachereg] "r" (ld_d_reg(cpu)), + [icachereg] "r" (ld_i_reg(cpu)), + [lockval] "r" (lock_val) + : "cc"); +} + +void do_partition(enum crit_level lv, int cpu) +{ + u32 regs; + unsigned long flags; + + if (lock_all || !use_part) + return; + raw_spin_lock_irqsave(&cache_lock, flags); + switch(lv) { + case CRIT_LEVEL_A: + regs = ~way_partitions[cpu*2]; + regs &= 0x0000ffff; + break; + case CRIT_LEVEL_B: + regs = ~way_partitions[cpu*2+1]; + regs &= 0x0000ffff; + break; + case CRIT_LEVEL_C: + case NUM_CRIT_LEVELS: + regs = ~way_partitions[8]; + regs &= 0x0000ffff; + break; + default: + BUG(); + + } + barrier(); + + writel_relaxed(regs, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + cpu * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(regs, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + cpu * L2X0_LOCKDOWN_STRIDE); + barrier(); + + raw_spin_unlock_irqrestore(&cache_lock, flags); +} + +void lock_cache(int cpu, u32 val) +{ + unsigned long flags; + + local_irq_save(flags); + if (val != 0xffffffff) { + writel_relaxed(val, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + cpu * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(val, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + cpu * L2X0_LOCKDOWN_STRIDE); + } + else { + int i; + for (i = 0; i < 4; i++) + do_partition(CRIT_LEVEL_A, i); + } + local_irq_restore(flags); +} + +int use_part_proc_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + + printk("USE_PART HANDLER = %d\n", use_part); + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int os_isolation_proc_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + + printk("OS_ISOLATION HANDLER = %d\n", os_isolation); + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int lockdown_reg_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + } + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int lockdown_global_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + } + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +void inline enter_irq_mode(void) +{ + int cpu = smp_processor_id(); + + if (os_isolation == 0) + return; + prev_lockdown_i_reg[cpu] = readl_relaxed(ld_i_reg(cpu)); + prev_lockdown_d_reg[cpu] = readl_relaxed(ld_d_reg(cpu)); + writel_relaxed(way_partitions[8], ld_i_reg(cpu)); + writel_relaxed(way_partitions[8], ld_d_reg(cpu)); +} + +void inline exit_irq_mode(void) +{ + int cpu = smp_processor_id(); + + if (os_isolation == 0) + return; + + writel_relaxed(prev_lockdown_i_reg[cpu], ld_i_reg(cpu)); + writel_relaxed(prev_lockdown_d_reg[cpu], ld_d_reg(cpu)); +} + +/* Operate on the Cortex-A9's ACTLR register */ +#define ACTLR_L2_PREFETCH_HINT (1 << 1) +#define ACTLR_L1_PREFETCH (1 << 2) + +/* + * Change the ACTLR. + * @mode - If 1 (0), set (clear) the bit given in @mask in the ACTLR. + * @mask - A mask in which one bit is set to operate on the ACTLR. + */ +static void actlr_change(int mode, int mask) +{ + u32 orig_value, new_value, reread_value; + + if (0 != mode && 1 != mode) { + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n", + __FUNCTION__); + return; + } + + /* get the original value */ + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (orig_value)); + + if (0 == mode) + new_value = orig_value & ~(mask); + else + new_value = orig_value | mask; + + asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (new_value)); + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (reread_value)); + + printk("ACTLR: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n", + orig_value, new_value, reread_value); +} + +int litmus_l1_prefetch_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&actlr_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + + if (!ret && write) { + mode = *((int*)table->data); + actlr_change(mode, ACTLR_L1_PREFETCH); + } + mutex_unlock(&actlr_mutex); + + return ret; +} + +int litmus_l2_prefetch_hint_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&actlr_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + mode = *((int*)table->data); + actlr_change(mode, ACTLR_L2_PREFETCH_HINT); + } + mutex_unlock(&actlr_mutex); + + return ret; +} + + +/* Operate on the PL-310's Prefetch Control Register, L310_PREFETCH_CTRL */ +#define L2X0_PREFETCH_DOUBLE_LINEFILL (1 << 30) +#define L2X0_PREFETCH_INST_PREFETCH (1 << 29) +#define L2X0_PREFETCH_DATA_PREFETCH (1 << 28) +static void l2x0_prefetch_change(int mode, int mask) +{ + u32 orig_value, new_value, reread_value; + + if (0 != mode && 1 != mode) { + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n", + __FUNCTION__); + return; + } + + orig_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL); + + if (0 == mode) + new_value = orig_value & ~(mask); + else + new_value = orig_value | mask; + + writel_relaxed(new_value, cache_base + L310_PREFETCH_CTRL); + reread_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL); + + printk("l2x0 prefetch: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n", + orig_value, new_value, reread_value); +} + +int litmus_l2_double_linefill_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&l2x0_prefetch_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + mode = *((int*)table->data); + l2x0_prefetch_change(mode, L2X0_PREFETCH_DOUBLE_LINEFILL); + } + mutex_unlock(&l2x0_prefetch_mutex); + + return ret; +} + +int litmus_l2_data_prefetch_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&l2x0_prefetch_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + mode = *((int*)table->data); + l2x0_prefetch_change(mode, L2X0_PREFETCH_DATA_PREFETCH|L2X0_PREFETCH_INST_PREFETCH); + } + mutex_unlock(&l2x0_prefetch_mutex); + + return ret; +} + +extern void *msgvaddr; + +int do_measure(void) { + lt_t t1, t2; + int i; + + barrier(); + t1 = litmus_clock(); + color_read_in_mem_lock(0xFFFF7FFF, 0xFFFF8000, msgvaddr, msgvaddr + 65536); + t2 = litmus_clock() - t1; + barrier(); + + for (i = 0; i < 8; i++) { + cache_lockdown(0xFFFF8000, i); + } + printk("mem read time %lld\n", t2); + + return 0; +} + +int debug_test_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret; + + //mutex_lock(&debug_mutex); + //ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (write) { + ret = do_measure(); + } + + return ret; +} + + +int do_perf_test_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + +int setup_flusher_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + +static struct ctl_table cache_table[] = +{ + { + .procname = "C0_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[0], + .maxlen = sizeof(way_partitions[0]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C0_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[1], + .maxlen = sizeof(way_partitions[1]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C1_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[2], + .maxlen = sizeof(way_partitions[2]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C1_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[3], + .maxlen = sizeof(way_partitions[3]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C2_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[4], + .maxlen = sizeof(way_partitions[4]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C2_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[5], + .maxlen = sizeof(way_partitions[5]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C3_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[6], + .maxlen = sizeof(way_partitions[6]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C3_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[7], + .maxlen = sizeof(way_partitions[7]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "Call_LC_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[8], + .maxlen = sizeof(way_partitions[8]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lock_all", + .mode = 0666, + .proc_handler = lock_all_handler, + .data = &lock_all, + .maxlen = sizeof(lock_all), + .extra1 = &zero, + .extra2 = &one, + }, + { + .procname = "l1_prefetch", + .mode = 0644, + .proc_handler = litmus_l1_prefetch_proc_handler, + .data = &l1_prefetch_proc, + .maxlen = sizeof(l1_prefetch_proc), + }, + { + .procname = "l2_prefetch_hint", + .mode = 0644, + .proc_handler = litmus_l2_prefetch_hint_proc_handler, + .data = &l2_prefetch_hint_proc, + .maxlen = sizeof(l2_prefetch_hint_proc), + }, + { + .procname = "l2_double_linefill", + .mode = 0644, + .proc_handler = litmus_l2_double_linefill_proc_handler, + .data = &l2_double_linefill_proc, + .maxlen = sizeof(l2_double_linefill_proc), + }, + { + .procname = "l2_data_prefetch", + .mode = 0644, + .proc_handler = litmus_l2_data_prefetch_proc_handler, + .data = &l2_data_prefetch_proc, + .maxlen = sizeof(l2_data_prefetch_proc), + }, + { + .procname = "os_isolation", + .mode = 0644, + .proc_handler = os_isolation_proc_handler, + .data = &os_isolation, + .maxlen = sizeof(os_isolation), + }, + { + .procname = "use_part", + .mode = 0644, + .proc_handler = use_part_proc_handler, + .data = &use_part, + .maxlen = sizeof(use_part), + }, + { + .procname = "do_perf_test", + .mode = 0644, + .proc_handler = do_perf_test_proc_handler, + }, + { + .procname = "setup_flusher", + .mode = 0644, + .proc_handler = setup_flusher_proc_handler, + }, + { + .procname = "lockdown_reg_0", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[0], + .maxlen = sizeof(lockdown_reg[0]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_reg_1", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[1], + .maxlen = sizeof(lockdown_reg[1]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_reg_2", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[2], + .maxlen = sizeof(lockdown_reg[2]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_reg_3", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[3], + .maxlen = sizeof(lockdown_reg[3]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_regs", + .mode = 0644, + .proc_handler = lockdown_global_handler, + .data = &lockdown_reg[8], + .maxlen = sizeof(lockdown_reg[8]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "debug_test", + .mode = 0644, + .proc_handler = debug_test_handler, + }, + { } +}; + +static struct ctl_table litmus_dir_table[] = { + { + .procname = "litmus", + .mode = 0555, + .child = cache_table, + }, + { } +}; + +u32 color_read_in_mem(u32 lock_val, u32 unlock_val, void *start, void *end) +{ + u32 v = 0; + + __asm__ __volatile__ ( +" .align 5\n" +" str %[lockval], [%[cachereg]]\n" +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n" +" cmp %[end], %[addr] @ subtracts addr from end\n" +" bgt 1b\n @ read more, if necessary\n" + : [addr] "+r" (start), + [val] "+r" (v) + : [end] "r" (end), +#ifdef CONFIG_CACHE_L2X0 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())), +#else + [cachereg] "r" (lockreg_d), +#endif + [lockval] "r" (lock_val) + : "cc"); + + return v; +} + + +/* + * Prefetch by reading the first word of each cache line in a page. + * + * @lockdown_reg: address of the lockdown register to write + * @lock_val: value to be written to @lockdown_reg + * @unlock_val: will unlock the cache to this value + * @addr: start address to be prefetched + * @end_addr: end address to prefetch (exclusive) + * + * Assumes: addr < end_addr AND addr != end_addr + */ +u32 color_read_in_mem_lock(u32 lock_val, u32 unlock_val, void *start, void *end) +{ +#ifndef CONFIG_CACHE_L2X0 + unsigned long flags; +#endif + u32 v = 0; + +#ifndef CONFIG_CACHE_L2X0 + raw_spin_lock_irqsave(&prefetch_lock, flags); +#endif + + __asm__ __volatile__ ( +" .align 5\n" +" str %[lockval], [%[cachereg]]\n" +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n" +" cmp %[end], %[addr] @ subtracts addr from end\n" +" bgt 1b\n @ read more, if necessary\n" +" str %[unlockval], [%[cachereg]]\n" + : [addr] "+r" (start), + [val] "+r" (v) + : [end] "r" (end), +#ifdef CONFIG_CACHE_L2X0 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())), +#else + [cachereg] "r" (lockreg_d), +#endif + [lockval] "r" (lock_val), + [unlockval] "r" (unlock_val) + : "cc"); + +#ifndef CONFIG_CACHE_L2X0 + raw_spin_unlock_irqrestore(&prefetch_lock, flags); +#endif + + return v; +} + +static long update_timeval(struct timespec lhs, struct timespec rhs) +{ + long val; + struct timespec ts; + + ts = timespec_sub(rhs, lhs); + val = ts.tv_sec*NSEC_PER_SEC + ts.tv_nsec; + + return val; +} + +extern void v7_flush_kern_dcache_area(void *, size_t); +extern void v7_flush_kern_cache_all(void); +/* + * Ensure that this page is not in the L1 or L2 cache. + * Since the L1 cache is VIPT and the L2 cache is PIPT, we can use either the + * kernel or user vaddr. + */ +void color_flush_page(void *vaddr, size_t size) +{ + v7_flush_kern_dcache_area(vaddr, size); + //v7_flush_kern_cache_all(); +} + +extern struct page* get_colored_page(unsigned long color); + +int setup_flusher_array(void) +{ + int color, way, ret = 0; + struct page *page; + + if (flusher_pages != NULL) + goto out; + + flusher_pages = (void***) kmalloc(MAX_NR_WAYS + * sizeof(*flusher_pages), GFP_KERNEL); + if (!flusher_pages) { + printk(KERN_WARNING "No memory for flusher array!\n"); + ret = -EINVAL; + goto out; + } + for (way = 0; way < MAX_NR_WAYS; way++) { + void **flusher_color_arr; + flusher_color_arr = (void**) kmalloc(sizeof(**flusher_pages) + * MAX_NR_COLORS, GFP_KERNEL); + if (!flusher_color_arr) { + printk(KERN_WARNING "No memory for flusher array!\n"); + ret = -ENOMEM; + goto out_free; + } + + flusher_pages[way] = flusher_color_arr; + for (color = 0; color < MAX_NR_COLORS; color++) { + int node; + node = color + 112; // populate from bank 7 + page = get_colored_page(node); + if (!page) { + printk(KERN_WARNING "no more colored pages\n"); + ret = -EINVAL; + goto out_free; + } + flusher_pages[way][color] = page_address(page); + if (!flusher_pages[way][color]) { + printk(KERN_WARNING "bad page address\n"); + ret = -EINVAL; + goto out_free; + } + } + } + +out: + return ret; +out_free: + for (way = 0; way < MAX_NR_WAYS; way++) { + for (color = 0; color < MAX_NR_COLORS; color++) { + /* not bothering to try and give back colored pages */ + } + kfree(flusher_pages[way]); + } + kfree(flusher_pages); + flusher_pages = NULL; + return ret; +} + +void flush_cache(int all) +{ + int way, color, cpu; + unsigned long flags; + + raw_spin_lock_irqsave(&cache_lock, flags); + cpu = raw_smp_processor_id(); + + prev_lbm_i_reg[cpu] = readl_relaxed(ld_i_reg(cpu)); + prev_lbm_d_reg[cpu] = readl_relaxed(ld_d_reg(cpu)); + for (way=0;way>9); + } +/* + getnstimeofday(&before); + barrier(); + for (i = 0; i < TRIALS; i++) { + color_flush_page(vaddr, PAGE_SIZE*num_pages); + } + barrier(); + getnstimeofday(&after); + time = update_timeval(before, after); + printk("Average for flushes without re-reading: %ld\n", time / TRIALS); + flush_time = time / TRIALS; + + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + + barrier(); + getnstimeofday(&before); + barrier(); + for (i = 0; i < TRIALS; i++) { + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + } + barrier(); + getnstimeofday(&after); + time = update_timeval(before, after); + printk("Average for read from cache: %ld\n", time / TRIALS); + + getnstimeofday(&before); + barrier(); + for (i = 0; i < TRIALS; i++) { + color_read_in_mem(nr_unlocked_way[2], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + color_flush_page(vaddr, PAGE_SIZE*num_pages); + } + barrier(); + getnstimeofday(&after); + time = update_timeval(before, after); + printk("Average for read from mem: %ld (%ld)\n", time / TRIALS - flush_time, time / TRIALS); + + // write in locked way + color_read_in_mem_lock(nr_unlocked_way[2], LOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + for (i = 0; i < PAGE_SIZE*num_pages/sizeof(u32); i++) { + data[i] = i%63353; + } + // read + barrier(); + getnstimeofday(&before); + barrier(); + for (i = 0; i < TRIALS; i++) { + color_read_in_mem(unlocked_way[0], UNLOCK_ALL, vaddr, vaddr + PAGE_SIZE*num_pages); + } + barrier(); + getnstimeofday(&after); + time = update_timeval(before, after); + printk("Average for read in after write: %ld\n", time / TRIALS); + +*/ + //free_page((unsigned long)vaddr); + free_pages((unsigned long)vaddr, order); + free_pages((unsigned long)vaddr2, order); + + do_gettimeofday(&t); + t3 = get_cycles(); + t4 = litmus_clock(); + time_to_tm(t.tv_sec, 0, &broken); + printk(KERN_INFO "gettimeofday() - %d:%d:%d:%ld\n", broken.tm_hour, broken.tm_min, broken.tm_sec, t.tv_usec); + printk(KERN_INFO "get_cycles() %llu\n", t3); + printk(KERN_INFO "litmus_clock() %llu\n", t4); + + return 0; +} + +int do_perf_test_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret = 0; + + if (write) { + ret = perf_test(); + } + + return ret; +} + +int setup_flusher_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret = -EINVAL; + + if (write && flusher_pages == NULL) { + ret = setup_flusher_array(); + printk(KERN_INFO "setup flusher return: %d\n", ret); + + } + else if (flusher_pages) { + printk(KERN_INFO "flusher_pages is already set!\n"); + ret = 0; + } + + return ret; +} + +static struct ctl_table_header *litmus_sysctls; + +static int __init litmus_sysctl_init(void) +{ + int ret = 0; + + printk(KERN_INFO "Registering LITMUS^RT proc sysctl.\n"); + litmus_sysctls = register_sysctl_table(litmus_dir_table); + if (!litmus_sysctls) { + printk(KERN_WARNING "Could not register LITMUS^RT sysctl.\n"); + ret = -EFAULT; + goto out; + } + + way_partition_min = 0x00000000; + way_partition_max = 0x0000FFFF; + +out: + return ret; +} + +module_init(litmus_sysctl_init); diff --git litmus/color_shm.c litmus/color_shm.c new file mode 100644 index 0000000..0842623 --- /dev/null +++ litmus/color_shm.c @@ -0,0 +1,402 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define DEV_NAME "litmus/color_shm" + +/* Major number assigned to our device. + * Refer Documentation/devices.txt */ +#define SHM_MAJOR 240 +#define MAX_COLORED_PAGE 256 +#define NUM_BANKS 8 +#define NUM_COLORS 16 + +static struct mutex dev_lock; +static int bypass_cache; + +struct color_ioctl_cmd { + unsigned int color; + unsigned int bank; +}; + +struct color_ioctl_offset { + unsigned long offset; + int lock; +}; + +#define SET_COLOR_SHM_CMD _IOW(SHM_MAJOR, 0x1, struct color_ioctl_cmd) +#define SET_COLOR_SHM_OFFSET _IOW(SHM_MAJOR, 0x2, struct color_ioctl_offset) + +struct color_ioctl_cmd color_param; +struct color_ioctl_offset color_offset; + +static int mmap_common_checks(struct vm_area_struct *vma) +{ + /* you can only map the "first" page */ + if (vma->vm_pgoff != 0) + return -EINVAL; + + return 0; +} + +static void mmap_common_vma_flags(struct vm_area_struct *vma) +{ + /* This mapping should not be kept across forks, + * cannot be expanded, and is not a "normal" page. */ + //vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_IO | VM_SHARED | VM_MAYSHARE; + vma->vm_flags |= VM_SHARED | VM_MAYSHARE | VM_LOCKED; + + /* We don't want the first write access to trigger a "minor" page fault + * to mark the page as dirty. This is transient, private memory, we + * don't care if it was touched or not. __S011 means RW access, but not + * execute, and avoids copy-on-write behavior. + * See protection_map in mmap.c. */ + vma->vm_page_prot = PAGE_SHARED; +} + +#define vma_nr_pages(vma) \ + ({unsigned long v = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); v;}) + +extern struct page* get_colored_page(unsigned long color); + +static int do_map_colored_page(struct vm_area_struct *vma, + const unsigned long addr, + const unsigned long color_no) +{ + int err = 0; + unsigned long offset = 2048; + + struct page *page = get_colored_page(color_no); + + if (!page) { + printk(KERN_INFO "Could not get page with color %lu.\n", + color_no); + err = -ENOMEM; + goto out; + } + + printk(KERN_INFO "vma: %p addr: 0x%lx color_no: %lu\n", + vma, addr, color_no); + + printk(KERN_INFO "vm_start: %lu vm_end: %lu\n", + vma->vm_start, vma->vm_end); + + printk(KERN_INFO "inserting page (pa: 0x%lx) at vaddr: 0x%lx " + "flags: 0x%lx prot: 0x%lx\n", + page_to_phys(page), addr, + vma->vm_flags, pgprot_val(vma->vm_page_prot)); + + + err = vm_insert_page(vma, addr, page); + if (err) { + printk(KERN_INFO "vm_insert_page() failed (%d)\n", err); + err = -EINVAL; + goto out; + } +out: + return err; +} + +static int do_map_colored_pages(struct vm_area_struct *vma) +{ + const unsigned long nr_pages = vma_nr_pages(vma); + unsigned long nr_mapped; + int i, start_bank = -1, start_color = -1; + int cur_bank = -1, cur_color = -1, err = 0; + int colors[16] = {0}, banks[8] = {0}; + + if (bypass_cache == 1) + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + for (i = 0; i < NUM_BANKS; i++) { + if (((color_param.bank >> i)&0x1) == 1) + banks[i] = 1; + } + + for (i = 0; i < NUM_COLORS; i++) { + if (((color_param.color >> i)&0x1) == 1) + colors[i] = 1; + } + + for (i = 0; i < NUM_BANKS; i++) { + if (banks[i] == 1) { + start_bank = i; + break; + } + } + for (i = 0; i < NUM_COLORS; i++) { + if (colors[i] == 1) { + start_color = i; + break; + } + } + + cur_bank = start_bank; + cur_color = start_color; + + for (i = 0; i < NUM_BANKS; i++) { + printk(KERN_INFO "BANK[%d] = %d\n", i, banks[i]); + } + printk(KERN_INFO "cur_bank = %d\n", cur_bank); + for (i = 0; i < NUM_COLORS; i++) { + printk(KERN_INFO "COLOR[%d] = %d\n", i, colors[i]); + } + printk(KERN_INFO "cur_color = %d\n", cur_color); + + + TRACE_CUR("allocating %lu pages (flags:%lx prot:%lx)\n", + nr_pages, vma->vm_flags, pgprot_val(vma->vm_page_prot)); + + for (nr_mapped = 0; nr_mapped < nr_pages; nr_mapped++) { + const unsigned long addr = vma->vm_start + (nr_mapped << PAGE_SHIFT); + const unsigned long color_no = cur_bank*NUM_COLORS + cur_color; + + err = do_map_colored_page(vma, addr, color_no); + printk(KERN_INFO "mapped bank[%d], color[%d], color_no = %lu at 0x%lx\n", + cur_bank, cur_color, color_no, addr); + if (err) { + TRACE_CUR("Could not map colored page set.\n"); + err = -EINVAL; + goto out; + } + do { + cur_color++; + } while(colors[cur_color] == 0); + + if (cur_color >= NUM_COLORS) { + do { + cur_bank++; + } while(banks[cur_bank] == 0); + cur_color = start_color; + } + + if (cur_bank >= NUM_BANKS) { + cur_bank = start_bank; + } + } + TRACE_CUR("Successfully mapped %lu pages.\n", nr_mapped); + out: + return err; +} + +static int map_colored_pages(struct vm_area_struct *vma) +{ + int err = 0; + + printk(KERN_INFO "User requests %lu pages.\n", vma_nr_pages(vma)); + if (MAX_COLORED_PAGE < vma_nr_pages(vma)) { + TRACE_CUR("Max page request %lu but want %lu.\n", + MAX_COLORED_PAGE, vma_nr_pages(vma)); + err = -EINVAL; + goto out; + } + err = do_map_colored_pages(vma); +out: + return err; +} + +static void litmus_color_shm_vm_close(struct vm_area_struct *vma) +{ + + TRACE_CUR("flags=0x%lx prot=0x%lx\n", + vma->vm_flags, pgprot_val(vma->vm_page_prot)); + + TRACE_CUR("%p:%p vma:%p vma->vm_private_data:%p closed.\n", + (void*) vma->vm_start, (void*) vma->vm_end, vma, + vma->vm_private_data); + +} + +static int litmus_color_shm_vm_fault(struct vm_area_struct *vma, + struct vm_fault *vmf) +{ + /* This function should never be called, since + * all pages should have been mapped by mmap() + * already. */ + TRACE_CUR("flags=0x%lx (off:%ld)\n", vma->vm_flags, vmf->pgoff); + printk(KERN_INFO "flags=0x%lx (off:%ld)\n", vma->vm_flags, vmf->pgoff); + + printk(KERN_INFO "Page fault in color ctrl page! prot=0x%lx\n", pgprot_val(vma->vm_page_prot)); + + return VM_FAULT_SIGBUS; +} + +static struct vm_operations_struct litmus_color_shm_vm_ops = { + .close = litmus_color_shm_vm_close, + .fault = litmus_color_shm_vm_fault, +}; + +static int litmus_color_shm_mmap(struct file *filp, struct vm_area_struct *vma) +{ + int err = 0; + + printk(KERN_INFO "mmap called\n"); + + if (color_param.color == 0x00000000 || color_param.bank == 0x00000000) { + printk(KERN_INFO "color_info not set.\n"); + return -EINVAL; + } + if (color_offset.offset == 0xffffffff || color_offset.lock == -1) { + printk(KERN_INFO "color_offset not set.\n"); + return -EINVAL; + } + + err = mmap_common_checks(vma); + if (err) { + TRACE_CUR("failed mmap common checks\n"); + goto out; + } + + vma->vm_ops = &litmus_color_shm_vm_ops; + mmap_common_vma_flags(vma); + + err = map_colored_pages(vma); + + TRACE_CUR("flags=0x%lx prot=0x%lx\n", vma->vm_flags, + pgprot_val(vma->vm_page_prot)); +out: + color_param.color = 0x00000000; + color_param.bank = 0x00000000; + color_offset.offset = 0xffffffff; + color_offset.lock = -1; + + return err; + +} + +static long litmus_color_shm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + long err = -ENOIOCTLCMD; + struct color_ioctl_cmd color_info; + struct color_ioctl_offset color_off; + + printk(KERN_INFO "color_shm ioctl\n"); + + if (_IOC_TYPE(cmd) != SHM_MAJOR) + return -ENOTTY; + + + switch (cmd) { + case SET_COLOR_SHM_CMD: + + err = copy_from_user(&color_info, (void*)arg, sizeof(struct color_ioctl_cmd)); + + color_param.color = color_info.color; + color_param.bank = color_info.bank; + printk(KERN_INFO "COLOR = %x\n", color_param.color); + printk(KERN_INFO "BANK = %x\n", color_param.bank); + err = 0; + break; + case SET_COLOR_SHM_OFFSET: + err = copy_from_user(&color_off, (void*)arg, sizeof(struct color_ioctl_offset)); + + color_offset.offset = color_off.offset; + color_offset.lock = color_off.lock; + printk(KERN_INFO "OFFSET = %x\n", color_offset.offset); + printk(KERN_INFO "LOCK = %d\n", color_offset.lock); + err = 0; + break; + + default: + printk(KERN_INFO "Invalid IOCTL CMD\n"); + err = -EINVAL; + } + + return err; +} + +static struct file_operations litmus_color_shm_fops = { + .owner = THIS_MODULE, + .mmap = litmus_color_shm_mmap, + .unlocked_ioctl = litmus_color_shm_ioctl, +}; + +static struct miscdevice litmus_color_shm_dev = { + .name = DEV_NAME, + .minor = MISC_DYNAMIC_MINOR, + .fops = &litmus_color_shm_fops, +}; + +struct mutex bypass_mutex; + +int bypass_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&bypass_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + printk(KERN_INFO "shm_bypass = %d\n", bypass_cache); + mutex_unlock(&bypass_mutex); + + return ret; +} + +static int zero = 0; +static int one = 1; + +static struct ctl_table cache_table[] = +{ + { + .procname = "shm_bypass", + .mode = 0666, + .proc_handler = bypass_proc_handler, + .data = &bypass_cache, + .maxlen = sizeof(bypass_cache), + .extra1 = &zero, + .extra2 = &one, + }, + { } +}; + +static struct ctl_table litmus_dir_table[] = { + { + .procname = "litmus", + .mode = 0555, + .child = cache_table, + }, + { } +}; + +static struct ctl_table_header *litmus_sysctls; + +static int __init init_color_shm_devices(void) +{ + int err; + + printk(KERN_INFO "Registering LITMUS^RT color_shm devices.\n"); + litmus_sysctls = register_sysctl_table(litmus_dir_table); + if (!litmus_sysctls) { + printk(KERN_WARNING "Could not register LITMUS^RT color_shm sysctl.\n"); + err = -EFAULT; + } + + mutex_init(&dev_lock); + mutex_init(&bypass_mutex); + color_param.color = 0x00000000; + color_param.bank = 0x00000000; + color_offset.offset = 0xffffffff; + color_offset.lock = -1; + bypass_cache = 0; + err = misc_register(&litmus_color_shm_dev); + + return err; +} + +static void __exit exit_color_shm_devices(void) +{ + misc_deregister(&litmus_color_shm_dev); + printk(KERN_INFO "Unregistering %s device.\n", DEV_NAME); +} + +module_init(init_color_shm_devices); +module_exit(exit_color_shm_devices); \ No newline at end of file diff --git litmus/litmus.c litmus/litmus.c index db5ce0e9..e6d1ae9 100644 --- litmus/litmus.c +++ litmus/litmus.c @@ -14,6 +14,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include @@ -21,6 +26,10 @@ #include #include #include +#include +#include +#include +#include #ifdef CONFIG_SCHED_CPU_AFFINITY #include @@ -31,6 +40,8 @@ #include #endif +extern void l2c310_flush_all(void); + /* Number of RT tasks that exist in the system */ atomic_t rt_task_count = ATOMIC_INIT(0); @@ -160,6 +171,14 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param) pid, tp.budget_policy); goto out_unlock; } +#ifdef CONFIG_PGMRT_SUPPORT + if (tp.pgm_type < PGM_NOT_A_NODE || tp.pgm_type > PGM_INTERNAL) { + printk(KERN_INFO "litmus: real-time task %d rejected " + "because of unknown PGM node type specified (%d)\n", + pid, tp.pgm_type); + goto out_unlock; + } +#endif target->rt_param.task_params = tp; @@ -314,6 +333,378 @@ asmlinkage long sys_null_call(cycles_t __user *ts) return ret; } +asmlinkage long sys_reservation_create(int type, void __user *config) +{ + return litmus->reservation_create(type, config); +} + +asmlinkage long sys_reservation_destroy(unsigned int reservation_id, int cpu) +{ + return litmus->reservation_destroy(reservation_id, cpu); +} + +// This Address Decoding is used in imx6-sabredsd platform +#define NUM_BANKS 8 +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 + +#define NUM_COLORS 16 +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 +/* Decoding page color, 0~15 */ +static inline unsigned int page_color(struct page *page) +{ + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT); +} + +/* Decoding page bank number, 0~7 */ +static inline unsigned int page_bank(struct page *page) +{ + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT); +} + + +static unsigned long color_mask; + +extern int isolate_lru_page(struct page *page); +extern void putback_movable_page(struct page *page); +extern struct page *new_alloc_page(struct page *page, unsigned long node, int **x); + +static struct page *alloc_colored_page(struct page *page, unsigned long node, int **result) +{ + struct page *newpage; + gfp_t gfp_mask; + + gfp_mask = GFP_HIGHUSER_MOVABLE; + if (node != 8) + gfp_mask |= GFP_COLOR; + + newpage = alloc_pages(gfp_mask, 0); + + return newpage; +} + +#define INVALID_PFN (0xffffffff) +LIST_HEAD(shared_lib_pages); + +EXPORT_SYMBOL(shared_lib_pages); + +/* Reallocate pages of a task + * Private pages - Migrate to a new page. + * Shared pages - Use a replica. Make a replica if necessary. + * @cpu : CPU id of the calling task + * returns the number of pages that is not moved. + */ +asmlinkage long sys_set_page_color(int cpu) +{ + long ret = 0; + struct vm_area_struct *vma_itr = NULL; + int nr_pages = 0, nr_shared_pages = 0, nr_failed = 0, nr_not_migrated = 0; + unsigned long node; + enum crit_level lv; + struct mm_struct *mm; + + LIST_HEAD(pagelist); + LIST_HEAD(task_shared_pagelist); + + migrate_prep(); + + /* Find the current mm_struct */ + rcu_read_lock(); + get_task_struct(current); + rcu_read_unlock(); + mm = get_task_mm(current); + put_task_struct(current); + + down_read(&mm->mmap_sem); + + vma_itr = mm->mmap; + /* Iterate all vm_area_struct */ + while (vma_itr != NULL) { + unsigned int num_pages = 0, i; + struct page *old_page = NULL; + int pages_in_vma = 0; + + num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE; + /* Traverse all pages in vm_area_struct */ + for (i = 0; i < num_pages; i++) { + old_page = follow_page(vma_itr, vma_itr->vm_start + PAGE_SIZE*i, FOLL_GET|FOLL_SPLIT); + + if (IS_ERR(old_page)) + continue; + if (!old_page) + continue; + + if (PageReserved(old_page)) { + TRACE("Reserved Page!\n"); + put_page(old_page); + continue; + } + TRACE_TASK(current, "addr: %08x, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-"); + pages_in_vma++; + + /* Conditions for replicable pages */ + if (page_count(old_page) > 2 && vma_itr->vm_file != NULL && !(vma_itr->vm_flags&VM_WRITE)) { + struct shared_lib_page *lib_page; + int is_exist = 0; + + /* Update PSL (Per-core shared library (master)) list */ + /* Check if this page is in the PSL list */ + rcu_read_lock(); + list_for_each_entry(lib_page, &shared_lib_pages, list) + { + if (page_to_pfn(old_page) == lib_page->master_pfn) { + is_exist = 1; + break; + } + } + rcu_read_unlock(); + + if (is_exist == 0) { + int cpu_i; + lib_page = kmalloc(sizeof(struct shared_lib_page), GFP_KERNEL); + lib_page->master_page = old_page; + lib_page->master_pfn = page_to_pfn(old_page); + for (cpu_i = 0; cpu_i < NR_CPUS+1; cpu_i++) { + lib_page->r_page[cpu_i] = NULL; + lib_page->r_pfn[cpu_i] = INVALID_PFN; + } + list_add_tail(&lib_page->list, &shared_lib_pages); + } + + /* add to task_shared_pagelist */ + ret = isolate_lru_page(old_page); + if (!ret) { + list_add_tail(&old_page->lru, &task_shared_pagelist); + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page)); + nr_shared_pages++; + } else { + TRACE_TASK(current, "isolate_lru_page for a shared page failed\n"); + nr_failed++; + } + put_page(old_page); + } + else { + ret = isolate_lru_page(old_page); + if (!ret) { + list_add_tail(&old_page->lru, &pagelist); + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page)); + nr_pages++; + } else if (!is_in_correct_bank(old_page, cpu)) { + TRACE_TASK(current, "isolate_lru_page for a private page failed\n"); + nr_failed++; + } else { + TRACE_TASK(current, "page is already in the correct bank\n"); + } + put_page(old_page); + } + } + TRACE_TASK(current, "PAGES_IN_VMA = %d size = %d KB\n", pages_in_vma, pages_in_vma*4); + vma_itr = vma_itr->vm_next; + } + + ret = 0; + lv = tsk_rt(current)->mc2_data->crit; + if (cpu == -1) + node = 8; + else + node = cpu*2 + lv; +printk(KERN_INFO "READY TO MIGRATE\n"); + /* Migrate private pages */ + if (!list_empty(&pagelist)) { + ret = migrate_pages(&pagelist, alloc_colored_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL); + TRACE_TASK(current, "%ld pages not migrated.\n", ret); + nr_not_migrated = ret; + if (ret) { + putback_movable_pages(&pagelist); + } + } +printk(KERN_INFO "READY TO MIGRATE2\n"); + /* Replicate shared pages */ + if (!list_empty(&task_shared_pagelist)) { + ret = replicate_pages(&task_shared_pagelist, alloc_colored_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL); + TRACE_TASK(current, "%ld shared pages not migrated.\n", ret); + nr_not_migrated += ret; + if (ret) { + putback_movable_pages(&task_shared_pagelist); + } + } + + up_read(&mm->mmap_sem); + + TRACE_TASK(current, "nr_pages = %d nr_failed = %d nr_not_migrated = %d\n", nr_pages, nr_failed, nr_not_migrated); + printk(KERN_INFO "node = %ld, nr_private_pages = %d, nr_shared_pages = %d, nr_failed_to_isolate_lru = %d, nr_not_migrated = %d\n", node, nr_pages, nr_shared_pages, nr_failed, nr_not_migrated); + + return nr_not_migrated; +} + +/* sys_test_call() is a test system call for debugging */ +asmlinkage long sys_test_call(unsigned int param) +{ + long ret = 0; + struct vm_area_struct *vma_itr = NULL; + + TRACE_CUR("test_call param = %d\n", param); + + /* if param == 0, + * show vm regions and the page frame numbers + * associated with the vm region. + * if param == 1, + * print the master list. + */ + if (param == 0) { + down_read(¤t->mm->mmap_sem); + vma_itr = current->mm->mmap; + while (vma_itr != NULL) { + int i, num_pages; + struct page* old_page; + TRACE_TASK(current, "------------------------------------------------------\n"); + TRACE_TASK(current, "vm_start : %lx\n", vma_itr->vm_start); + TRACE_TASK(current, "vm_end : %lx\n", vma_itr->vm_end); + TRACE_TASK(current, "vm_flags : %lx\n", vma_itr->vm_flags); + TRACE_TASK(current, "vm_prot : %x\n", pgprot_val(vma_itr->vm_page_prot)); + TRACE_TASK(current, "VM_SHARED? %ld\n", vma_itr->vm_flags & VM_SHARED); + + num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE; + for (i = 0; i < num_pages; i++) { + old_page = follow_page(vma_itr, vma_itr->vm_start + PAGE_SIZE*i, FOLL_GET|FOLL_SPLIT); + + if (IS_ERR(old_page)) + continue; + if (!old_page) + continue; + + if (PageReserved(old_page)) { + TRACE("Reserved Page!\n"); + put_page(old_page); + continue; + } + + TRACE_TASK(current, "addr: %08x, phy: %08x, color: %d, bank: %d, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s mapping: %p\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_phys(old_page), page_color(old_page), page_bank(old_page), page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-", &(old_page->mapping)); + put_page(old_page); + } + vma_itr = vma_itr->vm_next; + } + TRACE_TASK(current, "------------------------------------------------------\n"); + up_read(¤t->mm->mmap_sem); + } else if (param == 1) { + TRACE_TASK(current, "Shared pages and replicas.\n"); + { + struct shared_lib_page *lpage; + + rcu_read_lock(); + list_for_each_entry(lpage, &shared_lib_pages, list) + { + TRACE_TASK(current, "master_PFN = %05lx r_PFN = %05lx, %05lx, %05lx, %05lx, %05lx\n", lpage->master_pfn, lpage->r_pfn[0], lpage->r_pfn[1], lpage->r_pfn[2], lpage->r_pfn[3], lpage->r_pfn[4]); + } + rcu_read_unlock(); + } + } else if (param == 2) { + flush_cache_all(); + } + + return ret; +} + +asmlinkage long sys_recolor_mem(void* mem, int n_pages, int cpu) +{ + long ret = 0; + struct vm_area_struct *vma_itr = NULL; + int nr_pages = 0, nr_failed = 0, nr_not_migrated = 0; + unsigned long node, user_vaddr; + enum crit_level lv; + struct mm_struct *mm; + + LIST_HEAD(pagelist); + + user_vaddr = (unsigned long)mem; + + migrate_prep(); + + /* Find the current mm_struct */ + rcu_read_lock(); + get_task_struct(current); + rcu_read_unlock(); + mm = get_task_mm(current); + put_task_struct(current); + + down_read(&mm->mmap_sem); + + vma_itr = mm->mmap; + /* Iterate all vm_area_struct */ + while (vma_itr != NULL) { + unsigned int num_pages = 0, i; + struct page *old_page = NULL; + int pages_in_vma = 0; + + num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE; + if ((num_pages != n_pages) || (vma_itr->vm_start != user_vaddr)) { + vma_itr = vma_itr->vm_next; + continue; + } + + /* Traverse all pages in vm_area_struct */ + for (i = 0; i < num_pages; i++) { + old_page = follow_page(vma_itr, vma_itr->vm_start + PAGE_SIZE*i, FOLL_GET|FOLL_SPLIT); + + if (IS_ERR(old_page)) + continue; + if (!old_page) + continue; + + if (PageReserved(old_page)) { + TRACE("Reserved Page!\n"); + put_page(old_page); + continue; + } + TRACE_TASK(current, "addr: %08x, pfn: %05lx, _mapcount: %d, _count: %d flags: %s%s%s\n", vma_itr->vm_start + PAGE_SIZE*i, page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page), vma_itr->vm_flags&VM_READ?"r":"-", vma_itr->vm_flags&VM_WRITE?"w":"-", vma_itr->vm_flags&VM_EXEC?"x":"-"); + pages_in_vma++; + + if (page_count(old_page) == 2) { + ret = isolate_lru_page(old_page); + if (!ret) { + list_add_tail(&old_page->lru, &pagelist); + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page)); + nr_pages++; + } else if (!is_in_correct_bank(old_page, cpu)) { + TRACE_TASK(current, "isolate_lru_page for a private page failed\n"); + nr_failed++; + } else { + TRACE_TASK(current, "page is already in the correct bank\n"); + } + put_page(old_page); + } + } + TRACE_TASK(current, "PAGES_IN_VMA = %d size = %d KB\n", pages_in_vma, pages_in_vma*4); + vma_itr = vma_itr->vm_next; + } + + ret = 0; + lv = tsk_rt(current)->mc2_data->crit; + if (cpu == -1) + node = 8; + else + node = cpu*2 + lv; + + /* Migrate private pages */ + + if (!list_empty(&pagelist)) { + ret = migrate_pages(&pagelist, alloc_colored_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL); + TRACE_TASK(current, "%ld pages not migrated.\n", ret); + nr_not_migrated = ret; + if (ret) { + putback_movable_pages(&pagelist); + } + } + + up_read(&mm->mmap_sem); + + TRACE_TASK(current, "nr_pages = %d nr_failed = %d nr_not_migrated = %d\n", nr_pages, nr_failed, nr_not_migrated); + printk(KERN_INFO "node = %ld, nr_private_pages = %d, nr_failed_to_isolate_lru = %d, nr_not_migrated = %d\n", node, nr_pages, nr_failed, nr_not_migrated); + + return nr_not_migrated; +} + /* p is a real-time task. Re-init its state as a best-effort task. */ static void reinit_litmus_state(struct task_struct* p, int restore) { @@ -645,12 +1036,45 @@ static struct notifier_block shutdown_notifier = { .notifier_call = litmus_shutdown_nb, }; +extern mempool_t *msgpool; +struct page *msgpages; +void *msgvaddr; + +/* Pre-allocate message buffers and lock in the LLC for CL. Default is 4 */ +static int litmus_msgpool_init(void) +{ + int i, order = 4, size; + + msgpages = alloc_pages(GFP_KERNEL, order); + if (!msgpages) { + printk(KERN_WARNING "No memory\n"); + return -ENOMEM; + } + msgvaddr = page_address(msgpages); + + + for (i = 0; i < 8; i++) { + cache_lockdown(0xFFFF8000, i); + } + size = order << (PAGE_SHIFT+2); + color_read_in_mem_lock(0xFFFF7FFF, 0xFFFF8000, msgvaddr, msgvaddr + size); + printk(KERN_INFO "SysV Msg Pool %d B initialized\n", size); + + return 0; +} + static int __init _init_litmus(void) { /* Common initializers, * mode change lock is used to enforce single mode change * operation. */ +#if defined(CONFIG_CPU_V7) + unsigned int line_size_log = 5; // 2^5 = 32 byte + unsigned int cache_info_sets = 2048; // 64KB (way_size) / 32B (line_size) = 2048 + printk("LITMIS^RT-ARM kernel\n"); +#endif + printk("Starting LITMUS^RT kernel\n"); register_sched_plugin(&linux_sched_plugin); @@ -665,11 +1089,17 @@ static int __init _init_litmus(void) else printk("Could not register kill rt tasks magic sysrq.\n"); #endif - init_litmus_proc(); register_reboot_notifier(&shutdown_notifier); +#if defined(CONFIG_CPU_V7) + color_mask = ((cache_info_sets << line_size_log) - 1) ^ (PAGE_SIZE - 1); + printk("Page color mask %lx\n", color_mask); +#endif + + litmus_msgpool_init(); + return 0; } diff --git litmus/mc2_common.c litmus/mc2_common.c new file mode 100644 index 0000000..a8ea5d9 --- /dev/null +++ litmus/mc2_common.c @@ -0,0 +1,78 @@ +/* + * litmus/mc2_common.c + * + * Common functions for MC2 plugin. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +long mc2_task_client_init(struct task_client *tc, struct mc2_task *mc2_param, struct task_struct *tsk, struct reservation *res) +{ + task_client_init(tc, tsk, res); + if ((mc2_param->crit < CRIT_LEVEL_A) || + (mc2_param->crit > CRIT_LEVEL_C)) + return -EINVAL; + + TRACE_TASK(tsk, "mc2_task_client_init: crit_level = %d\n", mc2_param->crit); + + return 0; +} + +asmlinkage long sys_set_mc2_task_param(pid_t pid, struct mc2_task __user * param) +{ + struct task_struct *target; + int retval = -EINVAL; + struct mc2_task *mp = kzalloc(sizeof(*mp), GFP_KERNEL); + + if (!mp) + return -ENOMEM; + + printk("Setting up mc^2 task parameters for process %d.\n", pid); + + if (pid < 0 || param == 0) { + goto out; + } + if (copy_from_user(mp, param, sizeof(*mp))) { + retval = -EFAULT; + goto out; + } + + /* Task search and manipulation must be protected */ + read_lock_irq(&tasklist_lock); + if (!(target = find_task_by_vpid(pid))) { + retval = -ESRCH; + goto out_unlock; + } + + if (is_realtime(target)) { + /* The task is already a real-time task. + * We cannot not allow parameter changes at this point. + */ + retval = -EBUSY; + goto out_unlock; + } + if (mp->crit < CRIT_LEVEL_A || mp->crit >= NUM_CRIT_LEVELS) { + printk(KERN_INFO "litmus: real-time task %d rejected " + "because of invalid criticality level\n", pid); + goto out_unlock; + } + + //target->rt_param.plugin_state = mp; + target->rt_param.mc2_data = mp; + + retval = 0; +out_unlock: + read_unlock_irq(&tasklist_lock); +out: + return retval; +} \ No newline at end of file diff --git litmus/page_dev.c litmus/page_dev.c new file mode 100644 index 0000000..a26a712 --- /dev/null +++ litmus/page_dev.c @@ -0,0 +1,414 @@ +#include +#include + +// This Address Decoding is used in imx6-sabredsd platform +#define NUM_BANKS 8 +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 + +#define NUM_COLORS 16 +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 + +#define NR_LLC_PARTITIONS 9 +#define NR_DRAM_PARTITIONS 5 + +struct mutex dev_mutex; + +/* Initial partitions for LLC and DRAM bank */ +/* 4 color for each core, all colors for Level C */ +unsigned int llc_partition[NR_LLC_PARTITIONS] = { + 0x0000ffff, /* Core 0, and Level A*/ + 0x0000ffff, /* Core 0, and Level B*/ + 0x0000ffff, /* Core 1, and Level A*/ + 0x0000ffff, /* Core 1, and Level B*/ + 0x0000ffff, /* Core 2, and Level A*/ + 0x0000ffff, /* Core 2, and Level B*/ + 0x0000ffff, /* Core 3, and Level A*/ + 0x0000ffff, /* Core 3, and Level B*/ + 0x0000ffff, /* Level C */ +}; + +/* 1 bank for each core, 2 banks for Level C */ +unsigned int dram_partition[NR_DRAM_PARTITIONS] = { + 0x00000010, + 0x00000020, + 0x00000040, + 0x00000080, + 0x0000000f, +}; + +/* Decoding page color, 0~15 */ +static inline unsigned int page_color(struct page *page) +{ + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT); +} + +/* Decoding page bank number, 0~7 */ +static inline unsigned int page_bank(struct page *page) +{ + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT); +} + +int bank_to_partition(unsigned int bank) +{ + int i; + unsigned int bank_bit = 0x1< + +#include +#include +#include + +static void periodic_polling_client_arrives( + struct reservation* res, + struct reservation_client *client +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + lt_t instances, tmp; + + list_add_tail(&client->list, &res->clients); + + switch (res->state) { + case RESERVATION_INACTIVE: + /* Figure out next replenishment time. */ + if (res->env->time_zero == 0) { + tmp = res->env->current_time - res->env->time_zero; + instances = div64_u64(tmp, pres->period); + res->next_replenishment = + (instances + 1) * pres->period + pres->offset; + } + else { + tmp = res->env->current_time - res->env->time_zero; + instances = div64_u64(tmp, pres->period); + res->next_replenishment = res->env->time_zero + instances * pres->period; + } + + TRACE("ENV_TIME_ZERO %llu\n", res->env->time_zero); + TRACE("pol-res: R%d activate tmp=%llu instances=%llu period=%llu nextrp=%llu cur=%llu\n", + res->id, tmp, instances, pres->period, res->next_replenishment, + res->env->current_time); + + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + break; + + case RESERVATION_ACTIVE: + case RESERVATION_DEPLETED: + /* do nothing */ + break; + + case RESERVATION_ACTIVE_IDLE: + res->blocked_by_ghost = 0; + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + } +} + + +static void periodic_polling_client_departs( + struct reservation *res, + struct reservation_client *client, + int did_signal_job_completion +) +{ + list_del(&client->list); + + switch (res->state) { + case RESERVATION_INACTIVE: + case RESERVATION_ACTIVE_IDLE: + BUG(); /* INACTIVE or IDLE <=> no client */ + break; + + case RESERVATION_ACTIVE: + if (list_empty(&res->clients)) { + res->env->change_state(res->env, res, +// RESERVATION_ACTIVE_IDLE); + res->cur_budget ? + RESERVATION_ACTIVE_IDLE : + RESERVATION_DEPLETED); +// did_signal_job_completion ? +// RESERVATION_DEPLETED : +// RESERVATION_ACTIVE_IDLE); + } /* else: nothing to do, more clients ready */ + break; + + case RESERVATION_DEPLETED: + /* do nothing */ + break; + } +} + +static void periodic_polling_on_replenishment( + struct reservation *res +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + + /* replenish budget */ + res->cur_budget = pres->max_budget; + res->next_replenishment += pres->period; + res->budget_consumed = 0; + + TRACE("polling_replenish(%u): next_replenishment=%llu\n", res->id, res->next_replenishment); + switch (res->state) { + case RESERVATION_DEPLETED: + case RESERVATION_INACTIVE: + case RESERVATION_ACTIVE_IDLE: + if (list_empty(&res->clients)) + /* no clients => poll again later */ + res->env->change_state(res->env, res, + RESERVATION_INACTIVE); + else + /* we have clients & budget => ACTIVE */ + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + + case RESERVATION_ACTIVE: + /* Replenished while active => tardy? In any case, + * go ahead and stay active. */ + break; + } +} + +static void periodic_polling_on_replenishment_edf( + struct reservation *res +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + + /* update current priority */ + res->priority = res->next_replenishment + pres->deadline; + + /* do common updates */ + periodic_polling_on_replenishment(res); +} + +static void common_drain_budget( + struct reservation *res, + lt_t how_much) +{ + if (how_much >= res->cur_budget) + res->cur_budget = 0; + else + res->cur_budget -= how_much; + + res->budget_consumed += how_much; + res->budget_consumed_total += how_much; + + switch (res->state) { + case RESERVATION_DEPLETED: + case RESERVATION_INACTIVE: + //BUG(); + TRACE("!!!!!!!!!!!!!!!STATE ERROR R%d STATE(%d)\n", res->id, res->state); + break; + + case RESERVATION_ACTIVE_IDLE: + case RESERVATION_ACTIVE: + if (!res->cur_budget) { + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + } /* else: stay in current state */ + break; + } +} + +static struct reservation_ops periodic_polling_ops_fp = { + .dispatch_client = default_dispatch_client, + .client_arrives = periodic_polling_client_arrives, + .client_departs = periodic_polling_client_departs, + .replenish = periodic_polling_on_replenishment, + .drain_budget = common_drain_budget, +}; + +static struct reservation_ops periodic_polling_ops_edf = { + .dispatch_client = default_dispatch_client, + .client_arrives = periodic_polling_client_arrives, + .client_departs = periodic_polling_client_departs, + .replenish = periodic_polling_on_replenishment_edf, + .drain_budget = common_drain_budget, +}; + + + + +static void sporadic_polling_client_arrives_fp( + struct reservation* res, + struct reservation_client *client +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + + list_add_tail(&client->list, &res->clients); + + switch (res->state) { + case RESERVATION_INACTIVE: + /* Replenish now. */ + res->cur_budget = pres->max_budget; + res->next_replenishment = + res->env->current_time + pres->period; + + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + + case RESERVATION_ACTIVE: + case RESERVATION_DEPLETED: + /* do nothing */ + break; + + case RESERVATION_ACTIVE_IDLE: + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + } +} + +static void sporadic_polling_client_arrives_edf( + struct reservation* res, + struct reservation_client *client +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + + list_add_tail(&client->list, &res->clients); + + switch (res->state) { + case RESERVATION_INACTIVE: + /* Replenish now. */ + res->cur_budget = pres->max_budget; + res->next_replenishment = + res->env->current_time + pres->period; + res->priority = + res->env->current_time + pres->deadline; + + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + + case RESERVATION_ACTIVE: + case RESERVATION_DEPLETED: + /* do nothing */ + break; + + case RESERVATION_ACTIVE_IDLE: + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + } +} + +static struct reservation_ops sporadic_polling_ops_fp = { + .dispatch_client = default_dispatch_client, + .client_arrives = sporadic_polling_client_arrives_fp, + .client_departs = periodic_polling_client_departs, + .replenish = periodic_polling_on_replenishment, + .drain_budget = common_drain_budget, +}; + +static struct reservation_ops sporadic_polling_ops_edf = { + .dispatch_client = default_dispatch_client, + .client_arrives = sporadic_polling_client_arrives_edf, + .client_departs = periodic_polling_client_departs, + .replenish = periodic_polling_on_replenishment_edf, + .drain_budget = common_drain_budget, +}; + +void polling_reservation_init( + struct polling_reservation *pres, + int use_edf_prio, + int use_periodic_polling, + lt_t budget, lt_t period, lt_t deadline, lt_t offset +) +{ + if (!deadline) + deadline = period; + BUG_ON(budget > period); + BUG_ON(budget > deadline); + BUG_ON(offset >= period); + + reservation_init(&pres->res); + pres->max_budget = budget; + pres->period = period; + pres->deadline = deadline; + pres->offset = offset; + TRACE_TASK(current, "polling_reservation_init: periodic %d, use_edf %d\n", use_periodic_polling, use_edf_prio); + if (use_periodic_polling) { + if (use_edf_prio) + pres->res.ops = &periodic_polling_ops_edf; + else + pres->res.ops = &periodic_polling_ops_fp; + } else { + if (use_edf_prio) + pres->res.ops = &sporadic_polling_ops_edf; + else + pres->res.ops = &sporadic_polling_ops_fp; + } +} + + +static lt_t td_cur_major_cycle_start(struct table_driven_reservation *tdres) +{ + lt_t x, tmp; + + tmp = tdres->res.env->current_time - tdres->res.env->time_zero; + x = div64_u64(tmp, tdres->major_cycle); + x *= tdres->major_cycle; + return x; +} + + +static lt_t td_next_major_cycle_start(struct table_driven_reservation *tdres) +{ + lt_t x, tmp; + + tmp = tdres->res.env->current_time - tdres->res.env->time_zero; + x = div64_u64(tmp, tdres->major_cycle) + 1; + x *= tdres->major_cycle; + return x; +} + +static void td_client_arrives( + struct reservation* res, + struct reservation_client *client +) +{ + struct table_driven_reservation *tdres = + container_of(res, struct table_driven_reservation, res); + + list_add_tail(&client->list, &res->clients); + + switch (res->state) { + case RESERVATION_INACTIVE: + /* Figure out first replenishment time. */ + tdres->major_cycle_start = td_next_major_cycle_start(tdres); + res->next_replenishment = tdres->major_cycle_start; + res->next_replenishment += tdres->intervals[0].start; + tdres->next_interval = 0; + + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + break; + + case RESERVATION_ACTIVE: + case RESERVATION_DEPLETED: + /* do nothing */ + break; + + case RESERVATION_ACTIVE_IDLE: + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + } +} + +static void td_client_departs( + struct reservation *res, + struct reservation_client *client, + int did_signal_job_completion +) +{ + list_del(&client->list); + + switch (res->state) { + case RESERVATION_INACTIVE: + case RESERVATION_ACTIVE_IDLE: + //BUG(); /* INACTIVE or IDLE <=> no client */ + break; + + case RESERVATION_ACTIVE: + if (list_empty(&res->clients)) { + res->env->change_state(res->env, res, + RESERVATION_ACTIVE_IDLE); + } /* else: nothing to do, more clients ready */ + break; + + case RESERVATION_DEPLETED: + /* do nothing */ + break; + } +} + +static lt_t td_time_remaining_until_end(struct table_driven_reservation *tdres) +{ + lt_t now = tdres->res.env->current_time; + lt_t end = tdres->cur_interval.end; + //TRACE("td_remaining(%u): start=%llu now=%llu end=%llu state=%d\n", tdres->res.id, tdres->cur_interval.start, now, end, tdres->res.state); + if (now >= end) + return 0; + else + return end - now; +} + +static void td_replenish( + struct reservation *res) +{ + struct table_driven_reservation *tdres = + container_of(res, struct table_driven_reservation, res); + + //TRACE("td_replenish(%u): expected_replenishment=%llu\n", res->id, res->next_replenishment); + + /* figure out current interval */ + tdres->cur_interval.start = tdres->major_cycle_start + + tdres->intervals[tdres->next_interval].start; + tdres->cur_interval.end = tdres->major_cycle_start + + tdres->intervals[tdres->next_interval].end; +/* TRACE("major_cycle_start=%llu => [%llu, %llu]\n", + tdres->major_cycle_start, + tdres->cur_interval.start, + tdres->cur_interval.end); +*/ + /* reset budget */ + res->cur_budget = td_time_remaining_until_end(tdres); + res->budget_consumed = 0; + //TRACE("td_replenish(%u): %s budget=%llu\n", res->id, res->cur_budget ? "" : "WARNING", res->cur_budget); + + /* prepare next slot */ + tdres->next_interval = (tdres->next_interval + 1) % tdres->num_intervals; + if (!tdres->next_interval) + /* wrap to next major cycle */ + tdres->major_cycle_start += tdres->major_cycle; + + /* determine next time this reservation becomes eligible to execute */ + res->next_replenishment = tdres->major_cycle_start; + res->next_replenishment += tdres->intervals[tdres->next_interval].start; + //TRACE("td_replenish(%u): next_replenishment=%llu\n", res->id, res->next_replenishment); + + + switch (res->state) { + case RESERVATION_DEPLETED: + case RESERVATION_ACTIVE: + case RESERVATION_ACTIVE_IDLE: + if (list_empty(&res->clients)) + res->env->change_state(res->env, res, + RESERVATION_ACTIVE_IDLE); + else + /* we have clients & budget => ACTIVE */ + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + + case RESERVATION_INACTIVE: + BUG(); + break; + } +} + +static void td_drain_budget( + struct reservation *res, + lt_t how_much) +{ + struct table_driven_reservation *tdres = + container_of(res, struct table_driven_reservation, res); + + res->budget_consumed += how_much; + res->budget_consumed_total += how_much; + + /* Table-driven scheduling: instead of tracking the budget, we compute + * how much time is left in this allocation interval. */ + + /* sanity check: we should never try to drain from future slots */ + //TRACE("TD_DRAIN STATE(%d) [%llu,%llu] %llu ?\n", res->state, tdres->cur_interval.start, tdres->cur_interval.end, res->env->current_time); + //BUG_ON(tdres->cur_interval.start > res->env->current_time); + if (tdres->cur_interval.start > res->env->current_time) + TRACE("TD_DRAIN BUG!!!!!!!!!!\n"); + + switch (res->state) { + case RESERVATION_DEPLETED: + case RESERVATION_INACTIVE: + //BUG(); + TRACE("TD_DRAIN!!!!!!!!! RES_STATE = %d\n", res->state); + break; + + case RESERVATION_ACTIVE_IDLE: + case RESERVATION_ACTIVE: + res->cur_budget = td_time_remaining_until_end(tdres); + //TRACE("td_drain_budget(%u): drained to budget=%llu\n", res->id, res->cur_budget); + if (!res->cur_budget) { + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + } else { + /* sanity check budget calculation */ + //BUG_ON(res->env->current_time >= tdres->cur_interval.end); + //BUG_ON(res->env->current_time < tdres->cur_interval.start); + if (res->env->current_time >= tdres->cur_interval.end) + printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING1\n"); + if (res->env->current_time < tdres->cur_interval.start) + printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING2\n"); + } + + break; + } +} + +static struct task_struct* td_dispatch_client( + struct reservation *res, + lt_t *for_at_most) +{ + struct task_struct *t; + struct table_driven_reservation *tdres = + container_of(res, struct table_driven_reservation, res); + + /* usual logic for selecting a client */ + t = default_dispatch_client(res, for_at_most); + + TRACE_TASK(t, "td_dispatch_client(%u): selected, budget=%llu\n", + res->id, res->cur_budget); + + /* check how much budget we have left in this time slot */ + res->cur_budget = td_time_remaining_until_end(tdres); + + TRACE_TASK(t, "td_dispatch_client(%u): updated to budget=%llu next=%d\n", + res->id, res->cur_budget, tdres->next_interval); + + if (unlikely(!res->cur_budget)) { + /* Unlikely case: if we ran out of budget, the user configured + * a broken scheduling table (overlapping table slots). + * Not much we can do about this, but we can't dispatch a job + * now without causing overload. So let's register this reservation + * as depleted and wait for the next allocation. */ + TRACE("td_dispatch_client(%u): budget unexpectedly depleted " + "(check scheduling table for unintended overlap)\n", + res->id); + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + return NULL; + } else + return t; +} + +static struct reservation_ops td_ops = { + .dispatch_client = td_dispatch_client, + .client_arrives = td_client_arrives, + .client_departs = td_client_departs, + .replenish = td_replenish, + .drain_budget = td_drain_budget, +}; + +void table_driven_reservation_init( + struct table_driven_reservation *tdres, + lt_t major_cycle, + struct lt_interval *intervals, + unsigned int num_intervals) +{ + unsigned int i; + + /* sanity checking */ + BUG_ON(!num_intervals); + for (i = 0; i < num_intervals; i++) + BUG_ON(intervals[i].end <= intervals[i].start); + for (i = 0; i + 1 < num_intervals; i++) + BUG_ON(intervals[i + 1].start <= intervals[i].end); + BUG_ON(intervals[num_intervals - 1].end > major_cycle); + + reservation_init(&tdres->res); + tdres->major_cycle = major_cycle; + tdres->intervals = intervals; + tdres->cur_interval.start = 0; + tdres->cur_interval.end = 0; + tdres->num_intervals = num_intervals; + tdres->res.ops = &td_ops; +} diff --git litmus/replicate_lib.c litmus/replicate_lib.c new file mode 100644 index 0000000..cfc5258 --- /dev/null +++ litmus/replicate_lib.c @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +DEFINE_PER_CPU(struct list_head, shared_lib_page_list); + +#define shared_lib_pages_for(cpu_id) (&per_cpu(shared_lib_page_list, cpu_id)) +#define local_shared_lib_pages() (this_cpu_ptr(&shared_lib_page_list)) + +#define INVALID_PFN (0xffffffff) + +static int __init litmus_replicate_lib_init(void) +{ + int cpu, ret = 0; + + printk(KERN_INFO "Registering LITMUS^RT Per-core Shared Library module.\n"); + + for_each_online_cpu(cpu) { + INIT_LIST_HEAD(shared_lib_pages_for(cpu)); + printk(KERN_INFO "CPU%d PSL-list initialized.\n", cpu); + } + + return ret; +} + +static void litmus_replicate_lib_exit(void) +{ + return; +} + +module_init(litmus_replicate_lib_init); +module_exit(litmus_replicate_lib_exit); \ No newline at end of file diff --git litmus/reservation.c litmus/reservation.c new file mode 100644 index 0000000..5eee01a --- /dev/null +++ litmus/reservation.c @@ -0,0 +1,706 @@ +#include +#include + +#include +#include + +#define BUDGET_ENFORCEMENT_AT_C 0 + +void reservation_init(struct reservation *res) +{ + memset(res, sizeof(*res), 0); + res->state = RESERVATION_INACTIVE; + INIT_LIST_HEAD(&res->clients); +} + +struct task_struct* default_dispatch_client( + struct reservation *res, + lt_t *for_at_most) +{ + struct reservation_client *client, *next; + struct task_struct* tsk; + + BUG_ON(res->state != RESERVATION_ACTIVE); + *for_at_most = 0; + + list_for_each_entry_safe(client, next, &res->clients, list) { + tsk = client->dispatch(client); + if (likely(tsk)) { + return tsk; + } + } + return NULL; +} + +static struct task_struct * task_client_dispatch(struct reservation_client *client) +{ + struct task_client *tc = container_of(client, struct task_client, client); + return tc->task; +} + +void task_client_init(struct task_client *tc, struct task_struct *tsk, + struct reservation *res) +{ + memset(&tc->client, sizeof(tc->client), 0); + tc->client.dispatch = task_client_dispatch; + tc->client.reservation = res; + tc->task = tsk; +} + +static void sup_scheduler_update_at( + struct sup_reservation_environment* sup_env, + lt_t when) +{ + //TRACE("SCHEDULER_UPDATE_AT update: %llu > when %llu\n", sup_env->next_scheduler_update, when); + if (sup_env->next_scheduler_update > when) + sup_env->next_scheduler_update = when; +} + +void sup_scheduler_update_after( + struct sup_reservation_environment* sup_env, + lt_t timeout) +{ + sup_scheduler_update_at(sup_env, sup_env->env.current_time + timeout); +} + +static int _sup_queue_depleted( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + struct list_head *pos; + struct reservation *queued; + int passed_earlier = 0; + + list_for_each(pos, &sup_env->depleted_reservations) { + queued = list_entry(pos, struct reservation, list); + if (queued->next_replenishment > res->next_replenishment) { + list_add(&res->list, pos->prev); + return passed_earlier; + } else + passed_earlier = 1; + } + + list_add_tail(&res->list, &sup_env->depleted_reservations); + + return passed_earlier; +} + +static void sup_queue_depleted( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + int passed_earlier = _sup_queue_depleted(sup_env, res); + + /* check for updated replenishment time */ + if (!passed_earlier) + sup_scheduler_update_at(sup_env, res->next_replenishment); +} + +static int _sup_queue_active( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + struct list_head *pos; + struct reservation *queued; + int passed_active = 0; + + list_for_each(pos, &sup_env->active_reservations) { + queued = list_entry(pos, struct reservation, list); + if (queued->priority > res->priority) { + list_add(&res->list, pos->prev); + return passed_active; + } else if (queued->state == RESERVATION_ACTIVE) + passed_active = 1; + } + + list_add_tail(&res->list, &sup_env->active_reservations); + return passed_active; +} + +static void sup_queue_active( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + int passed_active = _sup_queue_active(sup_env, res); + + /* check for possible preemption */ + if (res->state == RESERVATION_ACTIVE && !passed_active) + sup_env->next_scheduler_update = SUP_RESCHEDULE_NOW; + else { + /* Active means this reservation is draining budget => make sure + * the scheduler is called to notice when the reservation budget has been + * drained completely. */ + sup_scheduler_update_after(sup_env, res->cur_budget); + } +} + +static void sup_queue_reservation( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + switch (res->state) { + case RESERVATION_INACTIVE: + list_add(&res->list, &sup_env->inactive_reservations); + break; + + case RESERVATION_DEPLETED: + sup_queue_depleted(sup_env, res); + break; + + case RESERVATION_ACTIVE_IDLE: + case RESERVATION_ACTIVE: + sup_queue_active(sup_env, res); + break; + } +} + +void sup_add_new_reservation( + struct sup_reservation_environment* sup_env, + struct reservation* new_res) +{ + new_res->env = &sup_env->env; + sup_queue_reservation(sup_env, new_res); +} + +struct reservation* sup_find_by_id(struct sup_reservation_environment* sup_env, + unsigned int id) +{ + struct reservation *res; + + list_for_each_entry(res, &sup_env->active_reservations, list) { + if (res->id == id) + return res; + } + list_for_each_entry(res, &sup_env->inactive_reservations, list) { + if (res->id == id) + return res; + } + list_for_each_entry(res, &sup_env->depleted_reservations, list) { + if (res->id == id) + return res; + } + + return NULL; +} + +static void sup_charge_budget( + struct sup_reservation_environment* sup_env, + lt_t delta) +{ + struct list_head *pos, *next; + struct reservation *res; + + int encountered_active = 0; + + list_for_each_safe(pos, next, &sup_env->active_reservations) { + /* charge all ACTIVE_IDLE up to the first ACTIVE reservation */ + res = list_entry(pos, struct reservation, list); + if (res->state == RESERVATION_ACTIVE) { + TRACE("sup_charge_budget ACTIVE R%u drain %llu\n", res->id, delta); + if (encountered_active == 0 && res->blocked_by_ghost == 0) { + TRACE("DRAIN !!\n"); + res->ops->drain_budget(res, delta); + encountered_active = 1; + } + } else { + //BUG_ON(res->state != RESERVATION_ACTIVE_IDLE); + TRACE("sup_charge_budget INACTIVE R%u drain %llu\n", res->id, delta); + res->ops->drain_budget(res, delta); + } + if (res->state == RESERVATION_ACTIVE || + res->state == RESERVATION_ACTIVE_IDLE) + { + /* make sure scheduler is invoked when this reservation expires + * its remaining budget */ + TRACE("requesting scheduler update for reservation %u in %llu nanoseconds\n", + res->id, res->cur_budget); + sup_scheduler_update_after(sup_env, res->cur_budget); + } + //if (encountered_active == 2) + /* stop at the first ACTIVE reservation */ + // break; + } + //TRACE("finished charging budgets\n"); +} + +static void sup_replenish_budgets(struct sup_reservation_environment* sup_env) +{ + struct list_head *pos, *next; + struct reservation *res; + + list_for_each_safe(pos, next, &sup_env->depleted_reservations) { + res = list_entry(pos, struct reservation, list); + if (res->next_replenishment <= sup_env->env.current_time) { + res->ops->replenish(res); + } else { + /* list is ordered by increasing depletion times */ + break; + } + } + //TRACE("finished replenishing budgets\n"); + + /* request a scheduler update at the next replenishment instant */ + res = list_first_entry_or_null(&sup_env->depleted_reservations, + struct reservation, list); + if (res) + sup_scheduler_update_at(sup_env, res->next_replenishment); +} + +void sup_update_time( + struct sup_reservation_environment* sup_env, + lt_t now) +{ + lt_t delta; + + /* If the time didn't advance, there is nothing to do. + * This check makes it safe to call sup_advance_time() potentially + * multiple times (e.g., via different code paths. */ + //TRACE("(sup_update_time) now: %llu, current_time: %llu\n", now, sup_env->env.current_time); + if (unlikely(now <= sup_env->env.current_time)) + return; + + delta = now - sup_env->env.current_time; + sup_env->env.current_time = now; + + /* check if future updates are required */ + if (sup_env->next_scheduler_update <= sup_env->env.current_time) + sup_env->next_scheduler_update = SUP_NO_SCHEDULER_UPDATE; + + /* deplete budgets by passage of time */ + //TRACE("CHARGE###\n"); + sup_charge_budget(sup_env, delta); + + /* check if any budgets where replenished */ + //TRACE("REPLENISH###\n"); + sup_replenish_budgets(sup_env); +} + +struct task_struct* sup_dispatch(struct sup_reservation_environment* sup_env) +{ + struct reservation *res, *next; + struct task_struct *tsk = NULL; + lt_t time_slice; + + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) { + if (res->state == RESERVATION_ACTIVE) { + tsk = res->ops->dispatch_client(res, &time_slice); + if (likely(tsk)) { + if (time_slice) + sup_scheduler_update_after(sup_env, time_slice); + sup_scheduler_update_after(sup_env, res->cur_budget); + return tsk; + } + } + } + + return NULL; +} + +static void sup_res_change_state( + struct reservation_environment* env, + struct reservation *res, + reservation_state_t new_state) +{ + struct sup_reservation_environment* sup_env; + + sup_env = container_of(env, struct sup_reservation_environment, env); + + TRACE("reservation R%d state %d->%d at %llu\n", + res->id, res->state, new_state, env->current_time); + + list_del(&res->list); + /* check if we need to reschedule because we lost an active reservation */ + if (res->state == RESERVATION_ACTIVE && !sup_env->will_schedule) + sup_env->next_scheduler_update = SUP_RESCHEDULE_NOW; + res->state = new_state; + sup_queue_reservation(sup_env, res); +} + +void sup_init(struct sup_reservation_environment* sup_env) +{ + memset(sup_env, sizeof(*sup_env), 0); + + INIT_LIST_HEAD(&sup_env->active_reservations); + INIT_LIST_HEAD(&sup_env->depleted_reservations); + INIT_LIST_HEAD(&sup_env->inactive_reservations); + + sup_env->env.change_state = sup_res_change_state; + + sup_env->next_scheduler_update = SUP_NO_SCHEDULER_UPDATE; +} + +struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env, + unsigned int id) +{ + struct reservation *res; + + list_for_each_entry(res, &gmp_env->active_reservations, list) { + if (res->id == id) + return res; + } + list_for_each_entry(res, &gmp_env->inactive_reservations, list) { + if (res->id == id) + return res; + } + list_for_each_entry(res, &gmp_env->depleted_reservations, list) { + if (res->id == id) + return res; + } + + return NULL; +} + + +struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env, + unsigned int id) +{ + struct next_timer_event *event; + + list_for_each_entry(event, &gmp_env->next_events, list) { + if (event->id == id) + return event; + } + + return NULL; +} + + +struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env, + lt_t when) +{ + struct next_timer_event *event; + + list_for_each_entry(event, &gmp_env->next_events, list) { + if (event->next_update == when) + return event; + } + + return NULL; +} + +#define TIMER_RESOLUTION 100000L + +static void gmp_add_event( + struct gmp_reservation_environment* gmp_env, + lt_t when, unsigned int id, event_type_t type) +{ + struct next_timer_event *nevent, *queued; + struct list_head *pos; + int found = 0, update = 0; + + //when = div64_u64(when, TIMER_RESOLUTION); + //when *= TIMER_RESOLUTION; +//printk(KERN_ALERT "GMP_ADD id=%d type=%d when=%llu\n", id, type, when); + nevent = gmp_find_event_by_id(gmp_env, id); + + if (nevent) + TRACE("EVENT R%d update prev = %llu, new = %llu\n", nevent->id, nevent->next_update, when); + + if (nevent && nevent->next_update > when) { + list_del(&nevent->list); + update = 1; + + } + + if (!nevent || nevent->type != type || update == 1) { + if (update == 0) + nevent = kzalloc(sizeof(*nevent), GFP_ATOMIC); + BUG_ON(!nevent); + nevent->next_update = when; + nevent->id = id; + nevent->type = type; + nevent->timer_armed_on = NO_CPU; + + list_for_each(pos, &gmp_env->next_events) { + queued = list_entry(pos, struct next_timer_event, list); + if (queued->next_update > nevent->next_update) { + list_add(&nevent->list, pos->prev); + found = 1; + TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at before %llu\n", nevent->id, nevent->type, nevent->next_update, queued->next_update); + break; + } + } + + if (!found) { + list_add_tail(&nevent->list, &gmp_env->next_events); + TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at TAIL\n", nevent->id, nevent->type, nevent->next_update); + } + } else { + //TRACE("EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when); +; //printk(KERN_ALERT "EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when); + } + + TRACE("======START PRINTING EVENT LIST======\n"); + gmp_print_events(gmp_env, litmus_clock()); + TRACE("======FINISH PRINTING EVENT LIST======\n"); +} + +void gmp_add_event_after( + struct gmp_reservation_environment* gmp_env, lt_t timeout, unsigned int id, event_type_t type) +{ + //printk(KERN_ALERT "ADD_EVENT_AFTER id = %d\n", id); + gmp_add_event(gmp_env, gmp_env->env.current_time + timeout, id, type); +} + +static void gmp_queue_depleted( + struct gmp_reservation_environment* gmp_env, + struct reservation *res) +{ + struct list_head *pos; + struct reservation *queued; + int found = 0; + +//printk(KERN_ALERT "R%d request to enqueue depleted_list\n", res->id); + + list_for_each(pos, &gmp_env->depleted_reservations) { + queued = list_entry(pos, struct reservation, list); + if (queued && (queued->next_replenishment > res->next_replenishment)) { +//printk(KERN_ALERT "QUEUED R%d %llu\n", queued->id, queued->next_replenishment); + list_add(&res->list, pos->prev); + found = 1; + break; + } + } + + if (!found) + list_add_tail(&res->list, &gmp_env->depleted_reservations); + + TRACE("R%d queued to depleted_list\n", res->id); +//printk(KERN_ALERT "R%d queued to depleted_list\n", res->id); + gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH); +} + +static void gmp_queue_active( + struct gmp_reservation_environment* gmp_env, + struct reservation *res) +{ + struct list_head *pos; + struct reservation *queued; + int check_preempt = 1, found = 0; + + list_for_each(pos, &gmp_env->active_reservations) { + queued = list_entry(pos, struct reservation, list); + if (queued->priority > res->priority) { + list_add(&res->list, pos->prev); + found = 1; + break; + } else if (queued->scheduled_on == NO_CPU) + check_preempt = 0; + } + + if (!found) + list_add_tail(&res->list, &gmp_env->active_reservations); + + /* check for possible preemption */ + if (res->state == RESERVATION_ACTIVE && check_preempt) + gmp_env->schedule_now++; + +#if BUDGET_ENFORCEMENT_AT_C + gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN); +#endif + res->event_added = 1; +} + +static void gmp_queue_reservation( + struct gmp_reservation_environment* gmp_env, + struct reservation *res) +{ + +//printk(KERN_ALERT "DEBUG: Passed %s %d %p R%d STATE %d\n",__FUNCTION__,__LINE__, gmp_env, res->id, res->state); + switch (res->state) { + case RESERVATION_INACTIVE: + list_add(&res->list, &gmp_env->inactive_reservations); + break; + + case RESERVATION_DEPLETED: + gmp_queue_depleted(gmp_env, res); + break; + + case RESERVATION_ACTIVE_IDLE: + case RESERVATION_ACTIVE: + gmp_queue_active(gmp_env, res); + break; + } +} + +void gmp_add_new_reservation( + struct gmp_reservation_environment* gmp_env, + struct reservation* new_res) +{ + new_res->env = &gmp_env->env; + gmp_queue_reservation(gmp_env, new_res); +} + +#if BUDGET_ENFORCEMENT_AT_C +static void gmp_charge_budget( + struct gmp_reservation_environment* gmp_env, + lt_t delta) +{ + struct list_head *pos, *next; + struct reservation *res; + + list_for_each_safe(pos, next, &gmp_env->active_reservations) { + int drained = 0; + /* charge all ACTIVE_IDLE up to the first ACTIVE reservation */ + res = list_entry(pos, struct reservation, list); + if (res->state == RESERVATION_ACTIVE) { + TRACE("gmp_charge_budget ACTIVE R%u scheduled_on=%d drain %llu\n", res->id, res->scheduled_on, delta); + if (res->scheduled_on != NO_CPU && res->blocked_by_ghost == 0) { + TRACE("DRAIN !!\n"); + drained = 1; + res->ops->drain_budget(res, delta); + } else { + TRACE("NO DRAIN (not scheduled)!!\n"); + } + } else { + //BUG_ON(res->state != RESERVATION_ACTIVE_IDLE); + if (res->state != RESERVATION_ACTIVE_IDLE) + TRACE("BUG!!!!!!!!!!!! gmp_charge_budget()\n"); + TRACE("gmp_charge_budget INACTIVE R%u drain %llu\n", res->id, delta); + //if (res->is_ghost != NO_CPU) { + TRACE("DRAIN !!\n"); + drained = 1; + res->ops->drain_budget(res, delta); + //} + } + if ((res->state == RESERVATION_ACTIVE || + res->state == RESERVATION_ACTIVE_IDLE) && (drained == 1)) + { + /* make sure scheduler is invoked when this reservation expires + * its remaining budget */ + TRACE("requesting gmp_scheduler update for reservation %u in %llu nanoseconds\n", res->id, res->cur_budget); + gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN); + res->event_added = 1; + } + //if (encountered_active == 2) + /* stop at the first ACTIVE reservation */ + // break; + } + //TRACE("finished charging budgets\n"); +} +#else + +static void gmp_charge_budget( + struct gmp_reservation_environment* gmp_env, + lt_t delta) +{ + return; +} + +#endif + +static void gmp_replenish_budgets(struct gmp_reservation_environment* gmp_env) +{ + struct list_head *pos, *next; + struct reservation *res; + + list_for_each_safe(pos, next, &gmp_env->depleted_reservations) { + res = list_entry(pos, struct reservation, list); + if (res->next_replenishment <= gmp_env->env.current_time) { + res->ops->replenish(res); + if (res->is_ghost != NO_CPU) { + TRACE("R%d replenished! scheduled_on=%d\n", res->id, res->scheduled_on); + } + } else { + /* list is ordered by increasing depletion times */ + break; + } + } + //TRACE("finished replenishing budgets\n"); +} + +#define EPSILON 50 + +/* return schedule_now */ +int gmp_update_time( + struct gmp_reservation_environment* gmp_env, + lt_t now) +{ + struct next_timer_event *event, *next; + lt_t delta, ret; + + /* If the time didn't advance, there is nothing to do. + * This check makes it safe to call sup_advance_time() potentially + * multiple times (e.g., via different code paths. */ + //TRACE("(gmp_update_time) now: %llu, current_time: %llu\n", now, gmp_env->env.current_time); + if (unlikely(now <= gmp_env->env.current_time + EPSILON)) + return 0; + + delta = now - gmp_env->env.current_time; + gmp_env->env.current_time = now; + + + //gmp_print_events(gmp_env, now); + /* deplete budgets by passage of time */ + //TRACE("CHARGE###\n"); + gmp_charge_budget(gmp_env, delta); + + /* check if any budgets where replenished */ + //TRACE("REPLENISH###\n"); + gmp_replenish_budgets(gmp_env); + + + list_for_each_entry_safe(event, next, &gmp_env->next_events, list) { + if (event->next_update < now) { + list_del(&event->list); + //TRACE("EVENT at %llu IS DELETED\n", event->next_update); + kfree(event); + } else { + break; + } + } + + //gmp_print_events(gmp_env, litmus_clock()); + + ret = min(gmp_env->schedule_now, NR_CPUS); + gmp_env->schedule_now = 0; + + return ret; +} + +void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now) +{ + struct next_timer_event *event, *next; + + TRACE("GLOBAL EVENTS now=%llu\n", now); + list_for_each_entry_safe(event, next, &gmp_env->next_events, list) { + TRACE("at %llu type=%d id=%d armed_on=%d\n", event->next_update, event->type, event->id, event->timer_armed_on); + } +} + +static void gmp_res_change_state( + struct reservation_environment* env, + struct reservation *res, + reservation_state_t new_state) +{ + struct gmp_reservation_environment* gmp_env; + + gmp_env = container_of(env, struct gmp_reservation_environment, env); + + TRACE("GMP reservation R%d state %d->%d at %llu\n", + res->id, res->state, new_state, env->current_time); + + list_del(&res->list); + /* check if we need to reschedule because we lost an active reservation */ + if (res->state == RESERVATION_ACTIVE) + gmp_env->schedule_now++; + res->state = new_state; + gmp_queue_reservation(gmp_env, res); +} + +void gmp_init(struct gmp_reservation_environment* gmp_env) +{ + memset(gmp_env, sizeof(*gmp_env), 0); + + INIT_LIST_HEAD(&gmp_env->active_reservations); + INIT_LIST_HEAD(&gmp_env->depleted_reservations); + INIT_LIST_HEAD(&gmp_env->inactive_reservations); + INIT_LIST_HEAD(&gmp_env->next_events); + + gmp_env->env.change_state = gmp_res_change_state; + + gmp_env->schedule_now = 0; + gmp_env->will_schedule = false; + + raw_spin_lock_init(&gmp_env->lock); +} diff --git litmus/sched_mc2.c litmus/sched_mc2.c new file mode 100644 index 0000000..dba69df --- /dev/null +++ litmus/sched_mc2.c @@ -0,0 +1,1592 @@ +/* + * litmus/sched_mc2.c + * + * Implementation of the Mixed-Criticality on MultiCore scheduler + * + * This plugin implements a scheduling algorithm proposed in + * "Mixed-Criticality Real-Time Scheduling for Multicore System" paper. + */ + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define BUDGET_ENFORCEMENT_AT_C 0 + +extern void do_partition(enum crit_level lv, int cpu); + +/* _global_env - reservation container for level-C tasks*/ +struct gmp_reservation_environment _global_env; + +/* cpu_entry - keep track of a running task on a cpu + * This state is used to decide the lowest priority cpu + */ +struct cpu_entry { + struct task_struct *scheduled; + lt_t deadline; + int cpu; + enum crit_level lv; + /* if will_schedule is true, this cpu is already selected and + call mc2_schedule() soon. */ + bool will_schedule; +}; + +/* cpu_priority - a global state for choosing the lowest priority CPU */ +struct cpu_priority { + raw_spinlock_t lock; + struct cpu_entry cpu_entries[NR_CPUS]; +}; + +struct cpu_priority _lowest_prio_cpu; + +/* mc2_task_state - a task state structure */ +struct mc2_task_state { + struct task_client res_info; + /* if cpu == -1, this task is a global task (level C) */ + int cpu; + bool has_departed; + struct mc2_task mc2_param; +}; + +/* crit_entry - maintain the logically running job (ghost job) */ +struct crit_entry { + enum crit_level level; + struct task_struct *running; +}; + +/* mc2_cpu_state - maintain the scheduled state and ghost jobs + * timer : timer for partitioned tasks (level A and B) + * g_timer : timer for global tasks (level C) + */ +struct mc2_cpu_state { + raw_spinlock_t lock; + + struct sup_reservation_environment sup_env; + struct hrtimer timer; + + int cpu; + struct task_struct* scheduled; + struct crit_entry crit_entries[NUM_CRIT_LEVELS]; +}; + +static int resched_cpu[NR_CPUS]; +static DEFINE_PER_CPU(struct mc2_cpu_state, mc2_cpu_state); +static int level_a_priorities[NR_CPUS]; + +#define cpu_state_for(cpu_id) (&per_cpu(mc2_cpu_state, cpu_id)) +#define local_cpu_state() (this_cpu_ptr(&mc2_cpu_state)) + +/* get_mc2_state - get the task's state */ +static struct mc2_task_state* get_mc2_state(struct task_struct *tsk) +{ + struct mc2_task_state* tinfo; + + tinfo = (struct mc2_task_state*)tsk_rt(tsk)->plugin_state; + + if (tinfo) + return tinfo; + else + return NULL; +} + +/* get_task_crit_level - return the criticaility level of a task */ +static enum crit_level get_task_crit_level(struct task_struct *tsk) +{ + struct mc2_task *mp; + + if (!tsk || !is_realtime(tsk)) + return NUM_CRIT_LEVELS; + + mp = tsk_rt(tsk)->mc2_data; + + if (!mp) + return NUM_CRIT_LEVELS; + else + return mp->crit; +} + +/* task_depart - remove a task from its reservation + * If the job has remaining budget, convert it to a ghost job + * and update crit_entries[] + * + * @job_complete indicate whether job completes or not + */ +static void task_departs(struct task_struct *tsk, int job_complete) +{ + struct mc2_task_state* tinfo = get_mc2_state(tsk); + + struct reservation* res = NULL; + struct reservation_client *client = NULL; + + BUG_ON(!is_realtime(tsk)); + + res = tinfo->res_info.client.reservation; + client = &tinfo->res_info.client; + BUG_ON(!res); + BUG_ON(!client); + + /* No ghost job handling, empty remaining budget */ + if (job_complete) { + res->cur_budget = 0; + } + + res->ops->client_departs(res, client, job_complete); + tinfo->has_departed = true; + TRACE_TASK(tsk, "Client departs with budget %llu at %llu\n", res->cur_budget, litmus_clock()); +} + +/* task_arrive - put a task into its reservation + * If the job was a ghost job, remove it from crit_entries[] + */ +static void task_arrives(struct mc2_cpu_state *state, struct task_struct *tsk) +{ + struct mc2_task_state* tinfo = get_mc2_state(tsk); + struct reservation* res; + struct reservation_client *client; + //enum crit_level lv = get_task_crit_level(tsk); + + res = tinfo->res_info.client.reservation; + client = &tinfo->res_info.client; + + tinfo->has_departed = false; + + res->ops->client_arrives(res, client); + TRACE_TASK(tsk, "Client arrives at %llu\n", litmus_clock()); +/* + if (lv != NUM_CRIT_LEVELS) { + struct crit_entry *ce; + ce = &state->crit_entries[lv]; + // if the currrent task is a ghost job, remove it + if (ce->running == tsk) + ce->running = NULL; + } +*/ +} + +/* get_lowest_prio_cpu - return the lowest priority cpu + * This will be used for scheduling level-C tasks. + * If all CPUs are running tasks which has + * higher priority than level C, return NO_CPU. + */ +static int get_lowest_prio_cpu(lt_t priority) +{ + struct cpu_entry *ce; + int cpu, ret = NO_CPU; + lt_t latest_deadline = 0; + + if (priority == LITMUS_NO_PRIORITY) + return ret; + + ce = &_lowest_prio_cpu.cpu_entries[local_cpu_state()->cpu]; + if (!ce->will_schedule && !ce->scheduled) { + TRACE("CPU %d (local) is the lowest!\n", ce->cpu); + return ce->cpu; + } else { + TRACE("Local CPU will_schedule=%d, scheduled=(%s/%d)\n", ce->will_schedule, ce->scheduled ? (ce->scheduled)->comm : "null", ce->scheduled ? (ce->scheduled)->pid : 0); + } + + for_each_online_cpu(cpu) { + ce = &_lowest_prio_cpu.cpu_entries[cpu]; + /* If a CPU will call schedule() in the near future, we don't + return that CPU. */ + /* + TRACE("CPU %d will_schedule=%d, scheduled=(%s/%d:%d)\n", cpu, ce->will_schedule, + ce->scheduled ? (ce->scheduled)->comm : "null", + ce->scheduled ? (ce->scheduled)->pid : 0, + ce->scheduled ? (ce->scheduled)->rt_param.job_params.job_no : 0); + */ + if (!ce->will_schedule) { + if (!ce->scheduled) { + /* Idle cpu, return this. */ + TRACE("CPU %d is the lowest!\n", ce->cpu); + return ce->cpu; + } else if (ce->lv == CRIT_LEVEL_C && + ce->deadline > latest_deadline) { + latest_deadline = ce->deadline; + ret = ce->cpu; + } + } + } + + if (priority >= latest_deadline) + ret = NO_CPU; + + TRACE("CPU %d is the lowest!\n", ret); + + return ret; +} + +/* NOTE: drops state->lock */ +/* mc2_update_timer_and_unlock - set a timer and g_timer and unlock + * Whenever res_env.current_time is updated, + * we check next_scheduler_update and set + * a timer. + * If there exist a global event which is + * not armed on any CPU and g_timer is not + * active, set a g_timer for that event. + */ +static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state) +{ + int local, cpus; + lt_t update, now; + struct next_timer_event *event, *next; + int reschedule[NR_CPUS]; + unsigned long flags; + + local_irq_save(flags); + + for (cpus = 0; cpussup_env.next_scheduler_update; + now = state->sup_env.env.current_time; + + /* Be sure we're actually running on the right core, + * as pres_update_timer() is also called from pres_task_resume(), + * which might be called on any CPU when a thread resumes. + */ + local = local_cpu_state() == state; + + raw_spin_lock(&_global_env.lock); + + list_for_each_entry_safe(event, next, &_global_env.next_events, list) { + /* If the event time is already passed, we call schedule() on + the lowest priority cpu */ + if (event->next_update >= update) { + break; + } + + if (event->next_update < litmus_clock()) { + if (event->timer_armed_on == NO_CPU) { + struct reservation *res = gmp_find_by_id(&_global_env, event->id); + int cpu = get_lowest_prio_cpu(res?res->priority:0); + //TRACE("GLOBAL EVENT PASSED!! poking CPU %d to reschedule\n", cpu); + list_del(&event->list); + kfree(event); + if (cpu != NO_CPU) { + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; + reschedule[cpu] = 1; + } + } + } else if (event->next_update < update && (event->timer_armed_on == NO_CPU || event->timer_armed_on == state->cpu)) { + event->timer_armed_on = state->cpu; + update = event->next_update; + break; + } + } + + /* Must drop state lock before calling into hrtimer_start(), which + * may raise a softirq, which in turn may wake ksoftirqd. */ + raw_spin_unlock(&_global_env.lock); + local_irq_restore(flags); + raw_spin_unlock(&state->lock); + + if (update <= now || reschedule[state->cpu]) { + reschedule[state->cpu] = 0; + litmus_reschedule(state->cpu); + } else if (likely(local && update != SUP_NO_SCHEDULER_UPDATE)) { + /* Reprogram only if not already set correctly. */ + if (!hrtimer_active(&state->timer) || + ktime_to_ns(hrtimer_get_expires(&state->timer)) != update) { + TRACE("canceling timer...at %llu\n", + ktime_to_ns(hrtimer_get_expires(&state->timer))); + hrtimer_cancel(&state->timer); + TRACE("setting scheduler timer for %llu\n", update); + /* We cannot use hrtimer_start() here because the + * wakeup flag must be set to zero. */ + __hrtimer_start_range_ns(&state->timer, + ns_to_ktime(update), + 0 /* timer coalescing slack */, + HRTIMER_MODE_ABS_PINNED, + 0 /* wakeup */); + if (update < litmus_clock()) { + /* uh oh, timer expired while trying to set it */ + TRACE("timer expired during setting " + "update:%llu now:%llu actual:%llu\n", + update, now, litmus_clock()); + /* The timer HW may not have been reprogrammed + * correctly; force rescheduling now. */ + litmus_reschedule(state->cpu); + } + } + } else if (unlikely(!local && update != SUP_NO_SCHEDULER_UPDATE)) { + /* Poke remote core only if timer needs to be set earlier than + * it is currently set. + */ + TRACE("mc2_update_timer for remote CPU %d (update=%llu, " + "active:%d, set:%llu)\n", + state->cpu, update, hrtimer_active(&state->timer), + ktime_to_ns(hrtimer_get_expires(&state->timer))); + if (!hrtimer_active(&state->timer) || + ktime_to_ns(hrtimer_get_expires(&state->timer)) > update) { + TRACE("poking CPU %d so that it can update its " + "scheduling timer (active:%d, set:%llu)\n", + state->cpu, + hrtimer_active(&state->timer), + ktime_to_ns(hrtimer_get_expires(&state->timer))); + //raw_spin_lock(&state->lock); + //preempt_if_preemptable(state->scheduled, state->cpu); + //raw_spin_unlock(&state->lock); + //reschedule[state->cpu] = 0; + } + } + + for (cpus = 0; cpuscpu]; + enum crit_level lv = get_task_crit_level(state->scheduled); + + if (!state->scheduled) { + /* cpu is idle. */ + ce->scheduled = NULL; + ce->deadline = ULLONG_MAX; + ce->lv = NUM_CRIT_LEVELS; + } else if (lv == CRIT_LEVEL_C) { + ce->scheduled = state->scheduled; + ce->deadline = get_deadline(state->scheduled); + ce->lv = lv; + } else if (lv < CRIT_LEVEL_C) { + /* If cpu is running level A or B tasks, it is not eligible + to run level-C tasks */ + ce->scheduled = state->scheduled; + ce->deadline = 0; + ce->lv = lv; + } +}; + +/* on_scheduling_timer - timer event for partitioned tasks + */ +static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer) +{ + unsigned long flags; + enum hrtimer_restart restart = HRTIMER_NORESTART; + struct mc2_cpu_state *state; + lt_t update, now; + int global_schedule_now; + int reschedule[NR_CPUS]; + int cpus; + + for (cpus = 0; cpuscpu != raw_smp_processor_id()); + + TS_ISR_START; + + TRACE("Timer fired at %llu\n", litmus_clock()); + raw_spin_lock_irqsave(&state->lock, flags); + now = litmus_clock(); + sup_update_time(&state->sup_env, now); + + update = state->sup_env.next_scheduler_update; + now = state->sup_env.env.current_time; + + if (update <= now) { + litmus_reschedule_local(); + } else if (update != SUP_NO_SCHEDULER_UPDATE) { + hrtimer_set_expires(timer, ns_to_ktime(update)); + restart = HRTIMER_RESTART; + } + + raw_spin_lock(&_global_env.lock); + global_schedule_now = gmp_update_time(&_global_env, now); + + BUG_ON(global_schedule_now < 0 || global_schedule_now > 4); + + /* Find the lowest cpu, and call reschedule */ + while (global_schedule_now--) { + int cpu = get_lowest_prio_cpu(0); + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) { + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; + if (cpu == state->cpu && update > now) + ; //litmus_reschedule_local(); + else + reschedule[cpu] = 1; + } + } + raw_spin_unlock(&_global_env.lock); + raw_spin_unlock_irqrestore(&state->lock, flags); + + TS_ISR_END; + + for (cpus = 0; cpuscompleted = 1; + + /* If this the first job instance, we need to reset replenish + time to the next release time */ + if (tsk_rt(current)->sporadic_release) { + struct mc2_cpu_state *state; + struct reservation_environment *env; + struct mc2_task_state *tinfo; + struct reservation *res = NULL; + unsigned long flags; + enum crit_level lv; + + //preempt_disable(); + local_irq_save(flags); + + tinfo = get_mc2_state(current); + lv = get_task_crit_level(current); + + if (lv < CRIT_LEVEL_C) { + state = cpu_state_for(tinfo->cpu); + raw_spin_lock(&state->lock); + env = &(state->sup_env.env); + res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id); + env->time_zero = tsk_rt(current)->sporadic_release_time; + } + else if (lv == CRIT_LEVEL_C) { + state = local_cpu_state(); + raw_spin_lock(&state->lock); + raw_spin_lock(&_global_env.lock); + res = gmp_find_by_id(&_global_env, tinfo->mc2_param.res_id); + _global_env.env.time_zero = tsk_rt(current)->sporadic_release_time; + } + else + BUG(); + + /* set next_replenishtime to synchronous release time */ + BUG_ON(!res); + res->next_replenishment = tsk_rt(current)->sporadic_release_time; + res->cur_budget = 0; + res->env->change_state(res->env, res, RESERVATION_DEPLETED); + + if (lv == CRIT_LEVEL_C) + raw_spin_unlock(&_global_env.lock); + + raw_spin_unlock(&state->lock); + local_irq_restore(flags); + //preempt_enable(); + } + + sched_trace_task_completion(current, 0); + /* update the next release time and deadline */ + sched_trace_task_release(current); + prepare_for_next_period(current); + next_release = ns_to_ktime(get_release(current)); + preempt_disable(); + TRACE_CUR("next_release=%llu\n", get_release(current)); + if (get_release(current) > litmus_clock()) { + /* sleep until next_release */ + set_current_state(TASK_INTERRUPTIBLE); + preempt_enable_no_resched(); + err = schedule_hrtimeout(&next_release, HRTIMER_MODE_ABS); + } else { + /* release the next job immediately */ + err = 0; + TRACE_CUR("TARDY: release=%llu now=%llu\n", get_release(current), litmus_clock()); + preempt_enable(); + } + + TRACE_CUR("mc2_complete_job returns at %llu\n", litmus_clock()); + + tsk_rt(current)->completed = 0; + return err; +} + +/* mc2_dispatch - Select the next task to schedule. + */ +struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, struct mc2_cpu_state* state) +{ + struct reservation *res, *next; + struct task_struct *tsk = NULL; + struct crit_entry *ce; + enum crit_level lv; + lt_t time_slice; + + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) { + if (res->state == RESERVATION_ACTIVE) { + tsk = res->ops->dispatch_client(res, &time_slice); + if (likely(tsk)) { + lv = get_task_crit_level(tsk); + if (lv == NUM_CRIT_LEVELS) { + sup_scheduler_update_after(sup_env, res->cur_budget); + return tsk; + } else { + ce = &state->crit_entries[lv]; + sup_scheduler_update_after(sup_env, res->cur_budget); + res->blocked_by_ghost = 0; + res->is_ghost = NO_CPU; + return tsk; + } + } + } + } + + return NULL; +} + +struct task_struct* mc2_global_dispatch(struct mc2_cpu_state* state) +{ + struct reservation *res, *next; + struct task_struct *tsk = NULL; + enum crit_level lv; + lt_t time_slice; + + list_for_each_entry_safe(res, next, &_global_env.active_reservations, list) { + BUG_ON(!res); + if (res->state == RESERVATION_ACTIVE && res->scheduled_on == NO_CPU) { + tsk = res->ops->dispatch_client(res, &time_slice); + if (likely(tsk)) { + lv = get_task_crit_level(tsk); + if (lv != CRIT_LEVEL_C) + BUG(); +#if BUDGET_ENFORCEMENT_AT_C + gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN); +#endif + res->event_added = 1; + res->blocked_by_ghost = 0; + res->is_ghost = NO_CPU; + res->scheduled_on = state->cpu; + return tsk; + } + } + } + + return NULL; +} + +static inline void pre_schedule(struct task_struct *prev, int cpu) +{ + TS_SCHED_A_START; + TS_SCHED_C_START; + + if (!prev || !is_realtime(prev)) + return; + + do_partition(CRIT_LEVEL_C, cpu); +} + +static inline void post_schedule(struct task_struct *next, int cpu) +{ + enum crit_level lev; + if ((!next) || !is_realtime(next)) { + //do_partition(NUM_CRIT_LEVELS, -1); + return; + } + + lev = get_task_crit_level(next); + do_partition(lev, cpu); + + switch(lev) { + case CRIT_LEVEL_A: + case CRIT_LEVEL_B: + TS_SCHED_A_END(next); + break; + case CRIT_LEVEL_C: + TS_SCHED_C_END(next); + break; + default: + break; + } + +} + +/* mc2_schedule - main scheduler function. pick the next task to run + */ +static struct task_struct* mc2_schedule(struct task_struct * prev) +{ + int np, blocks, exists, cpu; //preempt, to_schedule; + /* next == NULL means "schedule background work". */ + lt_t now = litmus_clock(); + struct mc2_cpu_state *state = local_cpu_state(); + + raw_spin_lock(&state->lock); + + pre_schedule(prev, state->cpu); + + if (state->scheduled && state->scheduled != prev) + printk(KERN_ALERT "BUG1!!!!!!!! %s %s\n", state->scheduled ? (state->scheduled)->comm : "null", prev ? (prev)->comm : "null"); + if (state->scheduled && !is_realtime(prev)) + printk(KERN_ALERT "BUG2!!!!!!!! \n"); + + /* (0) Determine state */ + exists = state->scheduled != NULL; + blocks = exists && !is_current_running(); + np = exists && is_np(state->scheduled); + + /* update time */ + state->sup_env.will_schedule = true; + sup_update_time(&state->sup_env, now); + + if (is_realtime(current) && blocks) { + if (get_task_crit_level(current) == CRIT_LEVEL_C) + raw_spin_lock(&_global_env.lock); + task_departs(current, is_completed(current)); + if (get_task_crit_level(current) == CRIT_LEVEL_C) + raw_spin_unlock(&_global_env.lock); + } + + /* figure out what to schedule next */ + if (!np) + state->scheduled = mc2_dispatch(&state->sup_env, state); + + if (!state->scheduled) { + raw_spin_lock(&_global_env.lock); + if (is_realtime(prev)) + gmp_update_time(&_global_env, now); + state->scheduled = mc2_global_dispatch(state); + _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false; + update_cpu_prio(state); + raw_spin_unlock(&_global_env.lock); + } else { + raw_spin_lock(&_global_env.lock); + _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false; + update_cpu_prio(state); + raw_spin_unlock(&_global_env.lock); + } + + /* Notify LITMUS^RT core that we've arrived at a scheduling decision. */ + sched_state_task_picked(); + + /* program scheduler timer */ + state->sup_env.will_schedule = false; + + /* NOTE: drops state->lock */ + mc2_update_timer_and_unlock(state); + + raw_spin_lock(&state->lock); + if (prev != state->scheduled && is_realtime(prev)) { + struct mc2_task_state* tinfo = get_mc2_state(prev); + struct reservation* res = tinfo->res_info.client.reservation; + res->scheduled_on = NO_CPU; + TRACE_TASK(prev, "descheduled at %llu.\n", litmus_clock()); + /* if prev is preempted and a global task, find the lowest cpu and reschedule */ + if (tinfo->has_departed == false && get_task_crit_level(prev) == CRIT_LEVEL_C) { + int cpu; + raw_spin_lock(&_global_env.lock); + cpu = get_lowest_prio_cpu(res?res->priority:LITMUS_NO_PRIORITY); + //TRACE("LEVEL-C TASK PREEMPTED!! poking CPU %d to reschedule\n", cpu); + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) { + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; + resched_cpu[cpu] = 1; + } + raw_spin_unlock(&_global_env.lock); + } + } + +/* + if (to_schedule != 0) { + raw_spin_lock(&_global_env.lock); + while (to_schedule--) { + int cpu = get_lowest_prio_cpu(0); + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) { + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; + resched_cpu[cpu] = 1; + } + } + raw_spin_unlock(&_global_env.lock); + } +*/ + + post_schedule(state->scheduled, state->cpu); + + raw_spin_unlock(&state->lock); + if (state->scheduled) { + TRACE_TASK(state->scheduled, "scheduled.\n"); + } + + return state->scheduled; +} + +/* mc2_task_resume - Called when the state of tsk changes back to + * TASK_RUNNING. We need to requeue the task. + */ +static void mc2_task_resume(struct task_struct *tsk) +{ + unsigned long flags; + struct mc2_task_state* tinfo; + struct mc2_cpu_state *state; + + TRACE_TASK(tsk, "thread wakes up at %llu\n", litmus_clock()); + + preempt_disable(); + tinfo = get_mc2_state(tsk); + if (tinfo->cpu != -1) + state = cpu_state_for(tinfo->cpu); + else + state = local_cpu_state(); + preempt_enable(); + + /* Requeue only if self-suspension was already processed. */ + if (tinfo->has_departed) + { +#ifdef CONFIG_SCHED_OVERHEAD_TRACE + switch(get_task_crit_level(tsk)) { + case CRIT_LEVEL_A: + TS_RELEASE_LATENCY_A(get_release(tsk)); + break; + case CRIT_LEVEL_B: + TS_RELEASE_LATENCY_B(get_release(tsk)); + break; + case CRIT_LEVEL_C: + TS_RELEASE_LATENCY_C(get_release(tsk)); + break; + default: + break; + } +#endif + + raw_spin_lock_irqsave(&state->lock, flags); + + /* Assumption: litmus_clock() is synchronized across cores, + * since we might not actually be executing on tinfo->cpu + * at the moment. */ + if (tinfo->cpu != -1) { + sup_update_time(&state->sup_env, litmus_clock()); + task_arrives(state, tsk); + } else { + raw_spin_lock(&_global_env.lock); + gmp_update_time(&_global_env, litmus_clock()); + task_arrives(state, tsk); + raw_spin_unlock(&_global_env.lock); + } + + /* NOTE: drops state->lock */ + TRACE_TASK(tsk, "mc2_resume()\n"); + raw_spin_unlock_irqrestore(&state->lock, flags); + + raw_spin_lock(&state->lock); + mc2_update_timer_and_unlock(state); + } else { + TRACE_TASK(tsk, "resume event ignored, still scheduled\n"); + } + +} + + +/* mc2_admit_task - Setup mc2 task parameters + */ +static long mc2_admit_task(struct task_struct *tsk) +{ + long err = 0; + unsigned long flags; + struct reservation *res; + struct mc2_cpu_state *state; + struct mc2_task_state *tinfo = kzalloc(sizeof(*tinfo), GFP_ATOMIC); + struct mc2_task *mp = tsk_rt(tsk)->mc2_data; + enum crit_level lv; + + if (!tinfo) + return -ENOMEM; + + if (!mp) { + printk(KERN_ERR "mc2_admit_task: criticality level has not been set\n"); + return -ESRCH; + } + + lv = mp->crit; + + if (lv < CRIT_LEVEL_C) { + state = cpu_state_for(task_cpu(tsk)); + raw_spin_lock_irqsave(&state->lock, flags); + + res = sup_find_by_id(&state->sup_env, mp->res_id); + + /* found the appropriate reservation */ + if (res) { + TRACE_TASK(tsk, "SUP FOUND RES ID\n"); + tinfo->mc2_param.crit = mp->crit; + tinfo->mc2_param.res_id = mp->res_id; + + /* initial values */ + err = mc2_task_client_init(&tinfo->res_info, &tinfo->mc2_param, tsk, res); + tinfo->cpu = task_cpu(tsk); + tinfo->has_departed = true; + tsk_rt(tsk)->plugin_state = tinfo; + + /* disable LITMUS^RT's per-thread budget enforcement */ + tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT; + } + else { + err = -ESRCH; + } + + raw_spin_unlock_irqrestore(&state->lock, flags); + } else if (lv == CRIT_LEVEL_C) { + state = local_cpu_state(); + raw_spin_lock_irqsave(&state->lock, flags); + raw_spin_lock(&_global_env.lock); + + res = gmp_find_by_id(&_global_env, mp->res_id); + + /* found the appropriate reservation (or vCPU) */ + if (res) { + TRACE_TASK(tsk, "GMP FOUND RES ID\n"); + tinfo->mc2_param.crit = mp->crit; + tinfo->mc2_param.res_id = mp->res_id; + + /* initial values */ + err = mc2_task_client_init(&tinfo->res_info, &tinfo->mc2_param, tsk, res); + tinfo->cpu = -1; + tinfo->has_departed = true; + tsk_rt(tsk)->plugin_state = tinfo; + + /* disable LITMUS^RT's per-thread budget enforcement */ + tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT; + } + else { + err = -ESRCH; + } + + raw_spin_unlock(&_global_env.lock); + raw_spin_unlock_irqrestore(&state->lock, flags); + } + + if (err) + kfree(tinfo); + + return err; +} + +/* mc2_task_new - A new real-time job is arrived. Release the next job + * at the next reservation replenish time + */ +static void mc2_task_new(struct task_struct *tsk, int on_runqueue, + int is_running) +{ + unsigned long flags; + struct mc2_task_state* tinfo = get_mc2_state(tsk); + struct mc2_cpu_state *state; // = cpu_state_for(tinfo->cpu); + struct reservation *res; + enum crit_level lv = get_task_crit_level(tsk); + lt_t release = 0; + + BUG_ON(lv < CRIT_LEVEL_A || lv > CRIT_LEVEL_C); + + TRACE_TASK(tsk, "new RT task %llu (on_rq:%d, running:%d)\n", + litmus_clock(), on_runqueue, is_running); + + if (tinfo->cpu == -1) + state = local_cpu_state(); + else + state = cpu_state_for(tinfo->cpu); + + + if (is_running) { + state->scheduled = tsk; + /* make sure this task should actually be running */ + litmus_reschedule_local(); + } + + /* acquire the lock protecting the state and disable interrupts */ + local_irq_save(flags); + raw_spin_lock(&state->lock); + + if (lv == CRIT_LEVEL_C) { + raw_spin_lock(&_global_env.lock); + res = gmp_find_by_id(&_global_env, tinfo->mc2_param.res_id); + } + else { + res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id); + } + + if (on_runqueue || is_running) { + /* Assumption: litmus_clock() is synchronized across cores + * [see comment in pres_task_resume()] */ + if (lv == CRIT_LEVEL_C) { + gmp_update_time(&_global_env, litmus_clock()); + } + else + sup_update_time(&state->sup_env, litmus_clock()); + + task_arrives(state, tsk); + if (lv == CRIT_LEVEL_C) + raw_spin_unlock(&_global_env.lock); + /* NOTE: drops state->lock */ + raw_spin_unlock(&state->lock); + local_irq_restore(flags); + + TRACE("mc2_new()\n"); + + raw_spin_lock(&state->lock); + mc2_update_timer_and_unlock(state); + } else { + if (lv == CRIT_LEVEL_C) + raw_spin_unlock(&_global_env.lock); + raw_spin_unlock(&state->lock); + local_irq_restore(flags); + } + release = res->next_replenishment; + + if (!release) { + TRACE_TASK(tsk, "mc2_task_new() next_release = %llu\n", release); + BUG(); + } + else + TRACE_TASK(tsk, "mc2_task_new() next_release = NULL\n"); +} + +/* mc2_reservation_destroy - reservation_destroy system call backend + */ +static long mc2_reservation_destroy(unsigned int reservation_id, int cpu) +{ + long ret = -EINVAL; + struct mc2_cpu_state *state; + struct reservation *res = NULL, *next; + struct sup_reservation_environment *sup_env; + int found = 0; + unsigned long flags; + + if (cpu == -1) { + struct next_timer_event *event, *e_next; + + /* if the reservation is global reservation */ + + local_irq_save(flags); + raw_spin_lock(&_global_env.lock); + + list_for_each_entry_safe(res, next, &_global_env.depleted_reservations, list) { + if (res->id == reservation_id) { + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + if (!found) { + list_for_each_entry_safe(res, next, &_global_env.inactive_reservations, list) { + if (res->id == reservation_id) { + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + } + if (!found) { + list_for_each_entry_safe(res, next, &_global_env.active_reservations, list) { + if (res->id == reservation_id) { + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + } + /* delete corresponding events */ + list_for_each_entry_safe(event, e_next, &_global_env.next_events, list) { + if (event->id == reservation_id) { + list_del(&event->list); + kfree(event); + } + } + + raw_spin_unlock(&_global_env.lock); + local_irq_restore(flags); + } else { + /* if the reservation is partitioned reservation */ + state = cpu_state_for(cpu); + local_irq_save(flags); + raw_spin_lock(&state->lock); + + sup_env = &state->sup_env; + list_for_each_entry_safe(res, next, &sup_env->depleted_reservations, list) { + if (res->id == reservation_id) { + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + if (!found) { + list_for_each_entry_safe(res, next, &sup_env->inactive_reservations, list) { + if (res->id == reservation_id) { + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + } + if (!found) { + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) { + if (res->id == reservation_id) { + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + } + + raw_spin_unlock(&state->lock); + local_irq_restore(flags); + } + + TRACE("Rerservation destroyed ret = %d\n", ret); + return ret; +} + +/* mc2_task_exit - Task became a normal task (not real-time task) + */ +static void mc2_task_exit(struct task_struct *tsk) +{ + unsigned long flags; + struct mc2_task_state* tinfo = get_mc2_state(tsk); + struct mc2_cpu_state *state; + enum crit_level lv = tinfo->mc2_param.crit; + struct crit_entry* ce; + int cpu; + + local_irq_save(flags); + if (tinfo->cpu != -1) + state = cpu_state_for(tinfo->cpu); + else + state = local_cpu_state(); + + raw_spin_lock(&state->lock); + + if (state->scheduled == tsk) + state->scheduled = NULL; + + ce = &state->crit_entries[lv]; + if (ce->running == tsk) + ce->running = NULL; + + /* remove from queues */ + if (is_running(tsk)) { + /* Assumption: litmus_clock() is synchronized across cores + * [see comment in pres_task_resume()] */ + + /* update both global and partitioned */ + if (lv < CRIT_LEVEL_C) { + sup_update_time(&state->sup_env, litmus_clock()); + } + else if (lv == CRIT_LEVEL_C) { + raw_spin_lock(&_global_env.lock); + gmp_update_time(&_global_env, litmus_clock()); + } + task_departs(tsk, 0); + if (lv == CRIT_LEVEL_C) + raw_spin_unlock(&_global_env.lock); + + /* NOTE: drops state->lock */ + TRACE("mc2_exit()\n"); + + mc2_update_timer_and_unlock(state); + } else { + raw_spin_unlock(&state->lock); + } + + if (lv == CRIT_LEVEL_C) { + for_each_online_cpu(cpu) { + state = cpu_state_for(cpu); + if (state == local_cpu_state()) + continue; + raw_spin_lock(&state->lock); + + if (state->scheduled == tsk) + state->scheduled = NULL; + + ce = &state->crit_entries[lv]; + if (ce->running == tsk) + ce->running = NULL; + + raw_spin_unlock(&state->lock); + } + } + + local_irq_restore(flags); + + kfree(tsk_rt(tsk)->plugin_state); + tsk_rt(tsk)->plugin_state = NULL; + kfree(tsk_rt(tsk)->mc2_data); + tsk_rt(tsk)->mc2_data = NULL; +} + +/* create_polling_reservation - create a new polling reservation + */ +static long create_polling_reservation( + int res_type, + struct reservation_config *config) +{ + struct mc2_cpu_state *state; + struct reservation* res; + struct polling_reservation *pres; + unsigned long flags; + int use_edf = config->priority == LITMUS_NO_PRIORITY; + int periodic = res_type == PERIODIC_POLLING; + long err = -EINVAL; + + /* sanity checks */ + if (config->polling_params.budget > + config->polling_params.period) { + printk(KERN_ERR "invalid polling reservation (%u): " + "budget > period\n", config->id); + return -EINVAL; + } + if (config->polling_params.budget > + config->polling_params.relative_deadline + && config->polling_params.relative_deadline) { + printk(KERN_ERR "invalid polling reservation (%u): " + "budget > deadline\n", config->id); + return -EINVAL; + } + if (config->polling_params.offset > + config->polling_params.period) { + printk(KERN_ERR "invalid polling reservation (%u): " + "offset > period\n", config->id); + return -EINVAL; + } + + /* Allocate before we grab a spin lock. + * Todo: would be nice to use a core-local allocation. + */ + pres = kzalloc(sizeof(*pres), GFP_KERNEL); + if (!pres) + return -ENOMEM; + + if (config->cpu != -1) { + state = cpu_state_for(config->cpu); + raw_spin_lock_irqsave(&state->lock, flags); + + res = sup_find_by_id(&state->sup_env, config->id); + if (!res) { + polling_reservation_init(pres, use_edf, periodic, + config->polling_params.budget, + config->polling_params.period, + config->polling_params.relative_deadline, + config->polling_params.offset); + pres->res.id = config->id; + pres->res.blocked_by_ghost = 0; + pres->res.is_ghost = NO_CPU; + if (!use_edf) + pres->res.priority = config->priority; + sup_add_new_reservation(&state->sup_env, &pres->res); + err = config->id; + TRACE_CUR("reservation created R%d priority : %llu\n", config->id, pres->res.priority); + } else { + err = -EEXIST; + } + + raw_spin_unlock_irqrestore(&state->lock, flags); + + } else { + raw_spin_lock_irqsave(&_global_env.lock, flags); + + res = gmp_find_by_id(&_global_env, config->id); + if (!res) { + polling_reservation_init(pres, use_edf, periodic, + config->polling_params.budget, + config->polling_params.period, + config->polling_params.relative_deadline, + config->polling_params.offset); + pres->res.id = config->id; + pres->res.blocked_by_ghost = 0; + pres->res.scheduled_on = NO_CPU; + pres->res.is_ghost = NO_CPU; + if (!use_edf) + pres->res.priority = config->priority; + gmp_add_new_reservation(&_global_env, &pres->res); + err = config->id; + } else { + err = -EEXIST; + } + raw_spin_unlock_irqrestore(&_global_env.lock, flags); + } + + if (err < 0) + kfree(pres); + + return err; +} + +#define MAX_INTERVALS 1024 + +/* create_table_driven_reservation - create a table_driven reservation + */ +static long create_table_driven_reservation( + struct reservation_config *config) +{ + struct mc2_cpu_state *state; + struct reservation* res; + struct table_driven_reservation *td_res = NULL; + struct lt_interval *slots = NULL; + size_t slots_size; + unsigned int i, num_slots; + unsigned long flags; + long err = -EINVAL; + + + if (!config->table_driven_params.num_intervals) { + printk(KERN_ERR "invalid table-driven reservation (%u): " + "no intervals\n", config->id); + return -EINVAL; + } + + if (config->table_driven_params.num_intervals > MAX_INTERVALS) { + printk(KERN_ERR "invalid table-driven reservation (%u): " + "too many intervals (max: %d)\n", config->id, MAX_INTERVALS); + return -EINVAL; + } + + num_slots = config->table_driven_params.num_intervals; + slots_size = sizeof(slots[0]) * num_slots; + slots = kzalloc(slots_size, GFP_KERNEL); + if (!slots) + return -ENOMEM; + + td_res = kzalloc(sizeof(*td_res), GFP_KERNEL); + if (!td_res) + err = -ENOMEM; + else + err = copy_from_user(slots, + config->table_driven_params.intervals, slots_size); + + if (!err) { + /* sanity checks */ + for (i = 0; !err && i < num_slots; i++) + if (slots[i].end <= slots[i].start) { + printk(KERN_ERR + "invalid table-driven reservation (%u): " + "invalid interval %u => [%llu, %llu]\n", + config->id, i, + slots[i].start, slots[i].end); + err = -EINVAL; + } + + for (i = 0; !err && i + 1 < num_slots; i++) + if (slots[i + 1].start <= slots[i].end) { + printk(KERN_ERR + "invalid table-driven reservation (%u): " + "overlapping intervals %u, %u\n", + config->id, i, i + 1); + err = -EINVAL; + } + + if (slots[num_slots - 1].end > + config->table_driven_params.major_cycle_length) { + printk(KERN_ERR + "invalid table-driven reservation (%u): last " + "interval ends past major cycle %llu > %llu\n", + config->id, + slots[num_slots - 1].end, + config->table_driven_params.major_cycle_length); + err = -EINVAL; + } + } + + if (!err) { + state = cpu_state_for(config->cpu); + raw_spin_lock_irqsave(&state->lock, flags); + + res = sup_find_by_id(&state->sup_env, config->id); + if (!res) { + table_driven_reservation_init(td_res, + config->table_driven_params.major_cycle_length, + slots, num_slots); + td_res->res.id = config->id; + td_res->res.priority = config->priority; + td_res->res.blocked_by_ghost = 0; + sup_add_new_reservation(&state->sup_env, &td_res->res); + err = config->id; + } else { + err = -EEXIST; + } + + raw_spin_unlock_irqrestore(&state->lock, flags); + } + + if (err < 0) { + kfree(slots); + kfree(td_res); + } + + return err; +} + +/* mc2_reservation_create - reservation_create system call backend + */ +static long mc2_reservation_create(int res_type, void* __user _config) +{ + long ret = -EINVAL; + struct reservation_config config; + + TRACE("Attempt to create reservation (%d)\n", res_type); + + if (copy_from_user(&config, _config, sizeof(config))) + return -EFAULT; + + if (config.cpu != -1) { + if (config.cpu < 0 || !cpu_online(config.cpu)) { + printk(KERN_ERR "invalid polling reservation (%u): " + "CPU %d offline\n", config.id, config.cpu); + return -EINVAL; + } + } + + switch (res_type) { + case PERIODIC_POLLING: + case SPORADIC_POLLING: + ret = create_polling_reservation(res_type, &config); + break; + + case TABLE_DRIVEN: + ret = create_table_driven_reservation(&config); + break; + + default: + return -EINVAL; + }; + + return ret; +} + +static struct domain_proc_info mc2_domain_proc_info; + +static long mc2_get_domain_proc_info(struct domain_proc_info **ret) +{ + *ret = &mc2_domain_proc_info; + return 0; +} + +static void mc2_setup_domain_proc(void) +{ + int i, cpu; + int num_rt_cpus = num_online_cpus(); + + struct cd_mapping *cpu_map, *domain_map; + + memset(&mc2_domain_proc_info, sizeof(mc2_domain_proc_info), 0); + init_domain_proc_info(&mc2_domain_proc_info, num_rt_cpus, num_rt_cpus); + mc2_domain_proc_info.num_cpus = num_rt_cpus; + mc2_domain_proc_info.num_domains = num_rt_cpus; + + i = 0; + for_each_online_cpu(cpu) { + cpu_map = &mc2_domain_proc_info.cpu_to_domains[i]; + domain_map = &mc2_domain_proc_info.domain_to_cpus[i]; + + cpu_map->id = cpu; + domain_map->id = i; + cpumask_set_cpu(i, cpu_map->mask); + cpumask_set_cpu(cpu, domain_map->mask); + ++i; + } +} + +static long mc2_activate_plugin(void) +{ + int cpu, lv; + struct mc2_cpu_state *state; + struct cpu_entry *ce; + + gmp_init(&_global_env); + //raw_spin_lock_init(&_lowest_prio_cpu.lock); + + for_each_online_cpu(cpu) { + TRACE("Initializing CPU%d...\n", cpu); + + resched_cpu[cpu] = 0; + level_a_priorities[cpu] = 0; + state = cpu_state_for(cpu); + ce = &_lowest_prio_cpu.cpu_entries[cpu]; + + ce->cpu = cpu; + ce->scheduled = NULL; + ce->deadline = ULLONG_MAX; + ce->lv = NUM_CRIT_LEVELS; + ce->will_schedule = false; + + raw_spin_lock_init(&state->lock); + state->cpu = cpu; + state->scheduled = NULL; + for (lv = 0; lv < NUM_CRIT_LEVELS; lv++) { + struct crit_entry *cr_entry = &state->crit_entries[lv]; + cr_entry->level = lv; + cr_entry->running = NULL; + } + sup_init(&state->sup_env); + + hrtimer_init(&state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); + state->timer.function = on_scheduling_timer; + } + + mc2_setup_domain_proc(); + + return 0; +} + +static void mc2_finish_switch(struct task_struct *prev) +{ + int cpus; + enum crit_level lv = get_task_crit_level(prev); + struct mc2_cpu_state *state = local_cpu_state(); + + state->scheduled = is_realtime(current) ? current : NULL; + if (lv == CRIT_LEVEL_C) { + for (cpus = 0; cpuscpu != cpus) { + resched_cpu[cpus] = 0; + litmus_reschedule(cpus); + } + } + } +} + +static long mc2_deactivate_plugin(void) +{ + int cpu; + struct mc2_cpu_state *state; + struct reservation *res; + struct next_timer_event *event; + struct cpu_entry *ce; + + for_each_online_cpu(cpu) { + state = cpu_state_for(cpu); + raw_spin_lock(&state->lock); + + hrtimer_cancel(&state->timer); + + ce = &_lowest_prio_cpu.cpu_entries[cpu]; + + ce->cpu = cpu; + ce->scheduled = NULL; + ce->deadline = ULLONG_MAX; + ce->lv = NUM_CRIT_LEVELS; + ce->will_schedule = false; + + /* Delete all reservations --- assumes struct reservation + * is prefix of containing struct. */ + + while (!list_empty(&state->sup_env.active_reservations)) { + res = list_first_entry( + &state->sup_env.active_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&state->sup_env.inactive_reservations)) { + res = list_first_entry( + &state->sup_env.inactive_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&state->sup_env.depleted_reservations)) { + res = list_first_entry( + &state->sup_env.depleted_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + raw_spin_unlock(&state->lock); + } + + raw_spin_lock(&_global_env.lock); + + while (!list_empty(&_global_env.active_reservations)) { + res = list_first_entry( + &_global_env.active_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&_global_env.inactive_reservations)) { + res = list_first_entry( + &_global_env.inactive_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&_global_env.depleted_reservations)) { + res = list_first_entry( + &_global_env.depleted_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&_global_env.next_events)) { + event = list_first_entry( + &_global_env.next_events, + struct next_timer_event, list); + list_del(&event->list); + kfree(event); + } + + raw_spin_unlock(&_global_env.lock); + + destroy_domain_proc_info(&mc2_domain_proc_info); + return 0; +} + +static struct sched_plugin mc2_plugin = { + .plugin_name = "MC2", + .schedule = mc2_schedule, + .finish_switch = mc2_finish_switch, + .task_wake_up = mc2_task_resume, + .admit_task = mc2_admit_task, + .task_new = mc2_task_new, + .task_exit = mc2_task_exit, + .complete_job = mc2_complete_job, + .get_domain_proc_info = mc2_get_domain_proc_info, + .activate_plugin = mc2_activate_plugin, + .deactivate_plugin = mc2_deactivate_plugin, + .reservation_create = mc2_reservation_create, + .reservation_destroy = mc2_reservation_destroy, +}; + +static int __init init_mc2(void) +{ + return register_sched_plugin(&mc2_plugin); +} + +module_init(init_mc2); diff --git litmus/sched_plugin.c litmus/sched_plugin.c index edd91e9..7b1eba0 100644 --- litmus/sched_plugin.c +++ litmus/sched_plugin.c @@ -13,6 +13,7 @@ #include #include #include +#include /* * Generic function to trigger preemption on either local or remote cpu @@ -197,6 +198,9 @@ int register_sched_plugin(struct sched_plugin* plugin) if (!plugin->wait_for_release_at) plugin->wait_for_release_at = default_wait_for_release_at; + if (!plugin->current_budget) + plugin->current_budget = litmus_current_budget; + raw_spin_lock(&sched_plugins_lock); list_add(&plugin->list, &sched_plugins); raw_spin_unlock(&sched_plugins_lock); diff --git litmus/sched_psn_edf.c litmus/sched_psn_edf.c index 2549a3f..216b9f3 100644 --- litmus/sched_psn_edf.c +++ litmus/sched_psn_edf.c @@ -23,6 +23,10 @@ #include #include +#ifdef CONFIG_PGMRT_SUPPORT +#include +#endif + /* to set up domain/cpu mappings */ #include @@ -199,6 +203,62 @@ static struct task_struct* psnedf_schedule(struct task_struct * prev) */ resched = preempt; +#ifdef CONFIG_PGMRT_SUPPORT + if (exists) { + if (is_pgm_sending(pedf->scheduled)) { + if (!is_pgm_satisfied(pedf->scheduled)) { + if (!is_priority_boosted(pedf->scheduled)) { + TRACE_TASK(pedf->scheduled, "is sending PGM tokens and needs boosting.\n"); + BUG_ON(is_pgm_satisfied(pedf->scheduled)); + + /* We are either sending tokens or waiting for tokes. + If waiting: Boost priority so we'll be scheduled + immediately when needed tokens arrive. + If sending: Boost priority so no one (specifically, our + consumers) will preempt us while signalling the token + transmission. + */ + tsk_rt(pedf->scheduled)->priority_boosted = 1; + tsk_rt(pedf->scheduled)->boost_start_time = litmus_clock(); + + if (likely(!blocks)) { + requeue(pedf->scheduled, edf); + /* we may regain the processor */ + if (preempt) { + preempt = edf_preemption_needed(edf, prev); + if (!preempt) { + TRACE_TASK(pedf->scheduled, "blocked preemption by lazy boosting.\n"); + } + } + } + } + } + else { /* sending is satisfied */ + tsk_rt(pedf->scheduled)->ctrl_page->pgm_sending = 0; + tsk_rt(pedf->scheduled)->ctrl_page->pgm_satisfied = 0; + + if (is_priority_boosted(pedf->scheduled)) { + TRACE_TASK(pedf->scheduled, + "is done sending PGM tokens must relinquish boosting.\n"); + /* clear boosting */ + tsk_rt(pedf->scheduled)->priority_boosted = 0; + if(likely(!blocks)) { + /* recheck priority */ + requeue(pedf->scheduled, edf); + /* we may lose the processor */ + if (!preempt) { + preempt = edf_preemption_needed(edf, prev); + if (preempt) { + TRACE_TASK(pedf->scheduled, "preempted by lazy unboosting.\n"); + } + } + } + } + } + } + } +#endif + /* If a task blocks we have no choice but to reschedule. */ if (blocks) @@ -243,7 +303,7 @@ static struct task_struct* psnedf_schedule(struct task_struct * prev) if (next) { TRACE_TASK(next, "scheduled at %llu\n", litmus_clock()); } else { - TRACE("becoming idle at %llu\n", litmus_clock()); + ; //TRACE("becoming idle at %llu\n", litmus_clock()); } pedf->scheduled = next; @@ -644,10 +704,14 @@ static long psnedf_admit_task(struct task_struct* tsk) /* don't allow tasks on release master CPU */ && task_cpu(tsk) != remote_edf(task_cpu(tsk))->release_master #endif - ) + ) { + TRACE_TASK(tsk, "admitted\n"); return 0; - else + } + else { + TRACE_TASK(tsk, "not admitted\n"); return -EINVAL; + } } /* Plugin object */ diff --git litmus/sched_task_trace.c litmus/sched_task_trace.c index c154ec4..c25bc5a 100644 --- litmus/sched_task_trace.c +++ litmus/sched_task_trace.c @@ -132,7 +132,6 @@ feather_callback void do_sched_trace_task_param(unsigned long id, unsigned long rec->data.param.period = get_rt_period(t); rec->data.param.phase = get_rt_phase(t); rec->data.param.partition = get_partition(t); - rec->data.param.class = get_class(t); put_record(rec); } } @@ -189,6 +188,7 @@ feather_callback void do_sched_trace_task_completion(unsigned long id, if (rec) { rec->data.completion.when = now(); rec->data.completion.forced = forced; + rec->data.completion.exec_time = get_exec_time(t); put_record(rec); } } diff --git litmus/uncachedev.c litmus/uncachedev.c index 06a6a7c..8687581 100644 --- litmus/uncachedev.c +++ litmus/uncachedev.c @@ -28,8 +28,8 @@ int litmus_uncache_vm_fault(struct vm_area_struct* vma, /* modeled after SG DMA video4linux, but without DMA. */ /* (see drivers/media/video/videobuf-dma-sg.c) */ struct page *page; - - page = alloc_page(GFP_USER); + + page = alloc_page(GFP_USER|GFP_COLOR); if (!page) return VM_FAULT_OOM; diff --git mm/dmapool.c mm/dmapool.c index fd5fe43..b69dc13 100644 --- mm/dmapool.c +++ mm/dmapool.c @@ -333,7 +333,7 @@ void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags, /* pool_alloc_page() might sleep, so temporarily drop &pool->lock */ spin_unlock_irqrestore(&pool->lock, flags); - + printk(KERN_INFO "dma_pool_alloc(): called with %x flags\n", mem_flags); page = pool_alloc_page(pool, mem_flags); if (!page) return NULL; diff --git mm/migrate.c mm/migrate.c index f53838f..29b69cd 100644 --- mm/migrate.c +++ mm/migrate.c @@ -38,6 +38,8 @@ #include #include +#include // for TRACE_TASK + #include #define CREATE_TRACE_POINTS @@ -391,6 +393,65 @@ int migrate_page_move_mapping(struct address_space *mapping, } /* + * Replace the page in the mapping. + * + * The number of remaining references must be: + * 1 for anonymous pages without a mapping + * 2 for pages with a mapping + * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. + */ +int replicate_page_move_mapping(struct address_space *mapping, + struct page *newpage, struct page *page, + struct buffer_head *head, enum migrate_mode mode, + int extra_count) +{ + int prev_count = page_count(page); + void **pslot; + + BUG_ON(!mapping); + + spin_lock_irq(&mapping->tree_lock); + + pslot = radix_tree_lookup_slot(&mapping->page_tree, page_index(page)); + + /* + * Now we know that no one else is looking at the page. + */ + get_page(newpage); /* add cache reference */ + if (PageSwapCache(page)) { + SetPageSwapCache(newpage); + set_page_private(newpage, page_private(page)); + } + + /* + * Drop cache reference from old page by unfreezing + * to the previous reference. + * We know this isn't the last reference. + */ + page_unfreeze_refs(page, prev_count); + + /* + * If moved to a different zone then also account + * the page for that zone. Other VM counters will be + * taken care of when we establish references to the + * new page and drop references to the old page. + * + * Note that anonymous pages are accounted for + * via NR_FILE_PAGES and NR_ANON_PAGES if they + * are mapped to swap space. + */ + __dec_zone_page_state(page, NR_FILE_PAGES); + __inc_zone_page_state(newpage, NR_FILE_PAGES); + if (!PageSwapCache(page) && PageSwapBacked(page)) { + __dec_zone_page_state(page, NR_SHMEM); + __inc_zone_page_state(newpage, NR_SHMEM); + } + spin_unlock_irq(&mapping->tree_lock); + + return MIGRATEPAGE_SUCCESS; +} + +/* * The expected number of remaining references is the same as that * of migrate_page_move_mapping(). */ @@ -550,6 +611,73 @@ void migrate_page_copy(struct page *newpage, struct page *page) end_page_writeback(newpage); } +/* + * Copy the page to its new location + */ +void replicate_page_copy(struct page *newpage, struct page *page) +{ + if (PageHuge(page) || PageTransHuge(page)) + copy_huge_page(newpage, page); + else + copy_highpage(newpage, page); + + if (PageError(page)) + SetPageError(newpage); + if (PageReferenced(page)) + SetPageReferenced(newpage); + if (PageUptodate(page)) + SetPageUptodate(newpage); + if (PageActive(page)) { + VM_BUG_ON_PAGE(PageUnevictable(page), page); + SetPageActive(newpage); + } else if (PageUnevictable(page)) + SetPageUnevictable(newpage); + if (PageChecked(page)) + SetPageChecked(newpage); + if (PageMappedToDisk(page)) + SetPageMappedToDisk(newpage); + + if (PageDirty(page)) { + clear_page_dirty_for_io(page); + /* + * Want to mark the page and the radix tree as dirty, and + * redo the accounting that clear_page_dirty_for_io undid, + * but we can't use set_page_dirty because that function + * is actually a signal that all of the page has become dirty. + * Whereas only part of our page may be dirty. + */ + if (PageSwapBacked(page)) + SetPageDirty(newpage); + else + __set_page_dirty_nobuffers(newpage); + } + + /* + * Copy NUMA information to the new page, to prevent over-eager + * future migrations of this same page. + */ +#ifdef CONFIG_NUMA_BALANCING + BUG(); +#endif + + if (PageMlocked(page)) { + unsigned long flags; + int nr_pages = hpage_nr_pages(page); + + local_irq_save(flags); + SetPageMlocked(newpage); + __mod_zone_page_state(page_zone(newpage), NR_MLOCK, nr_pages); + local_irq_restore(flags); + } + + /* + * If any waiters have accumulated on the new page then + * wake them up. + */ + if (PageWriteback(newpage)) + end_page_writeback(newpage); +} + /************************************************************ * Migration functions ***********************************************************/ @@ -578,6 +706,23 @@ int migrate_page(struct address_space *mapping, } EXPORT_SYMBOL(migrate_page); +int replicate_page(struct address_space *mapping, + struct page *newpage, struct page *page, + enum migrate_mode mode, int has_replica) +{ + int rc, extra_count = 0; + + BUG_ON(PageWriteback(page)); /* Writeback must be complete */ + + rc = replicate_page_move_mapping(mapping, newpage, page, NULL, mode, extra_count); + if (rc != MIGRATEPAGE_SUCCESS) + return rc; + + if (has_replica == 0) + replicate_page_copy(newpage, page); + return MIGRATEPAGE_SUCCESS; +} + #ifdef CONFIG_BLOCK /* * Migration function for pages with buffers. This function can only be used @@ -638,6 +783,8 @@ int buffer_migrate_page(struct address_space *mapping, EXPORT_SYMBOL(buffer_migrate_page); #endif +extern struct list_head shared_lib_pages; + /* * Writeback a page to clean the dirty state */ @@ -763,6 +910,64 @@ static int move_to_new_page(struct page *newpage, struct page *page, return rc; } +/* + * Copy a page to a newly allocated page + * The page is locked and all ptes have been successfully removed. + * + * The new page will have replaced the old page if this function + * is successful. + * + * Return value: + * < 0 - error code + * MIGRATEPAGE_SUCCESS - success + */ +static int copy_to_new_page(struct page *newpage, struct page *page, + int page_was_mapped, enum migrate_mode mode, + int has_replica) +{ + struct address_space *mapping; + int rc; + + /* + * Block others from accessing the page when we get around to + * establishing additional references. We are the only one + * holding a reference to the new page at this point. + */ + if (!trylock_page(newpage)) + BUG(); + + /* Prepare mapping for the new page.*/ + newpage->index = page->index; + newpage->mapping = page->mapping; + if (PageSwapBacked(page)) + SetPageSwapBacked(newpage); + + mapping = page_mapping(page); + if (!mapping) { + /* a shared library page must have a mapping. */ + BUG(); + } + else if (mapping->a_ops->migratepage) { + rc = replicate_page(mapping, newpage, page, mode, has_replica); + } + else { + rc = fallback_migrate_page(mapping, newpage, page, mode); + } + + if (rc != MIGRATEPAGE_SUCCESS) { + newpage->mapping = NULL; + } else { + if (page_was_mapped) { + remove_migration_ptes(page, newpage); + } + } + + unlock_page(newpage); + + return rc; +} + + static int __unmap_and_move(struct page *page, struct page *newpage, int force, enum migrate_mode mode) { @@ -901,6 +1106,106 @@ out: return rc; } +static int __unmap_and_copy(struct page *page, struct page *newpage, + int force, enum migrate_mode mode, int has_replica) +{ + int rc = -EAGAIN; + int ttu_ret = SWAP_AGAIN; + int page_was_mapped = 0; + struct anon_vma *anon_vma = NULL; + + if (!trylock_page(page)) { + if (!force || mode == MIGRATE_ASYNC) + goto out; + + /* + * It's not safe for direct compaction to call lock_page. + * For example, during page readahead pages are added locked + * to the LRU. Later, when the IO completes the pages are + * marked uptodate and unlocked. However, the queueing + * could be merging multiple pages for one bio (e.g. + * mpage_readpages). If an allocation happens for the + * second or third page, the process can end up locking + * the same page twice and deadlocking. Rather than + * trying to be clever about what pages can be locked, + * avoid the use of lock_page for direct compaction + * altogether. + */ + if (current->flags & PF_MEMALLOC) + goto out; + + lock_page(page); + } + + if (PageWriteback(page)) { + /* + * The code of shared library cannot be written. + */ + BUG(); + } + + if (PageAnon(page) && !PageKsm(page)) { + /* The shared library pages must be backed by a file. */ + BUG(); + } + + if (unlikely(isolated_balloon_page(page))) { + BUG(); + } + + /* + * Corner case handling: + * 1. When a new swap-cache page is read into, it is added to the LRU + * and treated as swapcache but it has no rmap yet. + * Calling try_to_unmap() against a page->mapping==NULL page will + * trigger a BUG. So handle it here. + * 2. An orphaned page (see truncate_complete_page) might have + * fs-private metadata. The page can be picked up due to memory + * offlining. Everywhere else except page reclaim, the page is + * invisible to the vm, so the page can not be migrated. So try to + * free the metadata, so the page can be freed. + */ + if (!page->mapping) { + VM_BUG_ON_PAGE(PageAnon(page), page); + if (page_has_private(page)) { + try_to_free_buffers(page); + goto out_unlock; + } + goto skip_unmap; + } + + /* Establish migration ptes or remove ptes */ + if (page_mapped(page)) { + struct rmap_walk_control rwc = { + .rmap_one = try_to_unmap_one_only, + .arg = (void *)(TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS), + }; + ttu_ret = rmap_walk(page, &rwc); + + page_was_mapped = 1; + } + +skip_unmap: + if (ttu_ret == SWAP_SUCCESS) { + rc = copy_to_new_page(newpage, page, page_was_mapped, mode, has_replica); + } else if (ttu_ret == SWAP_AGAIN) + printk(KERN_ERR "rmap_walk returned SWAP_AGAIN\n"); + else + printk(KERN_ERR "rmap_walk failed\n"); + + if (rc && page_was_mapped) + remove_migration_ptes(page, page); + + /* Drop an anon_vma reference if we took one */ + if (anon_vma) + put_anon_vma(anon_vma); + +out_unlock: + unlock_page(page); +out: + return rc; +} + /* * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move(). Work * around it. @@ -976,6 +1281,97 @@ out: } /* + * Obtain the lock on page, remove all ptes. + * 1) If r_pfn == INVALID_PFN, then copy the page to the newly allocated page in newpage. + * 2) If r_pfn != INVALID_PFN, then unmap and modify ptes. + */ +#include + +static ICE_noinline int unmap_and_copy(new_page_t get_new_page, + free_page_t put_new_page, + unsigned long private, struct page *page, + int force, enum migrate_mode mode) +{ + int rc = 0; + int *result = NULL; + struct page *newpage; + struct shared_lib_page *lib_page; + int master_exist_in_psl = 0, has_replica = 0, cpu = private/2; + + /* check if this page is in the PSL list */ + rcu_read_lock(); + list_for_each_entry(lib_page, &shared_lib_pages, list) + { + if (page_to_pfn(page) == lib_page->master_pfn) { + master_exist_in_psl = 1; + break; + } + } + rcu_read_unlock(); + + if (lib_page->r_page[cpu] == NULL) { + newpage = get_new_page(page, private, &result); + if (!newpage) + return -ENOMEM; + } else { + newpage = lib_page->r_page[cpu]; + has_replica = 1; + } + + if (page_count(page) == 1) { + /* page was freed from under us. So we are done. */ + goto out; + } + + if (unlikely(PageTransHuge(page))) + if (unlikely(split_huge_page(page))) + goto out; + + rc = __unmap_and_copy(page, newpage, force, mode, has_replica); + + if (has_replica == 0 && rc == MIGRATEPAGE_SUCCESS) { + lib_page->r_page[cpu] = newpage; + lib_page->r_pfn[cpu] = page_to_pfn(newpage); + } + +out: + if (rc != -EAGAIN) { + /* + * A page that has been migrated has all references + * removed and will be freed. A page that has not been + * migrated will have kepts its references and be + * restored. + */ + list_del(&page->lru); + dec_zone_page_state(page, NR_ISOLATED_ANON + + page_is_file_cache(page)); + putback_lru_page(page); + } + + /* + * If migration was not successful and there's a freeing callback, use + * it. Otherwise, putback_lru_page() will drop the reference grabbed + * during isolation. + */ + if (rc != MIGRATEPAGE_SUCCESS && put_new_page) { + ClearPageSwapBacked(newpage); + put_new_page(newpage, private); + } else if (unlikely(__is_movable_balloon_page(newpage))) { + /* drop our reference, page already in the balloon */ + put_page(newpage); + } else + putback_lru_page(newpage); + + if (result) { + if (rc) + *result = rc; + else + *result = page_to_nid(newpage); + } + return rc; +} + +/* * Counterpart of unmap_and_move_page() for hugepage migration. * * This function doesn't wait the completion of hugepage I/O @@ -1159,6 +1555,85 @@ out: return rc; } +/* + * replicate_pages - replicate the pages specified in a list + * + * @from: The list of pages to be migrated. + * @get_new_page: The function used to allocate free pages to be used + * if there is no replicated page. + * @put_new_page: The function used to free target pages if migration + * fails, or NULL if no special handling is necessary. + * @private: Private data to be passed on to get_new_page() + * @mode: The migration mode that specifies the constraints for + * page migration, if any. + * @reason: The reason for page migration. + * + * The function returns after 10 attempts or if no pages are movable any more + * because the list has become empty or no retryable pages exist any more. + * The caller should call putback_lru_pages() to return pages to the LRU + * or free list only if ret != 0. + * + * Returns the number of pages that were not migrated, or an error code. + */ +int replicate_pages(struct list_head *from, new_page_t get_new_page, + free_page_t put_new_page, unsigned long private, + enum migrate_mode mode, int reason) +{ + int retry = 1; + int nr_failed = 0; + int nr_succeeded = 0; + int pass = 0; + struct page *page; + struct page *page2; + int swapwrite = current->flags & PF_SWAPWRITE; + int rc; + + if (!swapwrite) + current->flags |= PF_SWAPWRITE; + + for(pass = 0; pass < 10 && retry; pass++) { + retry = 0; + + list_for_each_entry_safe(page, page2, from, lru) { + cond_resched(); + + rc = unmap_and_copy(get_new_page, put_new_page, private, page, pass > 2, mode); + + switch(rc) { + case -ENOMEM: + goto out; + case -EAGAIN: + retry++; + break; + case MIGRATEPAGE_SUCCESS: + nr_succeeded++; + break; + default: + /* + * Permanent failure (-EBUSY, -ENOSYS, etc.): + * unlike -EAGAIN case, the failed page is + * removed from migration page list and not + * retried in the next outer loop. + */ + nr_failed++; + break; + } + } + } + rc = nr_failed + retry; +out: + if (nr_succeeded) + count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded); + if (nr_failed) + count_vm_events(PGMIGRATE_FAIL, nr_failed); + trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason); + + if (!swapwrite) + current->flags &= ~PF_SWAPWRITE; + + return rc; +} + #ifdef CONFIG_NUMA /* * Move a list of individual pages diff --git mm/page_alloc.c mm/page_alloc.c index 950c002..574eb21 100644 --- mm/page_alloc.c +++ mm/page_alloc.c @@ -65,11 +65,32 @@ #include #include /* for is_realtime() */ +#include /* for coloring pages */ #include #include #include "internal.h" +// This Address Decoding is used in imx6-sabredsd platform +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 + +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 +#define MAX_COLOR_NODE 128 + +/* Decoding page color, 0~15 */ +static inline unsigned int page_color(struct page *page) +{ + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT); +} + +/* Decoding page bank number, 0~7 */ +static inline unsigned int page_bank(struct page *page) +{ + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT); +} + /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ static DEFINE_MUTEX(pcp_batch_high_lock); #define MIN_PERCPU_PAGELIST_FRACTION (8) @@ -464,7 +485,7 @@ static inline void set_page_guard(struct zone *zone, struct page *page, INIT_LIST_HEAD(&page->lru); set_page_private(page, order); /* Guard pages are not available for any usage */ - __mod_zone_freepage_state(zone, -(1 << order), migratetype); + __mod_zone_freepage_state(zone, -(1 << order), migratetype, bank_to_partition(page_bank(page))); } static inline void clear_page_guard(struct zone *zone, struct page *page, @@ -480,7 +501,7 @@ static inline void clear_page_guard(struct zone *zone, struct page *page, set_page_private(page, 0); if (!is_migrate_isolate(migratetype)) - __mod_zone_freepage_state(zone, (1 << order), migratetype); + __mod_zone_freepage_state(zone, (1 << order), migratetype, bank_to_partition(page_bank(page))); } #else struct page_ext_operations debug_guardpage_ops = { NULL, }; @@ -582,76 +603,151 @@ static inline void __free_one_page(struct page *page, unsigned long combined_idx; unsigned long uninitialized_var(buddy_idx); struct page *buddy; - int max_order = MAX_ORDER; + int max_order, parti_no; + + parti_no = bank_to_partition(page_bank(page)); + BUG_ON(parti_no < 0 || parti_no > NR_CPUS); - VM_BUG_ON(!zone_is_initialized(zone)); - VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); + if (parti_no < NR_CPUS) { + max_order = MAX_PARTITIONED_ORDER; - VM_BUG_ON(migratetype == -1); - if (is_migrate_isolate(migratetype)) { - /* - * We restrict max order of merging to prevent merge - * between freepages on isolate pageblock and normal - * pageblock. Without this, pageblock isolation - * could cause incorrect freepage accounting. - */ - max_order = min(MAX_ORDER, pageblock_order + 1); - } else { - __mod_zone_freepage_state(zone, 1 << order, migratetype); - } + VM_BUG_ON(!zone_is_initialized(zone)); + VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); - page_idx = pfn & ((1 << max_order) - 1); + VM_BUG_ON(migratetype == -1); + if (is_migrate_isolate(migratetype)) { + max_order = min(MAX_PARTITIONED_ORDER, pageblock_order + 1); + } else { + __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no); + } - VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); - VM_BUG_ON_PAGE(bad_range(zone, page), page); + page_idx = pfn & ((1 << max_order) - 1); - while (order < max_order - 1) { - buddy_idx = __find_buddy_index(page_idx, order); - buddy = page + (buddy_idx - page_idx); - if (!page_is_buddy(page, buddy, order)) - break; - /* - * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, - * merge with it and move up one order. - */ - if (page_is_guard(buddy)) { - clear_page_guard(zone, buddy, order, migratetype); + VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); + VM_BUG_ON_PAGE(bad_range(zone, page), page); + + while (order < max_order - 1) { + buddy_idx = __find_buddy_index(page_idx, order); + buddy = page + (buddy_idx - page_idx); + if (!page_is_buddy(page, buddy, order)) + break; + + if (page_is_guard(buddy)) { + clear_page_guard(zone, buddy, order, migratetype); + } else { + list_del(&buddy->lru); + zone->free_area_d[parti_no][order].nr_free--; + rmv_page_order(buddy); + } + combined_idx = buddy_idx & page_idx; + page = page + (combined_idx - page_idx); + page_idx = combined_idx; + order++; + } + set_page_order(page, order); + + if ((order < MAX_PARTITIONED_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { + struct page *higher_page, *higher_buddy; + combined_idx = buddy_idx & page_idx; + higher_page = page + (combined_idx - page_idx); + buddy_idx = __find_buddy_index(combined_idx, order + 1); + higher_buddy = higher_page + (buddy_idx - combined_idx); + if (page_is_buddy(higher_page, higher_buddy, order + 1)) { + list_add_tail(&page->lru, + &zone->free_area_d[parti_no][order].free_list[migratetype]); + zone->free_area_d[parti_no][order].nr_free++; + return; + } + } + + if (order >= MAX_PARTITIONED_ORDER) { + int n_idx = 0; + struct page *lower_page; + for (n_idx = 0 ; n_idx < (1 << (order - MAX_PARTITIONED_ORDER + 1)); n_idx++) { + lower_page = page + (n_idx << (MAX_PARTITIONED_ORDER - 1)); + if (lower_page->flags & PAGE_FLAGS_CHECK_AT_PREP) + lower_page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; + set_page_order(lower_page, MAX_PARTITIONED_ORDER-1); + list_add(&lower_page->lru, &zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].free_list[migratetype]); + zone->free_area_d[parti_no][MAX_PARTITIONED_ORDER-1].nr_free++; + } } else { - list_del(&buddy->lru); - zone->free_area[order].nr_free--; - rmv_page_order(buddy); + list_add(&page->lru, &zone->free_area_d[parti_no][order].free_list[migratetype]); + zone->free_area_d[parti_no][order].nr_free++; } - combined_idx = buddy_idx & page_idx; - page = page + (combined_idx - page_idx); - page_idx = combined_idx; - order++; } - set_page_order(page, order); + else { + max_order = MAX_ORDER; - /* - * If this is not the largest possible page, check if the buddy - * of the next-highest order is free. If it is, it's possible - * that pages are being freed that will coalesce soon. In case, - * that is happening, add the free page to the tail of the list - * so it's less likely to be used soon and more likely to be merged - * as a higher order page - */ - if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { - struct page *higher_page, *higher_buddy; - combined_idx = buddy_idx & page_idx; - higher_page = page + (combined_idx - page_idx); - buddy_idx = __find_buddy_index(combined_idx, order + 1); - higher_buddy = higher_page + (buddy_idx - combined_idx); - if (page_is_buddy(higher_page, higher_buddy, order + 1)) { - list_add_tail(&page->lru, - &zone->free_area[order].free_list[migratetype]); - goto out; + VM_BUG_ON(!zone_is_initialized(zone)); + VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); + + VM_BUG_ON(migratetype == -1); + if (is_migrate_isolate(migratetype)) { + /* + * We restrict max order of merging to prevent merge + * between freepages on isolate pageblock and normal + * pageblock. Without this, pageblock isolation + * could cause incorrect freepage accounting. + */ + max_order = min(MAX_ORDER, pageblock_order + 1); + } else { + __mod_zone_freepage_state(zone, 1 << order, migratetype, parti_no); + } + + page_idx = pfn & ((1 << max_order) - 1); + + VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); + VM_BUG_ON_PAGE(bad_range(zone, page), page); + + while (order < max_order - 1) { + buddy_idx = __find_buddy_index(page_idx, order); + buddy = page + (buddy_idx - page_idx); + if (!page_is_buddy(page, buddy, order)) + break; + /* + * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, + * merge with it and move up one order. + */ + if (page_is_guard(buddy)) { + clear_page_guard(zone, buddy, order, migratetype); + } else { + list_del(&buddy->lru); + zone->free_area[order].nr_free--; + rmv_page_order(buddy); + } + combined_idx = buddy_idx & page_idx; + page = page + (combined_idx - page_idx); + page_idx = combined_idx; + order++; + } + set_page_order(page, order); + + /* + * If this is not the largest possible page, check if the buddy + * of the next-highest order is free. If it is, it's possible + * that pages are being freed that will coalesce soon. In case, + * that is happening, add the free page to the tail of the list + * so it's less likely to be used soon and more likely to be merged + * as a higher order page + */ + if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { + struct page *higher_page, *higher_buddy; + combined_idx = buddy_idx & page_idx; + higher_page = page + (combined_idx - page_idx); + buddy_idx = __find_buddy_index(combined_idx, order + 1); + higher_buddy = higher_page + (buddy_idx - combined_idx); + if (page_is_buddy(higher_page, higher_buddy, order + 1)) { + list_add_tail(&page->lru, + &zone->free_area[order].free_list[migratetype]); + goto out; + } } - } - list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); + list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); out: - zone->free_area[order].nr_free++; + zone->free_area[order].nr_free++; + } } static inline int free_pages_check(struct page *page) @@ -829,7 +925,10 @@ static void __free_pages_ok(struct page *page, unsigned int order) migratetype = get_pfnblock_migratetype(page, pfn); local_irq_save(flags); - __count_vm_events(PGFREE, 1 << order); + if (bank_to_partition(page_bank(page)) == NR_CPUS) + __count_vm_events(PGFREE, 1 << order); + else if (bank_to_partition(page_bank(page)) < NR_CPUS) + __count_vm_events(PGFREE_HC, 1 << order); set_freepage_migratetype(page, migratetype); free_one_page(page_zone(page), page, pfn, order, migratetype); local_irq_restore(flags); @@ -930,6 +1029,43 @@ static inline void expand(struct zone *zone, struct page *page, } } +static inline int expand_middle(struct zone *zone, struct page *page, + int offset, int low, int high, struct free_area *area, + int migratetype) +{ + unsigned long size = 1 << high; + + while ((size>>1) > offset) { + area--; + high--; + size >>= 1; + VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); + + list_add(&page[size].lru, &area->free_list[migratetype]); + area->nr_free++; + set_page_order(&page[size], high); + } + area--; + high--; + size >>= 1; + VM_BUG_ON_PAGE(bad_range(zone, page), page); + list_add(&page[0].lru, &area->free_list[migratetype]); + area->nr_free++; + set_page_order(&page[0], high); + + if (offset == size) + return high; + + area--; + high--; + VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); + list_add(&page[size].lru, &area->free_list[migratetype]); + area->nr_free++; + set_page_order(&page[size], high); + + return high; +} + /* * This page is about to be returned from the page allocator */ @@ -996,34 +1132,176 @@ static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, return 0; } +/* Kernel page coloring */ + +/* build colored page list */ +#if 0 +static void build_colored_pages(struct zone *zone, struct page *page, int order) +{ + int i, color, bank; + + list_del(&page->lru); + zone->free_area[order].nr_free--; + + /* insert pages to zone->color_list[] */ + for (i = 0; i < (1<color_list[node]); + bitmap_set(zone->color_map, node, 1); + zone->free_area[0].nr_free++; + rmv_page_order(&page[i]); + } +} + +int color_seq_index[9] = { + 0, /* Core 0, and Level A*/ + 0, /* Core 0, and Level B*/ + 0, /* Core 1, and Level A*/ + 0, /* Core 1, and Level B*/ + 0, /* Core 2, and Level A*/ + 0, /* Core 2, and Level B*/ + 0, /* Core 3, and Level A*/ + 0, /* Core 3, and Level B*/ + 0, /* Level C */ +}; + +/* return a colored page */ +static inline struct page *get_colored_page(struct zone *zone, unsigned long req_color_map[BITS_TO_LONGS(MAX_COLOR_NODE)], int order, int partition) +{ + struct page *page; + unsigned int color, bank, index; + int i; + DECLARE_BITMAP(candidate_bit, MAX_COLOR_NODE); + + /* if req_color_map does not exist in zone, return NULL */ + if (!bitmap_intersects(zone->color_map, req_color_map, MAX_COLOR_NODE)) + return NULL; + + bitmap_and(candidate_bit, zone->color_map, req_color_map, MAX_COLOR_NODE); + index = color_seq_index[partition]; + + for_each_set_bit(i, candidate_bit, MAX_COLOR_NODE) { + if (index-- <= 0) + break; + } + + BUG_ON(i >= MAX_COLOR_NODE); + BUG_ON(list_empty(&zone->color_list[i])); + + page = list_entry(zone->color_list[i].next, struct page, lru); + + list_del(&page->lru); + + if (list_empty(&zone->color_list[i])) + bitmap_clear(zone->color_map, i, 1); + + zone->free_area[0].nr_free--; + color = page_color(page); + bank = page_bank(page); + + return page; +} +#endif + /* * Go through the free lists for the given migratetype and remove * the smallest available page from the freelists */ static inline struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, - int migratetype) + int migratetype, int color_req) { unsigned int current_order; struct free_area *area; struct page *page; + int cpu = raw_smp_processor_id(); + + if (color_req != 0) { + int found = 0; + int area_index; + unsigned long s_pfn = zone->zone_start_pfn; + unsigned long e_pfn = zone_end_pfn(zone); + + area_index = get_area_index(cpu); + + /* Find a page of the appropriate size in the preferred list */ + for (current_order = order; current_order < MAX_PARTITIONED_ORDER; ++current_order) { + int offset = 0; + area = &(zone->free_area_d[area_index][current_order]); + if (list_empty(&area->free_list[migratetype])) { + continue; + } + + if (order >= MAX_CONTIG_ORDER) { + page = list_entry(area->free_list[migratetype].next, struct page, lru); + found = 1; + } else { + list_for_each_entry(page, &area->free_list[migratetype], lru) { + if (current_order < MAX_CONTIG_ORDER) { + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { + offset = 0; + found = 1; + break; + } + } else { + int size = 1 << current_order; + for (offset = 0; offset < size; offset++) { + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) { + found = 1; + break; + } + } + if (found) + break; + } + } + } + + + if (!found) + continue; + + list_del(&page->lru); + rmv_page_order(page); + area->nr_free--; + + if (offset == 0) { + expand(zone, page, order, current_order, area, migratetype); + } else { + int frac = expand_middle(zone, page, offset, order, current_order, area, migratetype); + page = &page[offset]; + area = &(zone->free_area_d[area_index][frac]); + expand(zone, page, order, frac, area, migratetype); + } + + set_freepage_migratetype(page, migratetype); - /* Find a page of the appropriate size in the preferred list */ - for (current_order = order; current_order < MAX_ORDER; ++current_order) { - area = &(zone->free_area[current_order]); - if (list_empty(&area->free_list[migratetype])) - continue; + return page; + } + } else { + /* Buddy allocator */ + /* Find a page of the appropriate size in the preferred list */ + for (current_order = order; current_order < MAX_ORDER; ++current_order) { + area = &(zone->free_area[current_order]); + if (list_empty(&area->free_list[migratetype])) + continue; - page = list_entry(area->free_list[migratetype].next, - struct page, lru); - list_del(&page->lru); - rmv_page_order(page); - area->nr_free--; - expand(zone, page, order, current_order, area, migratetype); - set_freepage_migratetype(page, migratetype); - return page; + page = list_entry(area->free_list[migratetype].next, + struct page, lru); + list_del(&page->lru); + rmv_page_order(page); + area->nr_free--; + expand(zone, page, order, current_order, area, migratetype); + set_freepage_migratetype(page, migratetype); + return page; + } } - + return NULL; } @@ -1049,7 +1327,7 @@ static int fallbacks[MIGRATE_TYPES][4] = { static struct page *__rmqueue_cma_fallback(struct zone *zone, unsigned int order) { - return __rmqueue_smallest(zone, order, MIGRATE_CMA); + return __rmqueue_smallest(zone, order, MIGRATE_CMA, 0); } #else static inline struct page *__rmqueue_cma_fallback(struct zone *zone, @@ -1236,7 +1514,7 @@ int find_suitable_fallback(struct free_area *area, unsigned int order, /* Remove an element from the buddy allocator from the fallback list */ static inline struct page * -__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) +__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype, int color_req) { struct free_area *area; unsigned int current_order; @@ -1244,44 +1522,128 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) int fallback_mt; bool can_steal; - /* Find the largest possible block of pages in the other list */ - for (current_order = MAX_ORDER-1; - current_order >= order && current_order <= MAX_ORDER-1; - --current_order) { - area = &(zone->free_area[current_order]); - fallback_mt = find_suitable_fallback(area, current_order, - start_migratetype, false, &can_steal); - if (fallback_mt == -1) - continue; + if (color_req != 0) { + int cpu = raw_smp_processor_id(); + int found = 0; + int area_index; + unsigned long s_pfn = zone->zone_start_pfn; + unsigned long e_pfn = zone_end_pfn(zone); + + area_index = get_area_index(cpu); + + /* Find the largest possible block of pages in the other list */ + for (current_order = MAX_PARTITIONED_ORDER-1; + current_order >= order && current_order <= MAX_PARTITIONED_ORDER-1; + --current_order) { + int offset = 0; + area = &(zone->free_area_d[area_index][current_order]); + fallback_mt = find_suitable_fallback(area, current_order, + start_migratetype, false, &can_steal); + if (fallback_mt == -1) + continue; + + if (order >= MAX_CONTIG_ORDER) { + page = list_entry(area->free_list[fallback_mt].next, struct page, lru); + found = 1; + } else { + list_for_each_entry(page, &area->free_list[fallback_mt], lru) { + if (current_order < MAX_CONTIG_ORDER) { + if (is_in_llc_partition(page, cpu) && (page_to_pfn(page) >= s_pfn && page_to_pfn(page) < e_pfn)) { + found = 1; + offset = 0; + break; + } + } else { + int size = 1 << current_order; + for (offset = 0; offset < size; offset++) { + if (is_in_llc_partition(&page[offset], cpu) && (page_to_pfn(&page[offset]) >= s_pfn && page_to_pfn(&page[offset]) < e_pfn)) { + found = 1; + break; + } + } + if (found) + break; + } + } + } - page = list_entry(area->free_list[fallback_mt].next, - struct page, lru); - if (can_steal) - steal_suitable_fallback(zone, page, start_migratetype); + if (!found) + continue; + + if (can_steal) + steal_suitable_fallback(zone, page, start_migratetype); - /* Remove the page from the freelists */ - area->nr_free--; - list_del(&page->lru); - rmv_page_order(page); + /* Remove the page from the freelists */ + area->nr_free--; + list_del(&page->lru); + rmv_page_order(page); + + if (offset == 0) + expand(zone, page, order, current_order, area, start_migratetype); + else { + int frac = expand_middle(zone, page, offset, order, current_order, area, start_migratetype); + page = &page[offset]; + area = &(zone->free_area_d[area_index][frac]); + expand(zone, page, order, frac, area, start_migratetype); + + } + - expand(zone, page, order, current_order, area, - start_migratetype); - /* - * The freepage_migratetype may differ from pageblock's - * migratetype depending on the decisions in - * try_to_steal_freepages(). This is OK as long as it - * does not differ for MIGRATE_CMA pageblocks. For CMA - * we need to make sure unallocated pages flushed from - * pcp lists are returned to the correct freelist. - */ - set_freepage_migratetype(page, start_migratetype); + /* + * The freepage_migratetype may differ from pageblock's + * migratetype depending on the decisions in + * try_to_steal_freepages(). This is OK as long as it + * does not differ for MIGRATE_CMA pageblocks. For CMA + * we need to make sure unallocated pages flushed from + * pcp lists are returned to the correct freelist. + */ + set_freepage_migratetype(page, start_migratetype); - trace_mm_page_alloc_extfrag(page, order, current_order, - start_migratetype, fallback_mt); + trace_mm_page_alloc_extfrag(page, order, current_order, + start_migratetype, fallback_mt); - return page; - } + return page; + } + } else { + /* Find the largest possible block of pages in the other list */ + for (current_order = MAX_ORDER-1; + current_order >= order && current_order <= MAX_ORDER-1; + --current_order) { + area = &(zone->free_area[current_order]); + fallback_mt = find_suitable_fallback(area, current_order, + start_migratetype, false, &can_steal); + if (fallback_mt == -1) + continue; + + page = list_entry(area->free_list[fallback_mt].next, + struct page, lru); + if (can_steal) + steal_suitable_fallback(zone, page, start_migratetype); + /* Remove the page from the freelists */ + area->nr_free--; + list_del(&page->lru); + rmv_page_order(page); + + expand(zone, page, order, current_order, area, + start_migratetype); + /* + * The freepage_migratetype may differ from pageblock's + * migratetype depending on the decisions in + * try_to_steal_freepages(). This is OK as long as it + * does not differ for MIGRATE_CMA pageblocks. For CMA + * we need to make sure unallocated pages flushed from + * pcp lists are returned to the correct freelist. + */ + set_freepage_migratetype(page, start_migratetype); + + trace_mm_page_alloc_extfrag(page, order, current_order, + start_migratetype, fallback_mt); + + return page; + } + } + return NULL; } @@ -1290,26 +1652,27 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) * Call me with the zone->lock already held. */ static struct page *__rmqueue(struct zone *zone, unsigned int order, - int migratetype) + int migratetype, int color_req) { struct page *page; retry_reserve: - page = __rmqueue_smallest(zone, order, migratetype); + page = __rmqueue_smallest(zone, order, migratetype, color_req); if (unlikely(!page) && migratetype != MIGRATE_RESERVE) { if (migratetype == MIGRATE_MOVABLE) page = __rmqueue_cma_fallback(zone, order); - if (!page) - page = __rmqueue_fallback(zone, order, migratetype); + if (!page) { + page = __rmqueue_fallback(zone, order, migratetype, color_req); + } /* * Use MIGRATE_RESERVE rather than fail an allocation. goto * is used because __rmqueue_smallest is an inline function * and we want just one call site */ - if (!page) { + if (!page && !color_req) { migratetype = MIGRATE_RESERVE; goto retry_reserve; } @@ -1332,7 +1695,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, spin_lock(&zone->lock); for (i = 0; i < count; ++i) { - struct page *page = __rmqueue(zone, order, migratetype); + struct page *page = __rmqueue(zone, order, migratetype, 0); if (unlikely(page == NULL)) break; @@ -1549,8 +1912,16 @@ void free_hot_cold_page(struct page *page, bool cold) migratetype = get_pfnblock_migratetype(page, pfn); set_freepage_migratetype(page, migratetype); local_irq_save(flags); - __count_vm_event(PGFREE); - + + + if (bank_to_partition(page_bank(page)) == NR_CPUS) + __count_vm_event(PGFREE); + else if (bank_to_partition(page_bank(page)) < NR_CPUS) { + __count_vm_event(PGFREE_HC); + free_one_page(zone, page, pfn, 0, migratetype); + goto out; + } + /* * We only track unmovable, reclaimable and movable on pcp lists. * Free ISOLATE pages back to the allocator because they are being @@ -1565,19 +1936,18 @@ void free_hot_cold_page(struct page *page, bool cold) } migratetype = MIGRATE_MOVABLE; } - - pcp = &this_cpu_ptr(zone->pageset)->pcp; - if (!cold) - list_add(&page->lru, &pcp->lists[migratetype]); - else - list_add_tail(&page->lru, &pcp->lists[migratetype]); - pcp->count++; - if (pcp->count >= pcp->high) { - unsigned long batch = READ_ONCE(pcp->batch); - free_pcppages_bulk(zone, batch, pcp); - pcp->count -= batch; - } - + + pcp = &this_cpu_ptr(zone->pageset)->pcp; + if (!cold) + list_add(&page->lru, &pcp->lists[migratetype]); + else + list_add_tail(&page->lru, &pcp->lists[migratetype]); + pcp->count++; + if (pcp->count >= pcp->high) { + unsigned long batch = READ_ONCE(pcp->batch); + free_pcppages_bulk(zone, batch, pcp); + pcp->count -= batch; + } out: local_irq_restore(flags); } @@ -1590,8 +1960,12 @@ void free_hot_cold_page_list(struct list_head *list, bool cold) struct page *page, *next; list_for_each_entry_safe(page, next, list, lru) { + int parti_no = bank_to_partition(page_bank(page)); trace_mm_page_free_batched(page, cold); - free_hot_cold_page(page, cold); + if (parti_no == NR_CPUS) + free_hot_cold_page(page, cold); + else + __free_pages_ok(page, 0); } } @@ -1644,7 +2018,7 @@ int __isolate_free_page(struct page *page, unsigned int order) if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) return 0; - __mod_zone_freepage_state(zone, -(1UL << order), mt); + __mod_zone_freepage_state(zone, -(1UL << order), mt, bank_to_partition(page_bank(page))); } /* Remove page from free list */ @@ -1705,8 +2079,9 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, unsigned long flags; struct page *page; bool cold = ((gfp_flags & __GFP_COLD) != 0); + int colored_req = ((gfp_flags & __GFP_COLOR) != 0); - if (likely(order == 0)) { + if (likely(order == 0) && !colored_req) { struct per_cpu_pages *pcp; struct list_head *list; @@ -1743,12 +2118,12 @@ struct page *buffered_rmqueue(struct zone *preferred_zone, WARN_ON_ONCE(order > 1); } spin_lock_irqsave(&zone->lock, flags); - page = __rmqueue(zone, order, migratetype); + page = __rmqueue(zone, order, migratetype, colored_req); spin_unlock(&zone->lock); if (!page) goto failed; __mod_zone_freepage_state(zone, -(1 << order), - get_freepage_migratetype(page)); + get_freepage_migratetype(page), bank_to_partition(page_bank(page))); } __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); @@ -2950,8 +3325,9 @@ EXPORT_SYMBOL(get_zeroed_page); void __free_pages(struct page *page, unsigned int order) { + int parti_no = bank_to_partition(page_bank(page)); if (put_page_testzero(page)) { - if (order == 0) + if (order == 0 && parti_no == NR_CPUS) free_hot_cold_page(page, false); else __free_pages_ok(page, order); @@ -4230,10 +4606,22 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, static void __meminit zone_init_free_lists(struct zone *zone) { unsigned int order, t; + int cpu; + for_each_migratetype_order(order, t) { INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); zone->free_area[order].nr_free = 0; } + + /* Initialize per-partition free_area data structures */ + for (cpu = 0; cpu < NR_CPUS; cpu++) { + for (order = 0; order < MAX_PARTITIONED_ORDER; order++) { + for (t = 0; t < MIGRATE_TYPES; t++) { + INIT_LIST_HEAD(&zone->free_area_d[cpu][order].free_list[t]); + zone->free_area_d[cpu][order].nr_free = 0; + } + } + } } #ifndef __HAVE_ARCH_MEMMAP_INIT diff --git mm/page_isolation.c mm/page_isolation.c index 303c908..ed05910 100644 --- mm/page_isolation.c +++ mm/page_isolation.c @@ -58,12 +58,13 @@ out: if (!ret) { unsigned long nr_pages; int migratetype = get_pageblock_migratetype(page); + int partno = bank_to_partition(page_bank(page)); set_pageblock_migratetype(page, MIGRATE_ISOLATE); zone->nr_isolate_pageblock++; nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE); - __mod_zone_freepage_state(zone, -nr_pages, migratetype); + __mod_zone_freepage_state(zone, -nr_pages, migratetype, partno); } spin_unlock_irqrestore(&zone->lock, flags); @@ -117,8 +118,9 @@ void unset_migratetype_isolate(struct page *page, unsigned migratetype) * pageblock scanning for freepage moving. */ if (!isolated_page) { + int partno = bank_to_partition(page_bank(page)); nr_pages = move_freepages_block(zone, page, migratetype); - __mod_zone_freepage_state(zone, nr_pages, migratetype); + __mod_zone_freepage_state(zone, nr_pages, migratetype, partno); } set_pageblock_migratetype(page, migratetype); zone->nr_isolate_pageblock--; diff --git mm/rmap.c mm/rmap.c index 24dd3f9..0613210 100644 --- mm/rmap.c +++ mm/rmap.c @@ -1317,6 +1317,170 @@ out_mlock: return ret; } +/* + * @arg: enum ttu_flags will be passed to this argument + */ +static int try_to_unmap_one_entry(struct page *page, struct vm_area_struct *vma, + unsigned long address, void *arg) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t *pte; + pte_t pteval; + spinlock_t *ptl; + int ret = SWAP_AGAIN; + enum ttu_flags flags = (enum ttu_flags)arg; + + pte = page_check_address(page, mm, address, &ptl, 0); + if (!pte) + goto out; + + /* + * If the page is mlock()d, we cannot swap it out. + * If it's recently referenced (perhaps page_referenced + * skipped over this mm) then we should reactivate it. + */ + if (!(flags & TTU_IGNORE_MLOCK)) { + if (vma->vm_flags & VM_LOCKED) + goto out_mlock; + + if (flags & TTU_MUNLOCK) + goto out_unmap; + } + if (!(flags & TTU_IGNORE_ACCESS)) { + if (ptep_clear_flush_young_notify(vma, address, pte)) { + ret = SWAP_FAIL; + goto out_unmap; + } + } + + /* Nuke the page table entry. */ + flush_cache_page(vma, address, page_to_pfn(page)); + pteval = ptep_clear_flush(vma, address, pte); + + /* Move the dirty bit to the physical page now the pte is gone. */ + if (pte_dirty(pteval)) + set_page_dirty(page); + + /* Update high watermark before we lower rss */ + update_hiwater_rss(mm); + + if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { + if (!PageHuge(page)) { + if (PageAnon(page)) + dec_mm_counter(mm, MM_ANONPAGES); + else + dec_mm_counter(mm, MM_FILEPAGES); + } + set_pte_at(mm, address, pte, + swp_entry_to_pte(make_hwpoison_entry(page))); + } else if (pte_unused(pteval)) { + /* + * The guest indicated that the page content is of no + * interest anymore. Simply discard the pte, vmscan + * will take care of the rest. + */ + if (PageAnon(page)) + dec_mm_counter(mm, MM_ANONPAGES); + else + dec_mm_counter(mm, MM_FILEPAGES); + } else if (PageAnon(page)) { + swp_entry_t entry = { .val = page_private(page) }; + pte_t swp_pte; + + if (PageSwapCache(page)) { + /* + * Store the swap location in the pte. + * See handle_pte_fault() ... + */ + if (swap_duplicate(entry) < 0) { + set_pte_at(mm, address, pte, pteval); + ret = SWAP_FAIL; + goto out_unmap; + } + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + } else if (IS_ENABLED(CONFIG_MIGRATION)) { + /* + * Store the pfn of the page in a special migration + * pte. do_swap_page() will wait until the migration + * pte is removed and then restart fault handling. + */ + BUG_ON(!(flags & TTU_MIGRATION)); + entry = make_migration_entry(page, pte_write(pteval)); + } + swp_pte = swp_entry_to_pte(entry); + if (pte_soft_dirty(pteval)) + swp_pte = pte_swp_mksoft_dirty(swp_pte); + set_pte_at(mm, address, pte, swp_pte); + } else if (IS_ENABLED(CONFIG_MIGRATION) && + (flags & TTU_MIGRATION)) { + /* Establish migration entry for a file page */ + swp_entry_t entry; + entry = make_migration_entry(page, pte_write(pteval)); + set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); + } else + dec_mm_counter(mm, MM_FILEPAGES); + + page_remove_rmap(page); + page_cache_release(page); + +out_unmap: + pte_unmap_unlock(pte, ptl); + if (ret != SWAP_FAIL && !(flags & TTU_MUNLOCK)) { + mmu_notifier_invalidate_page(mm, address); + ret = SWAP_SUCCESS; + } +out: + return ret; + +out_mlock: + pte_unmap_unlock(pte, ptl); + + + /* + * We need mmap_sem locking, Otherwise VM_LOCKED check makes + * unstable result and race. Plus, We can't wait here because + * we now hold anon_vma->rwsem or mapping->i_mmap_rwsem. + * if trylock failed, the page remain in evictable lru and later + * vmscan could retry to move the page to unevictable lru if the + * page is actually mlocked. + */ + if (down_read_trylock(&vma->vm_mm->mmap_sem)) { + if (vma->vm_flags & VM_LOCKED) { + mlock_vma_page(page); + ret = SWAP_MLOCK; + } + up_read(&vma->vm_mm->mmap_sem); + } + return ret; +} + +int try_to_unmap_one_only(struct page *page, struct vm_area_struct *vma, + unsigned long address, void *arg) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_struct *current_mm; + + rcu_read_lock(); + get_task_struct(current); + rcu_read_unlock(); + current_mm = get_task_mm(current); + put_task_struct(current); + if (!current_mm) + BUG(); + + if (mm == current_mm) { + return try_to_unmap_one_entry(page, vma, address, arg); + } + return SWAP_AGAIN; +} + bool is_vma_temporary_stack(struct vm_area_struct *vma) { int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); diff --git mm/slab.h mm/slab.h index 4c3ac12..48be14c 100644 --- mm/slab.h +++ mm/slab.h @@ -114,7 +114,7 @@ static inline unsigned long kmem_cache_flags(unsigned long object_size, /* Legal flag mask for kmem_cache_create(), for various configurations */ #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \ - SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS ) + SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS | SLAB_NO_MERGE) #if defined(CONFIG_DEBUG_SLAB) #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) diff --git mm/slab_common.c mm/slab_common.c index 999bb34..350acdb 100644 --- mm/slab_common.c +++ mm/slab_common.c @@ -30,12 +30,18 @@ LIST_HEAD(slab_caches); DEFINE_MUTEX(slab_mutex); struct kmem_cache *kmem_cache; +#ifdef CONFIG_UVC_ALLOC_LEVEL_AB +#define KMEM_FLAG (GFP_COLOR) +#else +#define KMEM_FLAG (0) +#endif + /* * Set of flags that will prevent slab merging */ #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \ - SLAB_FAILSLAB) + SLAB_FAILSLAB | SLAB_NO_MERGE) #define SLAB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \ SLAB_CACHE_DMA | SLAB_NOTRACK) @@ -303,7 +309,7 @@ do_kmem_cache_create(const char *name, size_t object_size, size_t size, int err; err = -ENOMEM; - s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); + s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL|KMEM_FLAG); if (!s) goto out; @@ -703,7 +709,9 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size, panic("Out of memory when creating slab %s\n", name); create_boot_cache(s, name, size, flags); + list_add(&s->list, &slab_caches); + s->refcount = 1; return s; } @@ -711,6 +719,11 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size, struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; EXPORT_SYMBOL(kmalloc_caches); +/* for per-cpu kmalloc objects */ +struct kmem_cache *hc_kmalloc_caches[NR_CPUS][KMALLOC_SHIFT_HIGH + 1]; +//struct kmem_cache *hc_kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; +EXPORT_SYMBOL(hc_kmalloc_caches); + #ifdef CONFIG_ZONE_DMA struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; EXPORT_SYMBOL(kmalloc_dma_caches); @@ -780,7 +793,13 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) return kmalloc_dma_caches[index]; #endif - return kmalloc_caches[index]; + + if (flags & GFP_COLOR) { + int cpu = raw_smp_processor_id(); + return hc_kmalloc_caches[cpu][index]; + } + else + return kmalloc_caches[index]; } /* @@ -790,7 +809,7 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) */ void __init create_kmalloc_caches(unsigned long flags) { - int i; + int i, cpu = 0; /* * Patch up the size_index table if we have strange large alignment @@ -844,11 +863,36 @@ void __init create_kmalloc_caches(unsigned long flags) * These have to be created immediately after the * earlier power of two caches */ - if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6) + if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6) { kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags); + } - if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7) + if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7) { kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags); + } + } + +/* per-cpu kmalloc caches */ + for (cpu = 0; cpu < NR_CPUS; cpu++) { + for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { + char *n; + n = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(i)); + + hc_kmalloc_caches[cpu][i] = create_kmalloc_cache(n, 1 << i, SLAB_NO_MERGE|flags); + hc_kmalloc_caches[cpu][i]->cpu_id = cpu; + + if (KMALLOC_MIN_SIZE <= 32 && !hc_kmalloc_caches[cpu][1] && i == 6) { + char *nm; + nm = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(1)); + hc_kmalloc_caches[cpu][1] = create_kmalloc_cache(nm, 96, SLAB_NO_MERGE|flags); + } + + if (KMALLOC_MIN_SIZE <= 64 && !hc_kmalloc_caches[cpu][2] && i == 7) { + char *nm; + nm = kasprintf(GFP_NOWAIT, "cpu%01d-kmalloc-%d", cpu, kmalloc_size(2)); + hc_kmalloc_caches[cpu][2] = create_kmalloc_cache(nm, 192, SLAB_NO_MERGE|flags); + } + } } /* Kmalloc array is now usable */ diff --git mm/slub.c mm/slub.c index 54c0876..dc2e9eb 100644 --- mm/slub.c +++ mm/slub.c @@ -115,6 +115,26 @@ * the fast path and disables lockless freelists. */ +// This Address Decoding is used in imx6-sabredsd platform +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 + +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 +#define MAX_COLOR_NODE 128 + +/* Decoding page color, 0~15 */ +static inline unsigned int page_color(struct page *page) +{ + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT); +} + +/* Decoding page bank number, 0~7 */ +static inline unsigned int page_bank(struct page *page) +{ + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT); +} + static inline int kmem_cache_debug(struct kmem_cache *s) { #ifdef CONFIG_SLUB_DEBUG @@ -1320,8 +1340,9 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s, if (memcg_charge_slab(s, flags, order)) return NULL; - if (node == NUMA_NO_NODE) + if (node == NUMA_NO_NODE) { page = alloc_pages(flags, order); + } else page = alloc_pages_exact_node(node, flags, order); @@ -1336,7 +1357,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) struct page *page; struct kmem_cache_order_objects oo = s->oo; gfp_t alloc_gfp; - + flags &= gfp_allowed_mask; if (flags & __GFP_WAIT) @@ -1419,7 +1440,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) } page = allocate_slab(s, - flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); + flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK | GFP_COLOR), node); if (!page) goto out; @@ -2223,6 +2244,7 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, return freelist; page = new_slab(s, flags, node); + if (page) { c = raw_cpu_ptr(s->cpu_slab); if (c->page) @@ -2770,7 +2792,8 @@ EXPORT_SYMBOL(kmem_cache_free); * take the list_lock. */ static int slub_min_order; -static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; +//static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; +static int slub_max_order = 2; static int slub_min_objects; /* @@ -3311,7 +3334,7 @@ void *__kmalloc(size_t size, gfp_t flags) return kmalloc_large(size, flags); s = kmalloc_slab(size, flags); - + if (unlikely(ZERO_OR_NULL_PTR(s))) return s; @@ -5139,6 +5162,10 @@ static char *create_unique_id(struct kmem_cache *s) * are matched during merging to guarantee that the id is * unique. */ + if (s->flags & SLAB_NO_MERGE) { + *p++ = 'n'; + p += sprintf(p, "%01d", s->cpu_id); + } if (s->flags & SLAB_CACHE_DMA) *p++ = 'd'; if (s->flags & SLAB_RECLAIM_ACCOUNT) diff --git mm/vmalloc.c mm/vmalloc.c index 2faaa29..5b1c46b 100644 --- mm/vmalloc.c +++ mm/vmalloc.c @@ -1746,6 +1746,22 @@ void *vmalloc(unsigned long size) EXPORT_SYMBOL(vmalloc); /** + * vmalloc_color - allocate virtually contiguous memory from a dedicated free_area + * @size: allocation size + * Allocate enough pages to cover @size from the page level + * allocator and map them into contiguous kernel virtual space. + * + * For tight control over page level allocator and protection flags + * use __vmalloc() instead. + */ +void *vmalloc_color(unsigned long size) +{ + return __vmalloc_node_flags(size, NUMA_NO_NODE, + GFP_KERNEL | __GFP_HIGHMEM | GFP_COLOR); +} +EXPORT_SYMBOL(vmalloc_color); + +/** * vzalloc - allocate virtually contiguous memory with zero fill * @size: allocation size * Allocate enough pages to cover @size from the page level @@ -1787,6 +1803,30 @@ void *vmalloc_user(unsigned long size) EXPORT_SYMBOL(vmalloc_user); /** + * vmalloc_color_user - allocate zeroed virtually contiguous memory for userspace + * @size: allocation size + * + * The resulting memory area is zeroed so it can be mapped to userspace + * without leaking data. + */ +void *vmalloc_color_user(unsigned long size) +{ + struct vm_struct *area; + void *ret; + + ret = __vmalloc_node(size, SHMLBA, + GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | GFP_COLOR, + PAGE_KERNEL, NUMA_NO_NODE, + __builtin_return_address(0)); + if (ret) { + area = find_vm_area(ret); + area->flags |= VM_USERMAP; + } + return ret; +} +EXPORT_SYMBOL(vmalloc_color_user); + +/** * vmalloc_node - allocate memory on a specific node * @size: allocation size * @node: numa node diff --git mm/vmstat.c mm/vmstat.c index 4f5cd97..b09cfa5 100644 --- mm/vmstat.c +++ mm/vmstat.c @@ -739,6 +739,7 @@ const char * const vmstat_text[] = { "workingset_nodereclaim", "nr_anon_transparent_hugepages", "nr_free_cma", + "nr_free_hc_pages", /* enum writeback_stat_item counters */ "nr_dirty_threshold", @@ -754,6 +755,7 @@ const char * const vmstat_text[] = { TEXTS_FOR_ZONES("pgalloc") "pgfree", + "pgfree_hc", "pgactivate", "pgdeactivate", @@ -956,6 +958,67 @@ static void pagetypeinfo_showfree_print(struct seq_file *m, } } +static void pagetypeinfo_showpartitionfree_print(struct seq_file *m, + pg_data_t *pgdat, struct zone *zone, int cpu) +{ + int order, mtype; + + for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) { + seq_printf(m, "Node %4d, zone %8s, type %12s ", + pgdat->node_id, + zone->name, + migratetype_names[mtype]); + for (order = 0; order < MAX_PARTITIONED_ORDER; ++order) { + unsigned long freecount = 0; + struct free_area *area; + struct list_head *curr; + + area = &(zone->free_area_d[cpu][order]); + + list_for_each(curr, &area->free_list[mtype]) + freecount++; + seq_printf(m, "%6lu ", freecount); + } + seq_putc(m, '\n'); + } +} + +static void walk_zones_in_node_in_partition(struct seq_file *m, pg_data_t *pgdat, + int cpu, void (*print)(struct seq_file *m, pg_data_t *, struct zone *, int)) +{ + struct zone *zone; + struct zone *node_zones = pgdat->node_zones; + unsigned long flags; + + for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { + if (!populated_zone(zone)) + continue; + + spin_lock_irqsave(&zone->lock, flags); + print(m, pgdat, zone, cpu); + spin_unlock_irqrestore(&zone->lock, flags); + } +} + +/* Print out the free pages at each order for each migatetype and partition */ +static int pagetypeinfo_showpartitioned(struct seq_file *m, void *arg) +{ + int order, cpu; + pg_data_t *pgdat = (pg_data_t *)arg; + + for_each_online_cpu(cpu) { + /* Print header */ + seq_putc(m, '\n'); + seq_printf(m, "CPU%d %-43s ", cpu, "free pages count per migrate type at order"); + for (order = 0; order < MAX_PARTITIONED_ORDER; ++order) + seq_printf(m, "%6d ", order); + seq_putc(m, '\n'); + + walk_zones_in_node_in_partition(m, pgdat, cpu, pagetypeinfo_showpartitionfree_print); + } + return 0; +} + /* Print out the free pages at each order for each migatetype */ static int pagetypeinfo_showfree(struct seq_file *m, void *arg) { @@ -1138,7 +1201,7 @@ static int pagetypeinfo_show(struct seq_file *m, void *arg) pagetypeinfo_showfree(m, pgdat); pagetypeinfo_showblockcount(m, pgdat); pagetypeinfo_showmixedcount(m, pgdat); - + pagetypeinfo_showpartitioned(m, pgdat); return 0; } @@ -1180,10 +1243,26 @@ static const struct file_operations pagetypeinfo_file_ops = { .release = seq_release, }; +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 +/* Decoding page bank number, 0~7 */ +static inline unsigned int page_bank(struct page *page) +{ + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT); +} +/* Decoding page color, 0~15 */ +static inline unsigned int page_color(struct page *page) +{ + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT); +} + static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone) { int i; + seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name); seq_printf(m, "\n pages free %lu" @@ -1232,6 +1311,16 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, seq_printf(m, "\n vm stats threshold: %d", pageset->stat_threshold); #endif + /* pcp test */ +/* seq_printf(m, "\n"); + for (mtype = 0; mtype < MIGRATE_PCPTYPES; mtype++) { + struct page *p; + list_for_each_entry(p, &pageset->pcp.lists[mtype], lru) { + if (p) + seq_printf(m, "page bank = %d color = %d\n", page_bank(p), page_color(p)); + } + } +*/ } seq_printf(m, "\n all_unreclaimable: %u"