diff --git arch/arm/boot/compressed/Makefile arch/arm/boot/compressed/Makefile index 6e1fb2b..e2284fe 100644 --- arch/arm/boot/compressed/Makefile +++ arch/arm/boot/compressed/Makefile @@ -107,6 +107,7 @@ ifeq ($(CONFIG_FUNCTION_TRACER),y) ORIG_CFLAGS := $(KBUILD_CFLAGS) KBUILD_CFLAGS = $(subst -pg, , $(ORIG_CFLAGS)) endif +KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING ccflags-y := -fpic -mno-single-pic-base -fno-builtin -I$(obj) asflags-y := -DZIMAGE diff --git arch/arm/include/asm/unistd.h arch/arm/include/asm/unistd.h index 0c462a9..e6cb574 100644 --- arch/arm/include/asm/unistd.h +++ arch/arm/include/asm/unistd.h @@ -19,7 +19,8 @@ * This may need to be greater than __NR_last_syscall+1 in order to * account for the padding in the syscall table */ -#define __NR_syscalls (388 + NR_litmus_syscalls) +#define __NR_syscalls (392 + NR_litmus_syscalls + 0) + /* * *NOTE*: This is a ghost syscall private to the kernel. Only the diff --git arch/arm/kernel/calls.S arch/arm/kernel/calls.S index f4738a8..1bfbe24 100644 --- arch/arm/kernel/calls.S +++ arch/arm/kernel/calls.S @@ -409,6 +409,16 @@ CALL(sys_wait_for_ts_release) CALL(sys_release_ts) CALL(sys_null_call) +/* 400 */ CALL(sys_get_current_budget) + CALL(sys_reservation_create) + CALL(sys_reservation_destroy) + CALL(sys_set_mc2_task_param) + CALL(sys_set_page_color) +/* 405 */ CALL(sys_test_call) + CALL(sys_run_test) + CALL(sys_lock_buffer) + CALL(sys_request_mode) + CALL(sys_enact_mode) #ifndef syscalls_counted .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls diff --git arch/arm/mm/cache-l2x0.c arch/arm/mm/cache-l2x0.c index e309c8f..71c969a 100644 --- arch/arm/mm/cache-l2x0.c +++ arch/arm/mm/cache-l2x0.c @@ -33,6 +33,8 @@ #include "cache-tauros3.h" #include "cache-aurora-l2.h" +#include + struct l2c_init_data { const char *type; unsigned way_size_0; @@ -726,7 +728,6 @@ static void __init l2c310_fixup(void __iomem *base, u32 cache_id, if (n) { unsigned i; - pr_info("L2C-310 errat%s", n > 1 ? "a" : "um"); for (i = 0; i < n; i++) pr_cont(" %s", errata[i]); @@ -774,6 +775,11 @@ static const struct l2c_init_data l2c310_init_fns __initconst = { }, }; +void l2c310_flush_all(void) +{ + l2c210_flush_all(); +}; + static int __init __l2c_init(const struct l2c_init_data *data, u32 aux_val, u32 aux_mask, u32 cache_id) { @@ -876,6 +882,8 @@ static int __init __l2c_init(const struct l2c_init_data *data, pr_info("%s: CACHE_ID 0x%08x, AUX_CTRL 0x%08x\n", data->type, cache_id, aux); + litmus_setup_lockdown(l2x0_base, cache_id); + return 0; } diff --git arch/x86/syscalls/syscall_32.tbl arch/x86/syscalls/syscall_32.tbl index 34680a5..b303a9b 100644 --- arch/x86/syscalls/syscall_32.tbl +++ arch/x86/syscalls/syscall_32.tbl @@ -377,3 +377,11 @@ 368 i386 wait_for_ts_release sys_wait_for_ts_release 369 i386 release_ts sys_release_ts 370 i386 null_call sys_null_call +371 i386 get_current_budget sys_get_current_budget +372 i386 reservation_create sys_reservation_create +373 i386 reservation_destroy sys_reservation_destroy +374 i386 set_mc2_task_param sys_set_mc2_task_param +375 i386 set_page_color sys_set_page_color +376 i386 test_call sys_test_call +377 i386 run_test sys_run_test +378 i386 lock_buffer sys_lock_buffer diff --git arch/x86/syscalls/syscall_64.tbl arch/x86/syscalls/syscall_64.tbl index cbd1b6b..5f24a80 100644 --- arch/x86/syscalls/syscall_64.tbl +++ arch/x86/syscalls/syscall_64.tbl @@ -342,6 +342,14 @@ 360 common wait_for_ts_release sys_wait_for_ts_release 361 common release_ts sys_release_ts 362 common null_call sys_null_call +363 common get_current_budget sys_get_current_budget +364 common reservation_create sys_reservation_create +365 common reservation_destroy sys_reservation_destroy +366 common set_mc2_task_param sys_set_mc2_task_param +367 common set_page_color sys_set_page_color +368 common test_call sys_test_call +369 common run_test sys_run_test +370 common lock_buffer sys_lock_buffer # # x32-specific system call numbers start at 512 to avoid cache impact diff --git include/litmus/budget.h include/litmus/budget.h index bd2d5c9..60eb814 100644 --- include/litmus/budget.h +++ include/litmus/budget.h @@ -33,4 +33,6 @@ static inline int requeue_preempted_job(struct task_struct* t) (!budget_exhausted(t) || !budget_enforced(t)); } +void litmus_current_budget(lt_t *used_so_far, lt_t *remaining); + #endif diff --git include/litmus/cache_proc.h include/litmus/cache_proc.h new file mode 100644 index 0000000..e9440de --- /dev/null +++ include/litmus/cache_proc.h @@ -0,0 +1,17 @@ +#ifndef LITMUS_CACHE_PROC_H +#define LITMUS_CACHE_PROC_H + +#ifdef __KERNEL__ + +void litmus_setup_lockdown(void __iomem*, u32); +void enter_irq_mode(void); +void exit_irq_mode(void); +void flush_cache(int all); +void lock_cache(int cpu, u32 val); + +extern struct page *new_alloc_page_color(unsigned long color); + +#endif + +#endif + diff --git include/litmus/litmus.h include/litmus/litmus.h index a6eb534..1037b48 100644 --- include/litmus/litmus.h +++ include/litmus/litmus.h @@ -113,6 +113,13 @@ static inline lt_t litmus_clock(void) ((current)->state == TASK_RUNNING || \ preempt_count() & PREEMPT_ACTIVE) +#define is_running(t) \ + ((t)->state == TASK_RUNNING || \ + task_thread_info(t)->preempt_count & PREEMPT_ACTIVE) + +#define is_blocked(t) \ + (!is_running(t)) + #define is_released(t, now) \ (lt_before_eq(get_release(t), now)) #define is_tardy(t, now) \ @@ -130,6 +137,11 @@ void preempt_if_preemptable(struct task_struct* t, int on_cpu); #define bheap2task(hn) ((struct task_struct*) hn->value) +static inline int is_mode_poll_task(struct task_struct *t) +{ + return tsk_rt(t)->ctrl_page ? tsk_rt(t)->ctrl_page->mode_poll_task : 0; +} + #ifdef CONFIG_NP_SECTION static inline int is_kernel_np(struct task_struct *t) diff --git include/litmus/mc2_common.h include/litmus/mc2_common.h new file mode 100644 index 0000000..4b077ca --- /dev/null +++ include/litmus/mc2_common.h @@ -0,0 +1,35 @@ +/* + * MC^2 common data structures + */ + +#ifndef __UNC_MC2_COMMON_H__ +#define __UNC_MC2_COMMON_H__ + +#define NR_MODES 32 + +enum crit_level { + CRIT_LEVEL_A = 0, + CRIT_LEVEL_B = 1, + CRIT_LEVEL_C = 2, + NUM_CRIT_LEVELS = 3, + MODE_POLL_TASK = 4, +}; + +struct mc2_task { + enum crit_level crit; + unsigned int res_id; + uint32_t mode_mask; + int init_finished; +}; + +#ifdef __KERNEL__ + +#include + +#define tsk_mc2_data(t) (tsk_rt(t)->mc2_data) +long mc2_task_client_init(struct task_client *tc, struct mc2_task *mc2_param, struct task_struct *tsk, + struct reservation *res); + +#endif /* __KERNEL__ */ + +#endif diff --git include/litmus/polling_reservations.h include/litmus/polling_reservations.h new file mode 100644 index 0000000..66c9b1e --- /dev/null +++ include/litmus/polling_reservations.h @@ -0,0 +1,36 @@ +#ifndef LITMUS_POLLING_RESERVATIONS_H +#define LITMUS_POLLING_RESERVATIONS_H + +#include + +struct polling_reservation { + /* extend basic reservation */ + struct reservation res; + + lt_t max_budget; + lt_t period; + lt_t deadline; + lt_t offset; +}; + +void polling_reservation_init(struct polling_reservation *pres, int use_edf_prio, + int use_periodic_polling, lt_t budget, lt_t period, lt_t deadline, lt_t offset); + +struct table_driven_reservation { + /* extend basic reservation */ + struct reservation res; + + lt_t major_cycle; + unsigned int next_interval; + unsigned int num_intervals; + struct lt_interval *intervals; + + /* info about current scheduling slot */ + struct lt_interval cur_interval; + lt_t major_cycle_start; +}; + +void table_driven_reservation_init(struct table_driven_reservation *tdres, + lt_t major_cycle, struct lt_interval *intervals, unsigned int num_intervals); + +#endif diff --git include/litmus/reservation.h include/litmus/reservation.h new file mode 100644 index 0000000..ee3aac2 --- /dev/null +++ include/litmus/reservation.h @@ -0,0 +1,263 @@ +#ifndef LITMUS_RESERVATION_H +#define LITMUS_RESERVATION_H + +#include +#include + +struct reservation_client; +struct reservation_environment; +struct reservation; + +typedef enum { + /* reservation has no clients, is not consuming budget */ + RESERVATION_INACTIVE = 0, + + /* reservation has clients, consumes budget when scheduled */ + RESERVATION_ACTIVE, + + /* reservation has no clients, but may be consuming budget */ + RESERVATION_ACTIVE_IDLE, + + /* Reservation has no budget and waits for + * replenishment. May or may not have clients. */ + RESERVATION_DEPLETED, +} reservation_state_t; + + +/* ************************************************************************** */ + +/* Select which task to dispatch. If NULL is returned, it means there is nothing + * to schedule right now and background work can be scheduled. */ +typedef struct task_struct * (*dispatch_t) ( + struct reservation_client *client +); + +/* Something that can be managed in a reservation and that can yield + * a process for dispatching. Contains a pointer to the reservation + * to which it "belongs". */ +struct reservation_client { + struct list_head list; + struct reservation* reservation; + dispatch_t dispatch; +}; + + +/* ************************************************************************** */ + +/* Called by reservations to request state change. */ +typedef void (*reservation_change_state_t) ( + struct reservation_environment* env, + struct reservation *res, + reservation_state_t new_state +); + +/* The framework within wich reservations operate. */ +struct reservation_environment { + lt_t time_zero; + lt_t current_time; + + /* services invoked by reservations */ + reservation_change_state_t change_state; +}; + + +/* ************************************************************************** */ + +/* A new client is added or an existing client resumes. */ +typedef void (*client_arrives_t) ( + struct reservation *reservation, + struct reservation_client *client +); + +/* A client suspends or terminates. */ +typedef void (*client_departs_t) ( + struct reservation *reservation, + struct reservation_client *client, + int did_signal_job_completion +); + +/* A previously requested replenishment has occurred. */ +typedef void (*on_replenishment_timer_t) ( + struct reservation *reservation +); + +/* Update the reservation's budget to reflect execution or idling. */ +typedef void (*drain_budget_t) ( + struct reservation *reservation, + lt_t how_much +); + +/* Select a ready task from one of the clients for scheduling. */ +typedef struct task_struct* (*dispatch_client_t) ( + struct reservation *reservation, + lt_t *time_slice /* May be used to force rescheduling after + some amount of time. 0 => no limit */ +); + + +struct reservation_ops { + dispatch_client_t dispatch_client; + + client_arrives_t client_arrives; + client_departs_t client_departs; + + on_replenishment_timer_t replenish; + drain_budget_t drain_budget; +}; + +struct reservation { + /* used to queue in environment */ + struct list_head list; + + reservation_state_t state; + unsigned int id; + + /* exact meaning defined by impl. */ + lt_t priority; + lt_t cur_budget; + lt_t next_replenishment; + + /* budget stats */ + lt_t budget_consumed; /* how much budget consumed in this allocation cycle? */ + lt_t budget_consumed_total; + + /* interaction with framework */ + struct reservation_environment *env; + struct reservation_ops *ops; + + struct list_head clients; + + /* for global env. */ + int scheduled_on; + int event_added; + /* for blocked by ghost. Do not charge budget when ACTIVE */ + int blocked_by_ghost; + /* ghost_job. If it is clear, do not charge budget when ACTIVE_IDLE */ + int is_ghost; + + /* for mode change */ + int reported; + int mode; + //under mc2 assume only 1 task per reservation + //multiple reservation per task + struct task_struct *tsk; +}; + +void reservation_init(struct reservation *res); + +/* Default implementations */ + +/* simply select the first client in the list, set *for_at_most to zero */ +struct task_struct* default_dispatch_client( + struct reservation *res, + lt_t *for_at_most +); + +/* "connector" reservation client to hook up tasks with reservations */ +struct task_client { + struct reservation_client client; + struct task_struct *task; +}; + +void task_client_init(struct task_client *tc, struct task_struct *task, + struct reservation *reservation); + +#define SUP_RESCHEDULE_NOW (0) +#define SUP_NO_SCHEDULER_UPDATE (ULLONG_MAX) + +/* A simple uniprocessor (SUP) flat (i.e., non-hierarchical) reservation + * environment. + */ +struct sup_reservation_environment { + struct reservation_environment env; + + /* ordered by priority */ + struct list_head active_reservations; + + /* ordered by next_replenishment */ + struct list_head depleted_reservations; + + /* unordered */ + struct list_head inactive_reservations; + + /* - SUP_RESCHEDULE_NOW means call sup_dispatch() now + * - SUP_NO_SCHEDULER_UPDATE means nothing to do + * any other value means program a timer for the given time + */ + lt_t next_scheduler_update; + /* set to true if a call to sup_dispatch() is imminent */ + bool will_schedule; +}; + +/* Contract: + * - before calling into sup_ code, or any reservation methods, + * update the time with sup_update_time(); and + * - after calling into sup_ code, or any reservation methods, + * check next_scheduler_update and program timer or trigger + * scheduler invocation accordingly. + */ + +void sup_init(struct sup_reservation_environment* sup_env); +void sup_add_new_reservation(struct sup_reservation_environment* sup_env, + struct reservation* new_res); +void sup_scheduler_update_after(struct sup_reservation_environment* sup_env, + lt_t timeout); +void sup_update_time(struct sup_reservation_environment* sup_env, lt_t now); +struct task_struct* sup_dispatch(struct sup_reservation_environment* sup_env); + +struct reservation* sup_find_by_id(struct sup_reservation_environment* sup_env, + unsigned int id); + +/* A global multiprocessor reservation environment. */ + +typedef enum { + EVENT_REPLENISH = 0, + EVENT_DRAIN, + EVENT_OTHERS, +} event_type_t; + + +struct next_timer_event { + lt_t next_update; + int timer_armed_on; + unsigned int id; + event_type_t type; + struct list_head list; +}; + +struct gmp_reservation_environment { + //raw_spinlock_t lock; + struct reservation_environment env; + + /* ordered by priority */ + struct list_head active_reservations; + + /* ordered by next_replenishment */ + struct list_head depleted_reservations; + + /* unordered */ + struct list_head inactive_reservations; + + /* timer event ordered by next_update */ + struct list_head next_events; + + /* (schedule_now == true) means call gmp_dispatch() now */ + int schedule_now; + /* set to true if a call to gmp_dispatch() is imminent */ + bool will_schedule; +}; + +void gmp_init(struct gmp_reservation_environment* gmp_env); +void gmp_add_new_reservation(struct gmp_reservation_environment* gmp_env, + struct reservation* new_res); +void gmp_add_event_after(struct gmp_reservation_environment* gmp_env, + lt_t timeout, unsigned int id, event_type_t type); +void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now); +int gmp_update_time(struct gmp_reservation_environment* gmp_env, lt_t now); +struct task_struct* gmp_dispatch(struct gmp_reservation_environment* gmp_env); +struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env, unsigned int id); +struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env, lt_t when); +struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env, + unsigned int id); + +#endif diff --git include/litmus/rt_param.h include/litmus/rt_param.h index 7b9a909..2ec2d0c 100644 --- include/litmus/rt_param.h +++ include/litmus/rt_param.h @@ -51,6 +51,16 @@ typedef enum { TASK_EARLY } release_policy_t; +#ifdef CONFIG_PGMRT_SUPPORT +typedef enum { + PGM_NOT_A_NODE, + PGM_SRC, + PGM_SINK, + PGM_SRC_SINK, + PGM_INTERNAL +} pgm_node_type_t; +#endif + /* We use the common priority interpretation "lower index == higher priority", * which is commonly used in fixed-priority schedulability analysis papers. * So, a numerically lower priority value implies higher scheduling priority, @@ -62,6 +72,7 @@ typedef enum { #define LITMUS_MAX_PRIORITY 512 #define LITMUS_HIGHEST_PRIORITY 1 #define LITMUS_LOWEST_PRIORITY (LITMUS_MAX_PRIORITY - 1) +#define LITMUS_NO_PRIORITY UINT_MAX /* Provide generic comparison macros for userspace, * in case that we change this later. */ @@ -71,6 +82,49 @@ typedef enum { ((p) >= LITMUS_HIGHEST_PRIORITY && \ (p) <= LITMUS_LOWEST_PRIORITY) +/* reservation support */ + +typedef enum { + PERIODIC_POLLING, + SPORADIC_POLLING, + TABLE_DRIVEN, +} reservation_type_t; + +struct lt_interval { + lt_t start; + lt_t end; +}; + +#ifndef __KERNEL__ +#define __user +#endif + +struct reservation_config { + unsigned int id; + lt_t priority; + int cpu; + + union { + struct { + lt_t period; + lt_t budget; + lt_t relative_deadline; + lt_t offset; + } polling_params; + + struct { + lt_t major_cycle_length; + unsigned int num_intervals; + struct lt_interval __user *intervals; + } table_driven_params; + }; + + //Need to inform which mode reservation belongs to in mc2 + int mode; +}; + +/* regular sporadic task support */ + struct rt_task { lt_t exec_cost; lt_t period; @@ -81,6 +135,10 @@ struct rt_task { task_class_t cls; budget_policy_t budget_policy; /* ignored by pfair */ release_policy_t release_policy; +#ifdef CONFIG_PGMRT_SUPPORT + pgm_node_type_t pgm_type; + lt_t pgm_expected_etoe; +#endif }; union np_flag { @@ -120,6 +178,14 @@ struct control_page { uint64_t ts_syscall_start; /* Feather-Trace cycles */ uint64_t irq_syscall_start; /* Snapshot of irq_count when the syscall * started. */ + volatile uint64_t mode_poll_task; /* Set to 1 if the task invokes enact_mode syscall */ + +#ifdef CONFIG_PGMRT_SUPPORT + /* Flags from userspace signifying PGM wait states. */ + volatile uint32_t pgm_waiting; /* waiting for tokens */ + volatile uint32_t pgm_sending; /* sending tokens */ + volatile uint32_t pgm_satisfied; /* done waiting/sending */ +#endif /* to be extended */ }; @@ -130,6 +196,7 @@ struct control_page { #define LITMUS_CP_OFFSET_IRQ_COUNT 8 #define LITMUS_CP_OFFSET_TS_SC_START 16 #define LITMUS_CP_OFFSET_IRQ_SC_START 24 +#define LITMUS_CP_OFFSET_MODE_POLL_TASK 32 /* don't export internal data structures to user space (liblitmus) */ #ifdef __KERNEL__ @@ -165,6 +232,7 @@ struct rt_job { }; struct pfair_param; +struct mc2_task; /* RT task parameters for scheduling extensions * These parameters are inherited during clone and therefore must @@ -246,7 +314,10 @@ struct rt_param { volatile int linked_on; /* PFAIR/PD^2 state. Allocated on demand. */ - struct pfair_param* pfair; + union { + void *plugin_state; + struct pfair_param *pfair; + }; /* Fields saved before BE->RT transition. */ @@ -275,6 +346,10 @@ struct rt_param { /* Pointer to the page shared between userspace and kernel. */ struct control_page * ctrl_page; + + /* Mixed-criticality specific data */ + struct mc2_task* mc2_data; + unsigned long addr_ctrl_page; }; #endif diff --git include/litmus/sched_plugin.h include/litmus/sched_plugin.h index 0ccccd6..4c8aaa6 100644 --- include/litmus/sched_plugin.h +++ include/litmus/sched_plugin.h @@ -77,6 +77,17 @@ typedef long (*wait_for_release_at_t)(lt_t release_time); /* Informs the plugin when a synchronous release takes place. */ typedef void (*synchronous_release_at_t)(lt_t time_zero); +/* How much budget has the current task consumed so far, and how much + * has it left? The default implementation ties into the per-task + * budget enforcement code. Plugins can override this to report + * reservation-specific values. */ +typedef void (*current_budget_t)(lt_t *used_so_far, lt_t *remaining); + +/* Reservation creation/removal backends. Meaning of reservation_type and + * reservation_id are entirely plugin-specific. */ +typedef long (*reservation_create_t)(int reservation_type, void* __user config); +typedef long (*reservation_destroy_t)(unsigned int reservation_id, int cpu); + /************************ misc routines ***********************/ @@ -109,6 +120,12 @@ struct sched_plugin { task_exit_t task_exit; task_cleanup_t task_cleanup; + current_budget_t current_budget; + + /* Reservation support */ + reservation_create_t reservation_create; + reservation_destroy_t reservation_destroy; + #ifdef CONFIG_LITMUS_LOCKING /* locking protocols */ allocate_lock_t allocate_lock; diff --git include/litmus/sched_trace.h include/litmus/sched_trace.h index 82bde82..80d9523 100644 --- include/litmus/sched_trace.h +++ include/litmus/sched_trace.h @@ -50,13 +50,12 @@ struct st_switch_away_data { /* A process was switched away from on a given CPU. u64 exec_time; }; -struct st_completion_data { /* A job completed. */ - u64 when; - u8 forced:1; /* Set to 1 if job overran and kernel advanced to the - * next task automatically; set to 0 otherwise. - */ - u8 __uflags:7; - u8 __unused[7]; +struct st_completion_data { /* A job completed. */ + u64 when; + u8 forced:1; /* Set to 1 if job overran and kernel advanced to the + * next job automatically; set to 0 otherwise. + */ + u64 exec_time:63; /* Actual execution time of job. */ }; struct st_block_data { /* A task blocks. */ @@ -80,6 +79,21 @@ struct st_sys_release_data { u64 release; }; +struct st_enact_mode_data { + u64 when; + u8 __unused[8]; +}; + +struct st_request_mode_data { + u64 when; + u8 __unused[8]; +}; + +struct st_sys_start_data { + u64 when; + u64 start; +}; + #define DATA(x) struct st_ ## x ## _data x; typedef enum { @@ -87,14 +101,16 @@ typedef enum { * uninitialized records. */ ST_PARAM, ST_RELEASE, - ST_ASSIGNED, ST_SWITCH_TO, ST_SWITCH_AWAY, ST_COMPLETION, ST_BLOCK, ST_RESUME, ST_ACTION, - ST_SYS_RELEASE + ST_SYS_RELEASE, + ST_ENACT_MODE, + ST_REQUEST_MODE, + ST_SYS_START, } st_event_record_type_t; struct st_event_record { @@ -105,7 +121,6 @@ struct st_event_record { DATA(name); DATA(param); DATA(release); - DATA(assigned); DATA(switch_to); DATA(switch_away); DATA(completion); @@ -113,6 +128,9 @@ struct st_event_record { DATA(resume); DATA(action); DATA(sys_release); + DATA(enact_mode); + DATA(request_mode); + DATA(sys_start); } data; }; @@ -155,6 +173,14 @@ feather_callback void do_sched_trace_action(unsigned long id, feather_callback void do_sched_trace_sys_release(unsigned long id, lt_t* start); +feather_callback void do_sched_trace_enact_mode(unsigned long id, + struct task_struct* task); + +feather_callback void do_sched_trace_request_mode(unsigned long id, + struct task_struct* task); +feather_callback void do_sched_trace_sys_start(unsigned long id, + lt_t* start); + #endif #else @@ -179,6 +205,9 @@ feather_callback void do_sched_trace_sys_release(unsigned long id, #define trace_litmus_task_block(t) #define trace_litmus_task_resume(t) #define trace_litmus_sys_release(start) +#define trace_litmus_enact_mode(t) +#define trace_litmus_request_mode(t) +#define trace_litmus_sys_start(start) #endif @@ -254,6 +283,28 @@ feather_callback void do_sched_trace_sys_release(unsigned long id, #define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */ +#define sched_trace_enact_mode(t) \ + do { \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 11, \ + do_sched_trace_enact_mode, t); \ + trace_litmus_enact_mode(t); \ + } while (0) + +#define sched_trace_request_mode(t) \ + do { \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, \ + do_sched_trace_request_mode, t); \ + trace_litmus_request_mode(t); \ + } while (0) + +/* when is a pointer, it does not need an explicit cast to unsigned long */ +#define sched_trace_sys_start(when) \ + do { \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 13, \ + do_sched_trace_sys_start, when); \ + trace_litmus_sys_start(when); \ + } while (0) + #endif /* __KERNEL__ */ #endif diff --git include/litmus/trace.h include/litmus/trace.h index 6017872..4dbb39ea 100644 --- include/litmus/trace.h +++ include/litmus/trace.h @@ -3,7 +3,6 @@ #ifdef CONFIG_SCHED_OVERHEAD_TRACE - #include #include @@ -118,6 +117,9 @@ feather_callback void save_cpu_task_latency(unsigned long event, unsigned long w #define TS_TICK_START(t) CPU_TTIMESTAMP(110, t) #define TS_TICK_END(t) CPU_TTIMESTAMP(111, t) +#define TS_RELEASE_C_START CPU_DTIMESTAMP(108, TSK_RT) +#define TS_RELEASE_C_END CPU_DTIMESTAMP(109, TSK_RT) + #define TS_QUANTUM_BOUNDARY_START CPU_TIMESTAMP_CUR(112) #define TS_QUANTUM_BOUNDARY_END CPU_TIMESTAMP_CUR(113) @@ -137,6 +139,20 @@ feather_callback void save_cpu_task_latency(unsigned long event, unsigned long w #define TS_SEND_RESCHED_START(c) MSG_TIMESTAMP_SENT(190, c) #define TS_SEND_RESCHED_END MSG_TIMESTAMP_RECEIVED(191) -#define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when)) +#define TS_ISR_START CPU_TIMESTAMP_CUR(192) +#define TS_ISR_END CPU_TIMESTAMP_CUR(193) + +#define TS_MODE_CHANGE_START CPU_TIMESTAMP(194) +#define TS_MODE_CHANGE_END CPU_TIMESTAMP(195) + +#define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when)) +#define TS_RELEASE_LATENCY_A(when) CPU_LTIMESTAMP(209, &(when)) +#define TS_RELEASE_LATENCY_B(when) CPU_LTIMESTAMP(210, &(when)) +#define TS_RELEASE_LATENCY_C(when) CPU_LTIMESTAMP(211, &(when)) + +#define TS_SCHED_A_START CPU_DTIMESTAMP(212, TSK_UNKNOWN) +#define TS_SCHED_A_END(t) CPU_TTIMESTAMP(213, t) +#define TS_SCHED_C_START CPU_DTIMESTAMP(214, TSK_UNKNOWN) +#define TS_SCHED_C_END(t) CPU_TTIMESTAMP(215, t) #endif /* !_SYS_TRACE_H_ */ diff --git include/litmus/unistd_32.h include/litmus/unistd_32.h index 94264c2..86bbbb8d 100644 --- include/litmus/unistd_32.h +++ include/litmus/unistd_32.h @@ -17,5 +17,13 @@ #define __NR_wait_for_ts_release __LSC(9) #define __NR_release_ts __LSC(10) #define __NR_null_call __LSC(11) +#define __NR_get_current_budget __LSC(12) +#define __NR_reservation_create __LSC(13) +#define __NR_reservation_destroy __LSC(14) +#define __NR_set_mc2_task_param __LSC(15) +#define __NR_set_page_color __LSC(16) +#define __NR_test_call __LSC(17) +#define __NR_run_test __LSC(18) +#define __NR_lock_buffer __LSC(19) -#define NR_litmus_syscalls 12 +#define NR_litmus_syscalls 20 diff --git include/litmus/unistd_64.h include/litmus/unistd_64.h index d5ced0d..4b96e7c 100644 --- include/litmus/unistd_64.h +++ include/litmus/unistd_64.h @@ -29,5 +29,22 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release) __SYSCALL(__NR_release_ts, sys_release_ts) #define __NR_null_call __LSC(11) __SYSCALL(__NR_null_call, sys_null_call) +#define __NR_get_current_budget __LSC(12) +__SYSCALL(__NR_get_current_budget, sys_get_current_budget) +#define __NR_reservation_create __LSC(13) +__SYSCALL(__NR_reservation_create, sys_reservation_create) +#define __NR_reservation_destroy __LSC(14) +__SYSCALL(__NR_reservation_destroy, sys_reservation_destroy) +#define __NR_set_mc2_task_param __LSC(15) +__SYSCALL(__NR_set_mc2_task_param, sys_set_mc2_task_param) +#define __NR_set_page_color __LSC(16) +__SYSCALL(__NR_set_page_color, sys_set_page_color) +#define __NR_test_call __LSC(17) +__SYSCALL(__NR_test_call, sys_test_call) +#define __NR_run_test __LSC(18) +__SYSCALL(__NR_run_test, sys_run_test) +#define __NR_lock_buffer __LSC(19) +__SYACALL(__NR_lock_buffer, sys_lock_buffer) -#define NR_litmus_syscalls 12 + +#define NR_litmus_syscalls 20 diff --git include/trace/events/litmus.h include/trace/events/litmus.h index 0fffcee..ef8d60f2 100644 --- include/trace/events/litmus.h +++ include/trace/events/litmus.h @@ -225,6 +225,76 @@ TRACE_EVENT(litmus_sys_release, TP_printk("SynRelease(%Lu) at %Lu\n", __entry->rel, __entry->when) ); +/* + * Trace enact mode + */ +TRACE_EVENT(litmus_enact_mode, + + TP_PROTO(struct task_struct *t), + + TP_ARGS(t), + + TP_STRUCT__entry( + __field( pid_t, pid ) + __field( unsigned int, job ) + __field( lt_t, when ) + ), + + TP_fast_assign( + __entry->pid = t ? t->pid : 0; + __entry->job = t ? t->rt_param.job_params.job_no : 0; + __entry->when = litmus_clock(); + ), + + TP_printk("Mode Enact at %Lu\n", __entry->when) +); + +/* + * Trace notice mode request + */ +TRACE_EVENT(litmus_request_mode, + + TP_PROTO(struct task_struct *t), + + TP_ARGS(t), + + TP_STRUCT__entry( + __field( pid_t, pid ) + __field( unsigned int, job ) + __field( lt_t, when ) + ), + + TP_fast_assign( + __entry->pid = t ? t->pid : 0; + __entry->job = t ? t->rt_param.job_params.job_no : 0; + __entry->when = litmus_clock(); + ), + + TP_printk("Mode request at %Lu\n", __entry->when) +); + +/* + * Trace synchronous start + */ +TRACE_EVENT(litmus_sys_start, + + TP_PROTO(lt_t *start), + + TP_ARGS(start), + + TP_STRUCT__entry( + __field( lt_t, rel ) + __field( lt_t, when ) + ), + + TP_fast_assign( + __entry->rel = *start; + __entry->when = litmus_clock(); + ), + + TP_printk("SynStart(%Lu) at %Lu\n", __entry->rel, __entry->when) +); + #endif /* _SCHED_TASK_TRACEPOINT_H */ /* Must stay outside the protection */ diff --git kernel/sched/litmus.c kernel/sched/litmus.c index 9d58690..60be718 100644 --- kernel/sched/litmus.c +++ kernel/sched/litmus.c @@ -20,8 +20,9 @@ static void update_time_litmus(struct rq *rq, struct task_struct *p) /* task counter */ p->se.sum_exec_runtime += delta; if (delta) { - TRACE_TASK(p, "charged %llu exec time (total:%llu, rem:%llu)\n", - delta, p->rt_param.job_params.exec_time, budget_remaining(p)); + //TRACE_TASK(p, "charged %llu exec time (total:%llu, rem:%llu)\n", + // delta, p->rt_param.job_params.exec_time, budget_remaining(p)); + ; } /* sched_clock() */ p->se.exec_start = rq->clock; diff --git litmus/Kconfig litmus/Kconfig index babb43d..d47548d 100644 --- litmus/Kconfig +++ litmus/Kconfig @@ -243,7 +243,7 @@ config SCHED_TASK_TRACE config SCHED_TASK_TRACE_SHIFT int "Buffer size for sched_trace_xxx() events" depends on SCHED_TASK_TRACE - range 8 13 + range 8 22 default 9 help diff --git litmus/Makefile litmus/Makefile index 7970cd5..e274409 100644 --- litmus/Makefile +++ litmus/Makefile @@ -11,6 +11,7 @@ obj-y = sched_plugin.o litmus.o \ sync.o \ rt_domain.o \ edf_common.o \ + mc2_common.o \ fp_common.o \ fdso.o \ locking.o \ @@ -19,13 +20,18 @@ obj-y = sched_plugin.o litmus.o \ binheap.o \ ctrldev.o \ uncachedev.o \ + reservation.o \ + polling_reservations.o \ sched_gsn_edf.o \ sched_psn_edf.o \ - sched_pfp.o + sched_pfp.o \ + sched_mc2.o \ + bank_proc.o \ + color_shm.o \ + cache_proc.o obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o - obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o obj-$(CONFIG_SCHED_DEBUG_TRACE) += sched_trace.o diff --git litmus/bank_proc.c litmus/bank_proc.c new file mode 100644 index 0000000..097cff1 --- /dev/null +++ litmus/bank_proc.c @@ -0,0 +1,793 @@ +/* + * bank_proc.c -- Implementation of the page coloring for cache and bank partition. + * The file will keep a pool of colored pages. Users can require pages with + * specific color or bank number. + * Part of the code is modified from Jonathan Herman's code + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define LITMUS_LOCKDEP_NAME_MAX_LEN 50 + +// This Address Decoding is used in imx6-sabredsd platform +#define BANK_MASK 0x38000000 +#define BANK_SHIFT 27 +#define CACHE_MASK 0x0000f000 +#define CACHE_SHIFT 12 + +#define PAGES_PER_COLOR 2000 +#define NUM_BANKS 8 +#define NUM_COLORS 16 + +unsigned int NUM_PAGE_LIST; //8*16 + +unsigned int number_banks; +unsigned int number_cachecolors; + +unsigned int set_partition_max = 0x0000ffff; +unsigned int set_partition_min = 0; +unsigned int bank_partition_max = 0x000000ff; +unsigned int bank_partition_min = 0; + +int show_page_pool = 0; +int refill_page_pool = 0; +spinlock_t reclaim_lock; + +unsigned int set_partition[9] = { + 0x00000003, /* Core 0, and Level A*/ + 0x00000003, /* Core 0, and Level B*/ + 0x0000000C, /* Core 1, and Level A*/ + 0x0000000C, /* Core 1, and Level B*/ + 0x00000030, /* Core 2, and Level A*/ + 0x00000030, /* Core 2, and Level B*/ + 0x000000C0, /* Core 3, and Level A*/ + 0x000000C0, /* Core 3, and Level B*/ + 0x0000ff00, /* Level C */ +}; + +unsigned int bank_partition[9] = { + 0x00000010, /* Core 0, and Level A*/ + 0x00000010, /* Core 0, and Level B*/ + 0x00000020, /* Core 1, and Level A*/ + 0x00000020, /* Core 1, and Level B*/ + 0x00000040, /* Core 2, and Level A*/ + 0x00000040, /* Core 2, and Level B*/ + 0x00000080, /* Core 3, and Level A*/ + 0x00000080, /* Core 3, and Level B*/ + 0x0000000c, /* Level C */ +}; + +unsigned int set_index[9] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +unsigned int bank_index[9] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +int node_index[9] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1 +}; + +struct mutex void_lockdown_proc; + +/* + * Every page list should contain a lock, a list, and a number recording how many pages it store + */ +struct color_group { + spinlock_t lock; + char _lock_name[LITMUS_LOCKDEP_NAME_MAX_LEN]; + struct list_head list; + atomic_t nr_pages; +}; + + +static struct color_group *color_groups; + +/* + * Naive function to count the number of 1's + */ +unsigned int counting_one_set(unsigned int v) +{ + unsigned int c; // c accumulates the total bits set in v + + for (c = 0; v; v >>= 1) + { + c += v & 1; + } + return c; +} + +unsigned int two_exp(unsigned int e) +{ + unsigned int v = 1; + for (; e>0; e-- ) + { + v=v*2; + } + return v; +} + +unsigned int num_by_bitmask_index(unsigned int bitmask, unsigned int index) +{ + unsigned int pos = 0; + + while(true) + { + if(index ==0 && (bitmask & 1)==1) + { + break; + } + if(index !=0 && (bitmask & 1)==1){ + index--; + } + pos++; + bitmask = bitmask >>1; + + } + return pos; +} + +/* helper functions to find the next colored pool index */ +static inline unsigned int first_index(unsigned long node) +{ + unsigned int bank_no = 0, color_no = 0; + + while(bank_no < NUM_BANKS) { + if ((bank_partition[node]>>bank_no) & 0x1) + break; + bank_no++; + } + while(color_no < NUM_COLORS) { + if ((set_partition[node]>>color_no) & 0x1) + break; + color_no++; + } + return NUM_COLORS*bank_no + color_no; +} + +static inline unsigned int last_index(unsigned long node) +{ + unsigned int bank_no = 7, color_no = 15; + + while(bank_no >= 0) { + if ((bank_partition[node]>>bank_no) & 0x1) + break; + bank_no--; + } + while(color_no >= 0) { + if ((set_partition[node]>>color_no) & 0x1) + break; + color_no--; + } + return NUM_COLORS*bank_no + color_no; +} + +static inline unsigned int next_color(unsigned long node, unsigned int current_color) +{ + int try = 0, ret = 0; + current_color++; + if (current_color == NUM_COLORS) { + current_color = 0; + ret = 1; + } + + while (try < NUM_COLORS) { + if ((set_partition[node]>>current_color)&0x1) + break; + current_color++; + if (current_color == NUM_COLORS) { + current_color = 0; + ret = 1; + } + try++; + } + if (!ret) + return current_color; + else + return current_color + NUM_COLORS; +} + +static inline unsigned int next_bank(unsigned long node, unsigned int current_bank) +{ + int try = 0; + current_bank++; + if (current_bank == NUM_BANKS) { + current_bank = 0; + } + + while (try < NUM_BANKS) { + if ((bank_partition[node]>>current_bank)&0x1) + break; + current_bank++; + if (current_bank == NUM_BANKS) { + current_bank = 0; + } + try++; + } + return current_bank; +} + +static inline unsigned int get_next_index(unsigned long node, unsigned int current_index) +{ + unsigned int bank_no, color_no, color_ret, bank_ret; + bank_no = current_index>>4; // 2^4 = 16 colors + color_no = current_index - bank_no*NUM_COLORS; + bank_ret = bank_no; + color_ret = next_color(node, color_no); + if (color_ret >= NUM_COLORS) { + // next bank + color_ret -= NUM_COLORS; + bank_ret = next_bank(node, bank_no); + } + + return bank_ret * NUM_COLORS + color_ret; +} + +/* Decoding page color, 0~15 */ +static inline unsigned int page_color(struct page *page) +{ + return ((page_to_phys(page)& CACHE_MASK) >> CACHE_SHIFT); +} + +/* Decoding page bank number, 0~7 */ +static inline unsigned int page_bank(struct page *page) +{ + return ((page_to_phys(page)& BANK_MASK) >> BANK_SHIFT); +} + +static inline unsigned int page_list_index(struct page *page) +{ + unsigned int idx; + idx = (page_color(page) + page_bank(page)*(number_cachecolors)); + + return idx; +} + + + +/* + * It is used to determine the smallest number of page lists. + */ +static unsigned long smallest_nr_pages(void) +{ + unsigned long i, min_pages; + struct color_group *cgroup; + cgroup = &color_groups[16*2]; + min_pages =atomic_read(&cgroup->nr_pages); + for (i = 16*2; i < NUM_PAGE_LIST; ++i) { + cgroup = &color_groups[i]; + if (atomic_read(&cgroup->nr_pages) < min_pages) + min_pages = atomic_read(&cgroup->nr_pages); + } + return min_pages; +} + +static void show_nr_pages(void) +{ + unsigned long i; + struct color_group *cgroup; + printk("show nr pages***************************************\n"); + for (i = 0; i < NUM_PAGE_LIST; ++i) { + cgroup = &color_groups[i]; + printk("(%03ld) = %03d, ", i, atomic_read(&cgroup->nr_pages)); + if((i % 8) ==7) { + printk("\n"); + } + } +} + +/* + * Add a page to current pool. + */ +void add_page_to_color_list(struct page *page) +{ + const unsigned long color = page_list_index(page); + struct color_group *cgroup = &color_groups[color]; + BUG_ON(in_list(&page->lru) || PageLRU(page)); + BUG_ON(page_count(page) > 1); + spin_lock(&cgroup->lock); + list_add_tail(&page->lru, &cgroup->list); + atomic_inc(&cgroup->nr_pages); + SetPageLRU(page); + spin_unlock(&cgroup->lock); +} + +/* + * Replenish the page pool. + * If the newly allocate page is what we want, it will be pushed to the correct page list + * otherwise, it will be freed. + * A user needs to invoke this function until the page pool has enough pages. + */ +static int do_add_pages(void) +{ + struct page *page, *page_tmp; + LIST_HEAD(free_later); + unsigned long color; + int ret = 0; + int i = 0; + int free_counter = 0; + unsigned long counter[128]= {0}; + + // until all the page lists contain enough pages + for (i=0; i< 1024*20;i++) { + page = alloc_page(GFP_HIGHUSER_MOVABLE); + + if (unlikely(!page)) { + printk(KERN_WARNING "Could not allocate pages.\n"); + ret = -ENOMEM; + goto out; + } + color = page_list_index(page); + counter[color]++; + if (atomic_read(&color_groups[color].nr_pages) < PAGES_PER_COLOR && color>=0) { + add_page_to_color_list(page); + } else { + // Pages here will be freed later + list_add_tail(&page->lru, &free_later); + free_counter++; + } + } + + // Free the unwanted pages + list_for_each_entry_safe(page, page_tmp, &free_later, lru) { + list_del(&page->lru); + __free_page(page); + } +out: + return ret; +} + +/* + * Provide pages for replacement according cache color + * This should be the only implementation here + * This function should not be accessed by others directly. + * + */ +static struct page *new_alloc_page_color( unsigned long color) +{ +// printk("allocate new page color = %d\n", color); + struct color_group *cgroup; + struct page *rPage = NULL; + + if( (color <0) || (color)>(number_cachecolors*number_banks -1)) { + TRACE_CUR("Wrong color %lu\n", color); + goto out; + } + + + cgroup = &color_groups[color]; + spin_lock(&cgroup->lock); + if (unlikely(!atomic_read(&cgroup->nr_pages))) { + TRACE_CUR("No free %lu colored pages.\n", color); + goto out_unlock; + } + rPage = list_first_entry(&cgroup->list, struct page, lru); + BUG_ON(page_count(rPage) > 1); + //get_page(rPage); + list_del(&rPage->lru); + atomic_dec(&cgroup->nr_pages); + ClearPageLRU(rPage); +out_unlock: + spin_unlock(&cgroup->lock); +out: + return rPage; +} + +struct page* get_colored_page(unsigned long color) +{ + return new_alloc_page_color(color); +} + +/* + * provide pages for replacement according to + * node = 0 for Level A tasks in Cpu 0 + * node = 1 for Level B tasks in Cpu 0 + * node = 2 for Level A tasks in Cpu 1 + * node = 3 for Level B tasks in Cpu 1 + * node = 4 for Level A tasks in Cpu 2 + * node = 5 for Level B tasks in Cpu 2 + * node = 6 for Level A tasks in Cpu 3 + * node = 7 for Level B tasks in Cpu 3 + * node = 8 for Level C tasks + */ +struct page *new_alloc_page(struct page *page, unsigned long node, int **x) +{ + struct page *rPage = NULL; + int try = 0; + unsigned int idx; + + if (node_index[node] == -1) + idx = first_index(node); + else + idx = node_index[node]; + + BUG_ON(idx<0 || idx>127); + rPage = new_alloc_page_color(idx); + if (node_index[node] == last_index(node)) + node_index[node] = first_index(node); + else + node_index[node]++; + + while (!rPage) { + try++; + if (try>=256) + break; + idx = get_next_index(node, idx); + printk(KERN_ALERT "try = %d out of page! requesting node = %ld, idx = %d\n", try, node, idx); + BUG_ON(idx<0 || idx>127); + rPage = new_alloc_page_color(idx); + } + node_index[node] = idx; + return rPage; +} + + +/* + * Reclaim pages. + */ +void reclaim_page(struct page *page) +{ + const unsigned long color = page_list_index(page); + spin_lock(&reclaim_lock); + put_page(page); + add_page_to_color_list(page); + + spin_unlock(&reclaim_lock); + printk("Reclaimed page(%ld) = color %x, bank %x, [color] =%d \n", color, page_color(page), page_bank(page), atomic_read(&color_groups[color].nr_pages)); +} + + +/* + * Initialize the numbers of banks and cache colors + */ +static void __init init_variables(void) +{ + number_banks = counting_one_set(BANK_MASK); + number_banks = two_exp(number_banks); + + number_cachecolors = counting_one_set(CACHE_MASK); + number_cachecolors = two_exp(number_cachecolors); + NUM_PAGE_LIST = number_banks * number_cachecolors; + printk(KERN_WARNING "number of banks = %d, number of cachecolors=%d\n", number_banks, number_cachecolors); + mutex_init(&void_lockdown_proc); + spin_lock_init(&reclaim_lock); + +} + + +/* + * Initialize the page pool + */ +static int __init init_color_groups(void) +{ + struct color_group *cgroup; + unsigned long i; + int err = 0; + + printk("NUM_PAGE_LIST = %d\n", NUM_PAGE_LIST); + color_groups = kmalloc(NUM_PAGE_LIST *sizeof(struct color_group), GFP_KERNEL); + + if (!color_groups) { + printk(KERN_WARNING "Could not allocate color groups.\n"); + err = -ENOMEM; + }else{ + + for (i = 0; i < NUM_PAGE_LIST; ++i) { + cgroup = &color_groups[i]; + atomic_set(&cgroup->nr_pages, 0); + INIT_LIST_HEAD(&cgroup->list); + spin_lock_init(&cgroup->lock); + } + } + return err; +} + +int set_partition_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i = 0; + mutex_lock(&void_lockdown_proc); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + if (write) { + printk("New set Partition : \n"); + for(i =0;i <9;i++) + { + set_index[i] = 0; + printk("set[%d] = %x \n", i, set_partition[i]); + } + } +out: + mutex_unlock(&void_lockdown_proc); + return ret; +} + +int bank_partition_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i = 0; + mutex_lock(&void_lockdown_proc); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + if (write) { + for(i =0;i <9;i++) + { + bank_index[i] = 0; + } + } +out: + mutex_unlock(&void_lockdown_proc); + return ret; +} + +int show_page_pool_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0; + mutex_lock(&void_lockdown_proc); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + if (write) { + show_nr_pages(); + } +out: + mutex_unlock(&void_lockdown_proc); + return ret; +} + +int refill_page_pool_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0; + mutex_lock(&void_lockdown_proc); + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + if (write) { + do_add_pages(); + show_nr_pages(); + } +out: + mutex_unlock(&void_lockdown_proc); + return ret; +} + +static struct ctl_table cache_table[] = +{ + + { + .procname = "C0_LA_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[0], + .maxlen = sizeof(set_partition[0]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C0_LB_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[1], + .maxlen = sizeof(set_partition[1]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C1_LA_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[2], + .maxlen = sizeof(set_partition[2]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C1_LB_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[3], + .maxlen = sizeof(set_partition[3]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C2_LA_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[4], + .maxlen = sizeof(set_partition[4]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C2_LB_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[5], + .maxlen = sizeof(set_partition[5]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C3_LA_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[6], + .maxlen = sizeof(set_partition[6]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C3_LB_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[7], + .maxlen = sizeof(set_partition[7]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "Call_LC_set", + .mode = 0666, + .proc_handler = set_partition_handler, + .data = &set_partition[8], + .maxlen = sizeof(set_partition[8]), + .extra1 = &set_partition_min, + .extra2 = &set_partition_max, + }, + { + .procname = "C0_LA_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[0], + .maxlen = sizeof(set_partition[0]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C0_LB_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[1], + .maxlen = sizeof(set_partition[1]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C1_LA_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[2], + .maxlen = sizeof(set_partition[2]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C1_LB_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[3], + .maxlen = sizeof(set_partition[3]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C2_LA_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[4], + .maxlen = sizeof(set_partition[4]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C2_LB_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[5], + .maxlen = sizeof(set_partition[5]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C3_LA_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[6], + .maxlen = sizeof(set_partition[6]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "C3_LB_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[7], + .maxlen = sizeof(set_partition[7]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "Call_LC_bank", + .mode = 0666, + .proc_handler = bank_partition_handler, + .data = &bank_partition[8], + .maxlen = sizeof(set_partition[8]), + .extra1 = &bank_partition_min, + .extra2 = &bank_partition_max, + }, + { + .procname = "show_page_pool", + .mode = 0666, + .proc_handler = show_page_pool_handler, + .data = &show_page_pool, + .maxlen = sizeof(show_page_pool), + }, { + .procname = "refill_page_pool", + .mode = 0666, + .proc_handler = refill_page_pool_handler, + .data = &refill_page_pool, + .maxlen = sizeof(refill_page_pool), + }, + { } +}; + +static struct ctl_table litmus_dir_table[] = { + { + .procname = "litmus", + .mode = 0555, + .child = cache_table, + }, + { } +}; + + +static struct ctl_table_header *litmus_sysctls; + + +/* + * Initialzie this proc + */ +static int __init litmus_color_init(void) +{ + int err=0; + printk("Init bankproc.c\n"); + + init_variables(); + + printk(KERN_INFO "Registering LITMUS^RT proc color sysctl.\n"); + + litmus_sysctls = register_sysctl_table(litmus_dir_table); + if (!litmus_sysctls) { + printk(KERN_WARNING "Could not register LITMUS^RT color sysctl.\n"); + err = -EFAULT; + goto out; + } + + init_color_groups(); + do_add_pages(); + + printk(KERN_INFO "Registering LITMUS^RT color and bank proc.\n"); +out: + return err; +} + +module_init(litmus_color_init); + diff --git litmus/budget.c litmus/budget.c index 47bf78a..d67f4b3 100644 --- litmus/budget.c +++ litmus/budget.c @@ -1,9 +1,11 @@ #include #include #include +#include #include #include +#include #include @@ -113,4 +115,54 @@ static int __init init_budget_enforcement(void) return 0; } +void litmus_current_budget(lt_t *used_so_far, lt_t *remaining) +{ + struct task_struct *t = current; + unsigned long flags; + s64 delta; + + local_irq_save(flags); + + delta = sched_clock_cpu(smp_processor_id()) - t->se.exec_start; + if (delta < 0) + delta = 0; + + TRACE_CUR("current_budget: sc:%llu start:%llu lt_t:%llu delta:%lld exec-time:%llu rem:%llu\n", + sched_clock_cpu(smp_processor_id()), t->se.exec_start, + litmus_clock(), delta, + tsk_rt(t)->job_params.exec_time, + budget_remaining(t)); + + if (used_so_far) + *used_so_far = tsk_rt(t)->job_params.exec_time + delta; + + if (remaining) { + *remaining = budget_remaining(t); + if (*remaining > delta) + *remaining -= delta; + else + *remaining = 0; + } + + local_irq_restore(flags); +} + +asmlinkage long sys_get_current_budget( + lt_t __user * _expended, + lt_t __user *_remaining) +{ + lt_t expended = 0, remaining = 0; + + if (is_realtime(current)) + litmus->current_budget(&expended, &remaining); + + if (_expended && put_user(expended, _expended)) + return -EFAULT; + + if (_remaining && put_user(remaining, _remaining)) + return -EFAULT; + + return 0; +} + module_init(init_budget_enforcement); diff --git litmus/cache_proc.c litmus/cache_proc.c new file mode 100644 index 0000000..5cdf953 --- /dev/null +++ litmus/cache_proc.c @@ -0,0 +1,1414 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + + +#define UNLOCK_ALL 0x00000000 /* allocation in any way */ +#define LOCK_ALL (~UNLOCK_ALL) +#define MAX_NR_WAYS 16 +#define MAX_NR_COLORS 16 +#define CACHELINE_SIZE 32 +#define INTS_IN_CACHELINE (CACHELINE_SIZE/sizeof(int)) +#define CACHELINES_IN_1KB (1024 / sizeof(cacheline_t)) + +typedef struct cacheline +{ + int line[INTS_IN_CACHELINE]; +} __attribute__((aligned(CACHELINE_SIZE))) cacheline_t; + +void mem_lock(u32 lock_val, int cpu); + +/* + * unlocked_way[i] : allocation can occur in way i + * + * 0 = allocation can occur in the corresponding way + * 1 = allocation cannot occur in the corresponding way + */ +u32 unlocked_way[MAX_NR_WAYS] = { + 0xFFFFFFFE, /* way 0 unlocked */ + 0xFFFFFFFD, + 0xFFFFFFFB, + 0xFFFFFFF7, + 0xFFFFFFEF, /* way 4 unlocked */ + 0xFFFFFFDF, + 0xFFFFFFBF, + 0xFFFFFF7F, + 0xFFFFFEFF, /* way 8 unlocked */ + 0xFFFFFDFF, + 0xFFFFFBFF, + 0xFFFFF7FF, + 0xFFFFEFFF, /* way 12 unlocked */ + 0xFFFFDFFF, + 0xFFFFBFFF, + 0xFFFF7FFF, +}; + +u32 nr_unlocked_way[MAX_NR_WAYS+1] = { + 0x0000FFFF, /* all ways are locked. usable = 0*/ + 0x0000FFFE, /* way ~0 unlocked. usable = 1 */ + 0x0000FFFC, + 0x0000FFF8, + 0x0000FFF0, + 0x0000FFE0, + 0x0000FFC0, + 0x0000FF80, + 0x0000FF00, + 0x0000FE00, + 0x0000FC00, + 0x0000F800, + 0x0000F000, + 0x0000E000, + 0x0000C000, + 0x00008000, + 0x00000000, /* way ~15 unlocked. usable = 16 */ +}; + +u32 way_partition[4] = { + 0xfffffff0, /* cpu0 */ + 0xffffff0f, /* cpu1 */ + 0xfffff0ff, /* cpu2 */ + 0xffff0fff, /* cpu3 */ +}; + +u32 way_partitions[9] = { + 0xffff0003, /* cpu0 A */ + 0xffff0003, /* cpu0 B */ + 0xffff000C, /* cpu1 A */ + 0xffff000C, /* cpu1 B */ + 0xffff0030, /* cpu2 A */ + 0xffff0030, /* cpu2 B */ + 0xffff00C0, /* cpu3 A */ + 0xffff00C0, /* cpu3 B */ + 0xffffff00, /* lv C */ +}; + +u32 prev_lockdown_d_reg[5] = { + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x000000FF, /* share with level-C */ +}; + +u32 prev_lockdown_i_reg[5] = { + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x0000FF00, + 0x000000FF, /* share with level-C */ +}; + +u32 prev_lbm_i_reg[8] = { + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + +u32 prev_lbm_d_reg[8] = { + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + +static void __iomem *cache_base; +static void __iomem *lockreg_d; +static void __iomem *lockreg_i; + +static u32 cache_id; + +struct mutex actlr_mutex; +struct mutex l2x0_prefetch_mutex; +struct mutex lockdown_proc; +static u32 way_partition_min; +static u32 way_partition_max; + +static int zero = 0; +static int one = 1; + +static int l1_prefetch_proc; +static int l2_prefetch_hint_proc; +static int l2_double_linefill_proc; +static int l2_data_prefetch_proc; +static int os_isolation; +static int use_part; + +u32 lockdown_reg[9] = { + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, + 0x00000000, +}; + + +#define ld_d_reg(cpu) ({ int __cpu = cpu; \ + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_D_BASE + \ + __cpu * L2X0_LOCKDOWN_STRIDE; __v; }) +#define ld_i_reg(cpu) ({ int __cpu = cpu; \ + void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_I_BASE + \ + __cpu * L2X0_LOCKDOWN_STRIDE; __v; }) + +int lock_all; +int nr_lockregs; +static raw_spinlock_t cache_lock; +static raw_spinlock_t prefetch_lock; +static void ***flusher_pages = NULL; + +extern void l2c310_flush_all(void); + +static inline void cache_wait_way(void __iomem *reg, unsigned long mask) +{ + /* wait for cache operation by line or way to complete */ + while (readl_relaxed(reg) & mask) + cpu_relax(); +} + +#ifdef CONFIG_CACHE_L2X0 +static inline void cache_wait(void __iomem *reg, unsigned long mask) +{ + /* cache operations by line are atomic on PL310 */ +} +#else +#define cache_wait cache_wait_way +#endif + +static inline void cache_sync(void) +{ + void __iomem *base = cache_base; + + writel_relaxed(0, base + L2X0_CACHE_SYNC); + cache_wait(base + L2X0_CACHE_SYNC, 1); +} + +static void print_lockdown_registers(int cpu) +{ + int i; + //for (i = 0; i < nr_lockregs; i++) { + for (i = 0; i < 4; i++) { + printk("P%d Lockdown Data CPU %2d: 0x%04x\n", cpu, + i, readl_relaxed(ld_d_reg(i))); + printk("P%d Lockdown Inst CPU %2d: 0x%04x\n", cpu, + i, readl_relaxed(ld_i_reg(i))); + } +} + +static void test_lockdown(void *ignore) +{ + int i, cpu; + + cpu = smp_processor_id(); + printk("Start lockdown test on CPU %d.\n", cpu); + + for (i = 0; i < nr_lockregs; i++) { + printk("CPU %2d data reg: 0x%8p\n", i, ld_d_reg(i)); + printk("CPU %2d inst reg: 0x%8p\n", i, ld_i_reg(i)); + } + + printk("Lockdown initial state:\n"); + print_lockdown_registers(cpu); + printk("---\n"); + + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(1, ld_d_reg(i)); + writel_relaxed(2, ld_i_reg(i)); + } + printk("Lockdown all data=1 instr=2:\n"); + print_lockdown_registers(cpu); + printk("---\n"); + + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed((1 << i), ld_d_reg(i)); + writel_relaxed(((1 << 8) >> i), ld_i_reg(i)); + } + printk("Lockdown varies:\n"); + print_lockdown_registers(cpu); + printk("---\n"); + + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(UNLOCK_ALL, ld_d_reg(i)); + writel_relaxed(UNLOCK_ALL, ld_i_reg(i)); + } + printk("Lockdown all zero:\n"); + print_lockdown_registers(cpu); + + printk("End lockdown test.\n"); +} + +void litmus_setup_lockdown(void __iomem *base, u32 id) +{ + cache_base = base; + cache_id = id; + lockreg_d = cache_base + L2X0_LOCKDOWN_WAY_D_BASE; + lockreg_i = cache_base + L2X0_LOCKDOWN_WAY_I_BASE; + + if (L2X0_CACHE_ID_PART_L310 == (cache_id & L2X0_CACHE_ID_PART_MASK)) { + nr_lockregs = 8; + } else { + printk("Unknown cache ID!\n"); + nr_lockregs = 1; + } + + mutex_init(&actlr_mutex); + mutex_init(&l2x0_prefetch_mutex); + mutex_init(&lockdown_proc); + raw_spin_lock_init(&cache_lock); + raw_spin_lock_init(&prefetch_lock); + + test_lockdown(NULL); +} + +int way_partition_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + unsigned long flags; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write) { + printk("Way-partition settings:\n"); + for (i = 0; i < 9; i++) { + printk("0x%08X\n", way_partitions[i]); + } + for (i = 0; i < 4; i++) { + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(~way_partitions[i*2], cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + } + + local_irq_save(flags); + print_lockdown_registers(smp_processor_id()); + l2c310_flush_all(); + local_irq_restore(flags); +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int lock_all_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + unsigned long flags; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write && lock_all == 1) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(0xFFFF, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } +/* + for (i = 0; i < nr_lockregs; i++) { + barrier(); + mem_lock(LOCK_ALL, i); + barrier(); + //writel_relaxed(nr_unlocked_way[0], ld_d_reg(i)); + //writel_relaxed(nr_unlocked_way[0], ld_i_reg(i)); + } +*/ + } + if (write && lock_all == 0) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(0x0, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } +/* + for (i = 0; i < nr_lockregs; i++) { + barrier(); + mem_lock(UNLOCK_ALL, i); + barrier(); + //writel_relaxed(nr_unlocked_way[16], ld_d_reg(i)); + //writel_relaxed(nr_unlocked_way[16], ld_i_reg(i)); + } +*/ + } + printk("LOCK_ALL HANDLER\n"); + local_irq_save(flags); + print_lockdown_registers(smp_processor_id()); + l2c310_flush_all(); + local_irq_restore(flags); +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +void cache_lockdown(u32 lock_val, int cpu) +{ + //unsigned long flags; + //raw_spin_lock_irqsave(&cache_lock, flags); + + __asm__ __volatile__ ( +" str %[lockval], [%[dcachereg]]\n" +" str %[lockval], [%[icachereg]]\n" + : + : [dcachereg] "r" (ld_d_reg(cpu)), + [icachereg] "r" (ld_i_reg(cpu)), + [lockval] "r" (lock_val) + : "cc"); + + //raw_spin_unlock_irqrestore(&cache_lock, flags); +} + +void do_partition(enum crit_level lv, int cpu) +{ + u32 regs; + unsigned long flags; + + if (lock_all || !use_part) + return; + raw_spin_lock_irqsave(&cache_lock, flags); + switch(lv) { + case CRIT_LEVEL_A: + regs = ~way_partitions[cpu*2]; + regs &= 0x0000ffff; + break; + case CRIT_LEVEL_B: + regs = ~way_partitions[cpu*2+1]; + regs &= 0x0000ffff; + break; + case CRIT_LEVEL_C: + case NUM_CRIT_LEVELS: + regs = ~way_partitions[8]; + regs &= 0x0000ffff; + break; + case MODE_POLL_TASK: + regs = 0x0000ffff; + break; + default: + BUG(); + + } + barrier(); + //cache_lockdown(regs, cpu); + writel_relaxed(regs, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + cpu * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(regs, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + cpu * L2X0_LOCKDOWN_STRIDE); + barrier(); + + raw_spin_unlock_irqrestore(&cache_lock, flags); + + //flush_cache(0); +} + +void lock_cache(int cpu, u32 val) +{ + unsigned long flags; + + local_irq_save(flags); + if (val != 0xffffffff) { + writel_relaxed(val, cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + cpu * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(val, cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + cpu * L2X0_LOCKDOWN_STRIDE); + } + else { + int i; + for (i = 0; i < 4; i++) + do_partition(CRIT_LEVEL_A, i); + } + local_irq_restore(flags); +} + +int use_part_proc_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + + printk("USE_PART HANDLER = %d\n", use_part); + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int os_isolation_proc_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + + printk("OS_ISOLATION HANDLER = %d\n", os_isolation); + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int lockdown_reg_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(lockdown_reg[i], cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + } + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +int lockdown_global_handler(struct ctl_table *table, int write, void __user *buffer, + size_t *lenp, loff_t *ppos) +{ + int ret = 0, i; + + mutex_lock(&lockdown_proc); + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret) + goto out; + + if (write) { + for (i = 0; i < nr_lockregs; i++) { + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_D_BASE + + i * L2X0_LOCKDOWN_STRIDE); + writel_relaxed(lockdown_reg[8], cache_base + L2X0_LOCKDOWN_WAY_I_BASE + + i * L2X0_LOCKDOWN_STRIDE); + } + } + +out: + mutex_unlock(&lockdown_proc); + return ret; +} + +void inline enter_irq_mode(void) +{ + int cpu = smp_processor_id(); + + if (os_isolation == 0) + return; + + prev_lockdown_i_reg[cpu] = readl_relaxed(ld_i_reg(cpu)); + prev_lockdown_d_reg[cpu] = readl_relaxed(ld_d_reg(cpu)); + + writel_relaxed(way_partitions[8], ld_i_reg(cpu)); + writel_relaxed(way_partitions[8], ld_d_reg(cpu)); +} + +void inline exit_irq_mode(void) +{ + int cpu = smp_processor_id(); + + if (os_isolation == 0) + return; + writel_relaxed(prev_lockdown_i_reg[cpu], ld_i_reg(cpu)); + writel_relaxed(prev_lockdown_d_reg[cpu], ld_d_reg(cpu)); +} + +/* Operate on the Cortex-A9's ACTLR register */ +#define ACTLR_L2_PREFETCH_HINT (1 << 1) +#define ACTLR_L1_PREFETCH (1 << 2) + +/* + * Change the ACTLR. + * @mode - If 1 (0), set (clear) the bit given in @mask in the ACTLR. + * @mask - A mask in which one bit is set to operate on the ACTLR. + */ +static void actlr_change(int mode, int mask) +{ + u32 orig_value, new_value, reread_value; + + if (0 != mode && 1 != mode) { + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n", + __FUNCTION__); + return; + } + + /* get the original value */ + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (orig_value)); + + if (0 == mode) + new_value = orig_value & ~(mask); + else + new_value = orig_value | mask; + + asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (new_value)); + asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (reread_value)); + + printk("ACTLR: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n", + orig_value, new_value, reread_value); +} + +int litmus_l1_prefetch_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&actlr_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + + if (!ret && write) { + mode = *((int*)table->data); + actlr_change(mode, ACTLR_L1_PREFETCH); + } + mutex_unlock(&actlr_mutex); + + return ret; +} + +int litmus_l2_prefetch_hint_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&actlr_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + mode = *((int*)table->data); + actlr_change(mode, ACTLR_L2_PREFETCH_HINT); + } + mutex_unlock(&actlr_mutex); + + return ret; +} + + +/* Operate on the PL-310's Prefetch Control Register, L310_PREFETCH_CTRL */ +#define L2X0_PREFETCH_DOUBLE_LINEFILL (1 << 30) +#define L2X0_PREFETCH_INST_PREFETCH (1 << 29) +#define L2X0_PREFETCH_DATA_PREFETCH (1 << 28) +static void l2x0_prefetch_change(int mode, int mask) +{ + u32 orig_value, new_value, reread_value; + + if (0 != mode && 1 != mode) { + printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n", + __FUNCTION__); + return; + } + + orig_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL); + + if (0 == mode) + new_value = orig_value & ~(mask); + else + new_value = orig_value | mask; + + writel_relaxed(new_value, cache_base + L310_PREFETCH_CTRL); + reread_value = readl_relaxed(cache_base + L310_PREFETCH_CTRL); + + printk("l2x0 prefetch: orig: 0x%8x wanted: 0x%8x new: 0x%8x\n", + orig_value, new_value, reread_value); +} + +int litmus_l2_double_linefill_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&l2x0_prefetch_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + mode = *((int*)table->data); + l2x0_prefetch_change(mode, L2X0_PREFETCH_DOUBLE_LINEFILL); + } + mutex_unlock(&l2x0_prefetch_mutex); + + return ret; +} + +int litmus_l2_data_prefetch_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret, mode; + + mutex_lock(&l2x0_prefetch_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (!ret && write) { + mode = *((int*)table->data); + l2x0_prefetch_change(mode, L2X0_PREFETCH_DATA_PREFETCH|L2X0_PREFETCH_INST_PREFETCH); + } + mutex_unlock(&l2x0_prefetch_mutex); + + return ret; +} + +int do_perf_test_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + +int setup_flusher_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + +static struct ctl_table cache_table[] = +{ + { + .procname = "C0_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[0], + .maxlen = sizeof(way_partitions[0]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C0_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[1], + .maxlen = sizeof(way_partitions[1]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C1_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[2], + .maxlen = sizeof(way_partitions[2]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C1_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[3], + .maxlen = sizeof(way_partitions[3]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C2_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[4], + .maxlen = sizeof(way_partitions[4]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C2_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[5], + .maxlen = sizeof(way_partitions[5]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C3_LA_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[6], + .maxlen = sizeof(way_partitions[6]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "C3_LB_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[7], + .maxlen = sizeof(way_partitions[7]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "Call_LC_way", + .mode = 0666, + .proc_handler = way_partition_handler, + .data = &way_partitions[8], + .maxlen = sizeof(way_partitions[8]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lock_all", + .mode = 0666, + .proc_handler = lock_all_handler, + .data = &lock_all, + .maxlen = sizeof(lock_all), + .extra1 = &zero, + .extra2 = &one, + }, + { + .procname = "l1_prefetch", + .mode = 0644, + .proc_handler = litmus_l1_prefetch_proc_handler, + .data = &l1_prefetch_proc, + .maxlen = sizeof(l1_prefetch_proc), + }, + { + .procname = "l2_prefetch_hint", + .mode = 0644, + .proc_handler = litmus_l2_prefetch_hint_proc_handler, + .data = &l2_prefetch_hint_proc, + .maxlen = sizeof(l2_prefetch_hint_proc), + }, + { + .procname = "l2_double_linefill", + .mode = 0644, + .proc_handler = litmus_l2_double_linefill_proc_handler, + .data = &l2_double_linefill_proc, + .maxlen = sizeof(l2_double_linefill_proc), + }, + { + .procname = "l2_data_prefetch", + .mode = 0644, + .proc_handler = litmus_l2_data_prefetch_proc_handler, + .data = &l2_data_prefetch_proc, + .maxlen = sizeof(l2_data_prefetch_proc), + }, + { + .procname = "os_isolation", + .mode = 0644, + .proc_handler = os_isolation_proc_handler, + .data = &os_isolation, + .maxlen = sizeof(os_isolation), + }, + { + .procname = "use_part", + .mode = 0644, + .proc_handler = use_part_proc_handler, + .data = &use_part, + .maxlen = sizeof(use_part), + }, + { + .procname = "do_perf_test", + .mode = 0644, + .proc_handler = do_perf_test_proc_handler, + }, + { + .procname = "setup_flusher", + .mode = 0644, + .proc_handler = setup_flusher_proc_handler, + }, + { + .procname = "lockdown_reg_0", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[0], + .maxlen = sizeof(lockdown_reg[0]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_reg_1", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[1], + .maxlen = sizeof(lockdown_reg[1]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_reg_2", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[2], + .maxlen = sizeof(lockdown_reg[2]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_reg_3", + .mode = 0644, + .proc_handler = lockdown_reg_handler, + .data = &lockdown_reg[3], + .maxlen = sizeof(lockdown_reg[3]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { + .procname = "lockdown_regs", + .mode = 0644, + .proc_handler = lockdown_global_handler, + .data = &lockdown_reg[8], + .maxlen = sizeof(lockdown_reg[8]), + .extra1 = &way_partition_min, + .extra2 = &way_partition_max, + }, + { } +}; + +static struct ctl_table litmus_dir_table[] = { + { + .procname = "litmus", + .mode = 0555, + .child = cache_table, + }, + { } +}; + +u32 color_read_in_mem(u32 lock_val, u32 unlock_val, void *start, void *end) +{ + u32 v = 0; + + __asm__ __volatile__ ( +" .align 5\n" +" str %[lockval], [%[cachereg]]\n" +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n" +" cmp %[end], %[addr] @ subtracts addr from end\n" +" bgt 1b\n @ read more, if necessary\n" + : [addr] "+r" (start), + [val] "+r" (v) + : [end] "r" (end), +#ifdef CONFIG_CACHE_L2X0 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())), +#else + [cachereg] "r" (lockreg_d), +#endif + [lockval] "r" (lock_val) + : "cc"); + + return v; +} + + +/* + * Prefetch by reading the first word of each cache line in a page. + * + * @lockdown_reg: address of the lockdown register to write + * @lock_val: value to be written to @lockdown_reg + * @unlock_val: will unlock the cache to this value + * @addr: start address to be prefetched + * @end_addr: end address to prefetch (exclusive) + * + * Assumes: addr < end_addr AND addr != end_addr + */ +u32 color_read_in_mem_lock(u32 lock_val, u32 unlock_val, void *start, void *end) +{ +#ifndef CONFIG_CACHE_L2X0 + unsigned long flags; +#endif + u32 v = 0; + +#ifndef CONFIG_CACHE_L2X0 + raw_spin_lock_irqsave(&prefetch_lock, flags); +#endif + + __asm__ __volatile__ ( +" .align 5\n" +" str %[lockval], [%[cachereg]]\n" +"1: ldr %[val], [%[addr]], #32 @ 32 bytes = 1 cache line\n" +" cmp %[end], %[addr] @ subtracts addr from end\n" +" bgt 1b\n @ read more, if necessary\n" +" str %[unlockval], [%[cachereg]]\n" + : [addr] "+r" (start), + [val] "+r" (v) + : [end] "r" (end), +#ifdef CONFIG_CACHE_L2X0 + [cachereg] "r" (ld_d_reg(raw_smp_processor_id())), +#else + [cachereg] "r" (lockreg_d), +#endif + [lockval] "r" (lock_val), + [unlockval] "r" (unlock_val) + : "cc"); + +#ifndef CONFIG_CACHE_L2X0 + raw_spin_unlock_irqrestore(&prefetch_lock, flags); +#endif + + return v; +} + +static long update_timeval(struct timespec lhs, struct timespec rhs) +{ + long val; + struct timespec ts; + + ts = timespec_sub(rhs, lhs); + val = ts.tv_sec*NSEC_PER_SEC + ts.tv_nsec; + + return val; +} + +extern void v7_flush_kern_dcache_area(void *, size_t); +extern void v7_flush_kern_cache_all(void); +/* + * Ensure that this page is not in the L1 or L2 cache. + * Since the L1 cache is VIPT and the L2 cache is PIPT, we can use either the + * kernel or user vaddr. + */ +void color_flush_page(void *vaddr, size_t size) +{ + v7_flush_kern_dcache_area(vaddr, size); + //v7_flush_kern_cache_all(); +} + +extern struct page* get_colored_page(unsigned long color); + +int setup_flusher_array(void) +{ + int color, way, ret = 0; + struct page *page; + + if (flusher_pages != NULL) + goto out; + + flusher_pages = (void***) kmalloc(MAX_NR_WAYS + * sizeof(*flusher_pages), GFP_KERNEL); + if (!flusher_pages) { + printk(KERN_WARNING "No memory for flusher array!\n"); + ret = -EINVAL; + goto out; + } + + for (way = 0; way < MAX_NR_WAYS; way++) { + void **flusher_color_arr; + flusher_color_arr = (void**) kmalloc(sizeof(**flusher_pages) + * MAX_NR_COLORS, GFP_KERNEL); + if (!flusher_color_arr) { + printk(KERN_WARNING "No memory for flusher array!\n"); + ret = -ENOMEM; + goto out_free; + } + + flusher_pages[way] = flusher_color_arr; + + for (color = 0; color < MAX_NR_COLORS; color++) { + int node; + switch (color) { + case 0: + node = 48; + break; + case 1: + node = 49; + break; + case 2: + node = 50; + break; + case 3: + node = 51; + break; + case 4: + node = 68; + break; + case 5: + node = 69; + break; + case 6: + node = 86; + break; + case 7: + node = 87; + break; + case 8: + node = 88; + break; + case 9: + node = 105; + break; + case 10: + node = 106; + break; + case 11: + node = 107; + break; + case 12: + node = 108; + break; + case 13: + node = 125; + break; + case 14: + node = 126; + break; + case 15: + node = 127; + break; + } + page = get_colored_page(node); + if (!page) { + printk(KERN_WARNING "no more colored pages\n"); + ret = -EINVAL; + goto out_free; + } + flusher_pages[way][color] = page_address(page); + if (!flusher_pages[way][color]) { + printk(KERN_WARNING "bad page address\n"); + ret = -EINVAL; + goto out_free; + } + } + } +out: + return ret; +out_free: + for (way = 0; way < MAX_NR_WAYS; way++) { + for (color = 0; color < MAX_NR_COLORS; color++) { + /* not bothering to try and give back colored pages */ + } + kfree(flusher_pages[way]); + } + kfree(flusher_pages); + flusher_pages = NULL; + return ret; +} + +void flush_cache2(int all) +{ + int way, color, cpu; + unsigned long flags; + + raw_spin_lock_irqsave(&cache_lock, flags); + cpu = raw_smp_processor_id(); + + prev_lbm_i_reg[cpu] = readl_relaxed(ld_i_reg(cpu)); + prev_lbm_d_reg[cpu] = readl_relaxed(ld_d_reg(cpu)); + for (way=0;way +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define DEV_NAME "litmus/color_shm" + +/* Major number assigned to our device. + * Refer Documentation/devices.txt */ +#define SHM_MAJOR 240 +#define MAX_COLORED_PAGE 256 +#define NUM_BANKS 8 +#define NUM_COLORS 16 + +static struct mutex dev_lock; +static int bypass_cache; + +struct color_ioctl_cmd { + unsigned int color; + unsigned int bank; +}; + +struct color_ioctl_offset { + unsigned long offset; + int lock; +}; + +#define SET_COLOR_SHM_CMD _IOW(SHM_MAJOR, 0x1, struct color_ioctl_cmd) +#define SET_COLOR_SHM_OFFSET _IOW(SHM_MAJOR, 0x2, struct color_ioctl_offset) + +struct color_ioctl_cmd color_param; +struct color_ioctl_offset color_offset; + +static int mmap_common_checks(struct vm_area_struct *vma) +{ + /* you can only map the "first" page */ + if (vma->vm_pgoff != 0) + return -EINVAL; + + return 0; +} + +static void mmap_common_vma_flags(struct vm_area_struct *vma) +{ + /* This mapping should not be kept across forks, + * cannot be expanded, and is not a "normal" page. */ + //vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_IO | VM_SHARED | VM_MAYSHARE; + vma->vm_flags |= VM_SHARED | VM_MAYSHARE | VM_LOCKED; + + /* We don't want the first write access to trigger a "minor" page fault + * to mark the page as dirty. This is transient, private memory, we + * don't care if it was touched or not. __S011 means RW access, but not + * execute, and avoids copy-on-write behavior. + * See protection_map in mmap.c. */ + vma->vm_page_prot = PAGE_SHARED; +} + +#define vma_nr_pages(vma) \ + ({unsigned long v = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT); v;}) + +extern struct page* get_colored_page(unsigned long color); + +static int do_map_colored_page(struct vm_area_struct *vma, + const unsigned long addr, + const unsigned long color_no) +{ + int err = 0; + + struct page *page = get_colored_page(color_no); + + if (!page) { + printk(KERN_INFO "Could not get page with color %lu.\n", + color_no); + err = -ENOMEM; + goto out; + } + + printk(KERN_INFO "vma: %p addr: 0x%lx color_no: %lu\n", + vma, addr, color_no); + + printk(KERN_INFO "vm_start: %lu vm_end: %lu\n", + vma->vm_start, vma->vm_end); + + printk(KERN_INFO "inserting page (pa: 0x%x) at vaddr: 0x%lx " + "flags: 0x%lx prot: 0x%x\n", + page_to_phys(page), addr, + vma->vm_flags, pgprot_val(vma->vm_page_prot)); + + + err = vm_insert_page(vma, addr, page); + if (err) { + printk(KERN_INFO "vm_insert_page() failed (%d)\n", err); + err = -EINVAL; + goto out; + } +out: + return err; +} + +static int do_map_colored_pages(struct vm_area_struct *vma) +{ + const unsigned long nr_pages = vma_nr_pages(vma); + unsigned long nr_mapped; + int i, start_bank = -1, start_color = -1; + int cur_bank = -1, cur_color = -1, err = 0; + int colors[16] = {0}, banks[8] = {0}; + + if (bypass_cache == 1) + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + for (i = 0; i < NUM_BANKS; i++) { + if (((color_param.bank >> i)&0x1) == 1) + banks[i] = 1; + } + + for (i = 0; i < NUM_COLORS; i++) { + if (((color_param.color >> i)&0x1) == 1) + colors[i] = 1; + } + + for (i = 0; i < NUM_BANKS; i++) { + if (banks[i] == 1) { + start_bank = i; + break; + } + } + for (i = 0; i < NUM_COLORS; i++) { + if (colors[i] == 1) { + start_color = i; + break; + } + } + + cur_bank = start_bank; + cur_color = start_color; + + for (i = 0; i < NUM_BANKS; i++) { + printk(KERN_INFO "BANK[%d] = %d\n", i, banks[i]); + } + printk(KERN_INFO "cur_bank = %d\n", cur_bank); + for (i = 0; i < NUM_COLORS; i++) { + printk(KERN_INFO "COLOR[%d] = %d\n", i, colors[i]); + } + printk(KERN_INFO "cur_color = %d\n", cur_color); + + + TRACE_CUR("allocating %lu pages (flags:%lx prot:%lx)\n", + nr_pages, vma->vm_flags, pgprot_val(vma->vm_page_prot)); + + for (nr_mapped = 0; nr_mapped < nr_pages; nr_mapped++) { + const unsigned long addr = vma->vm_start + (nr_mapped << PAGE_SHIFT); + const unsigned long color_no = cur_bank*NUM_COLORS + cur_color; + + err = do_map_colored_page(vma, addr, color_no); + printk(KERN_INFO "mapped bank[%d], color[%d], color_no = %lu at 0x%lx\n", + cur_bank, cur_color, color_no, addr); + if (err) { + TRACE_CUR("Could not map colored page set.\n"); + err = -EINVAL; + goto out; + } + do { + cur_color++; + } while(colors[cur_color] == 0); + + if (cur_color >= NUM_COLORS) { + do { + cur_bank++; + } while(banks[cur_bank] == 0); + cur_color = start_color; + } + + if (cur_bank >= NUM_BANKS) { + cur_bank = start_bank; + } + } + TRACE_CUR("Successfully mapped %lu pages.\n", nr_mapped); + out: + return err; +} + +static int map_colored_pages(struct vm_area_struct *vma) +{ + int err = 0; + + printk(KERN_INFO "User requests %lu pages.\n", vma_nr_pages(vma)); + if (MAX_COLORED_PAGE < vma_nr_pages(vma)) { + TRACE_CUR("Max page request %lu but want %lu.\n", + MAX_COLORED_PAGE, vma_nr_pages(vma)); + err = -EINVAL; + goto out; + } + err = do_map_colored_pages(vma); +out: + return err; +} + +static void litmus_color_shm_vm_close(struct vm_area_struct *vma) +{ + + TRACE_CUR("flags=0x%lx prot=0x%lx\n", + vma->vm_flags, pgprot_val(vma->vm_page_prot)); + + TRACE_CUR("%p:%p vma:%p vma->vm_private_data:%p closed.\n", + (void*) vma->vm_start, (void*) vma->vm_end, vma, + vma->vm_private_data); + +} + +static int litmus_color_shm_vm_fault(struct vm_area_struct *vma, + struct vm_fault *vmf) +{ + /* This function should never be called, since + * all pages should have been mapped by mmap() + * already. */ + TRACE_CUR("flags=0x%lx (off:%ld)\n", vma->vm_flags, vmf->pgoff); + printk(KERN_INFO "flags=0x%lx (off:%ld)\n", vma->vm_flags, vmf->pgoff); + + printk(KERN_INFO "Page fault in color ctrl page! prot=0x%x\n", pgprot_val(vma->vm_page_prot)); + + return VM_FAULT_SIGBUS; +} + +static struct vm_operations_struct litmus_color_shm_vm_ops = { + .close = litmus_color_shm_vm_close, + .fault = litmus_color_shm_vm_fault, +}; + +static int litmus_color_shm_mmap(struct file *filp, struct vm_area_struct *vma) +{ + int err = 0; + + printk(KERN_INFO "mmap called\n"); + + if (color_param.color == 0x00000000 || color_param.bank == 0x00000000) { + printk(KERN_INFO "color_info not set.\n"); + return -EINVAL; + } + if (color_offset.offset == 0xffffffff || color_offset.lock == -1) { + printk(KERN_INFO "color_offset not set.\n"); + return -EINVAL; + } + + err = mmap_common_checks(vma); + if (err) { + TRACE_CUR("failed mmap common checks\n"); + goto out; + } + + vma->vm_ops = &litmus_color_shm_vm_ops; + mmap_common_vma_flags(vma); + + err = map_colored_pages(vma); + + TRACE_CUR("flags=0x%lx prot=0x%lx\n", vma->vm_flags, + pgprot_val(vma->vm_page_prot)); +out: + color_param.color = 0x00000000; + color_param.bank = 0x00000000; + color_offset.offset = 0xffffffff; + color_offset.lock = -1; + + return err; + +} + +static long litmus_color_shm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + long err = -ENOIOCTLCMD; + struct color_ioctl_cmd color_info; + struct color_ioctl_offset color_off; + + printk(KERN_INFO "color_shm ioctl\n"); + + if (_IOC_TYPE(cmd) != SHM_MAJOR) + return -ENOTTY; + + + switch (cmd) { + case SET_COLOR_SHM_CMD: + + err = copy_from_user(&color_info, (void*)arg, sizeof(struct color_ioctl_cmd)); + + color_param.color = color_info.color; + color_param.bank = color_info.bank; + printk(KERN_INFO "COLOR = %x\n", color_param.color); + printk(KERN_INFO "BANK = %x\n", color_param.bank); + err = 0; + break; + case SET_COLOR_SHM_OFFSET: + err = copy_from_user(&color_off, (void*)arg, sizeof(struct color_ioctl_offset)); + + color_offset.offset = color_off.offset; + color_offset.lock = color_off.lock; + printk(KERN_INFO "OFFSET = %lx\n", color_offset.offset); + printk(KERN_INFO "LOCK = %d\n", color_offset.lock); + err = 0; + break; + + default: + printk(KERN_INFO "Invalid IOCTL CMD\n"); + err = -EINVAL; + } + + return err; +} + +static struct file_operations litmus_color_shm_fops = { + .owner = THIS_MODULE, + .mmap = litmus_color_shm_mmap, + .unlocked_ioctl = litmus_color_shm_ioctl, +}; + +static struct miscdevice litmus_color_shm_dev = { + .name = DEV_NAME, + .minor = MISC_DYNAMIC_MINOR, + .fops = &litmus_color_shm_fops, +}; + +struct mutex bypass_mutex; + +int bypass_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + int ret; + + mutex_lock(&bypass_mutex); + ret = proc_dointvec(table, write, buffer, lenp, ppos); + printk(KERN_INFO "shm_bypass = %d\n", bypass_cache); + mutex_unlock(&bypass_mutex); + + return ret; +} + +static int zero = 0; +static int one = 1; + +static struct ctl_table cache_table[] = +{ + { + .procname = "shm_bypass", + .mode = 0666, + .proc_handler = bypass_proc_handler, + .data = &bypass_cache, + .maxlen = sizeof(bypass_cache), + .extra1 = &zero, + .extra2 = &one, + }, + { } +}; + +static struct ctl_table litmus_dir_table[] = { + { + .procname = "litmus", + .mode = 0555, + .child = cache_table, + }, + { } +}; + +static struct ctl_table_header *litmus_sysctls; + +static int __init init_color_shm_devices(void) +{ + int err; + + printk(KERN_INFO "Registering LITMUS^RT color_shm devices.\n"); + litmus_sysctls = register_sysctl_table(litmus_dir_table); + if (!litmus_sysctls) { + printk(KERN_WARNING "Could not register LITMUS^RT color_shm sysctl.\n"); + err = -EFAULT; + } + + mutex_init(&dev_lock); + mutex_init(&bypass_mutex); + color_param.color = 0x00000000; + color_param.bank = 0x00000000; + color_offset.offset = 0xffffffff; + color_offset.lock = -1; + bypass_cache = 0; + err = misc_register(&litmus_color_shm_dev); + + return err; +} + +static void __exit exit_color_shm_devices(void) +{ + misc_deregister(&litmus_color_shm_dev); + printk(KERN_INFO "Unregistering %s device.\n", DEV_NAME); +} + +module_init(init_color_shm_devices); +module_exit(exit_color_shm_devices); \ No newline at end of file diff --git litmus/jobs.c litmus/jobs.c index 0dd36b9..368e0b3 100644 --- litmus/jobs.c +++ litmus/jobs.c @@ -8,6 +8,9 @@ #include #include +extern int num_sync_released; +int n_init_phase; + static inline void setup_release(struct task_struct *t, lt_t release) { /* prepare next release */ @@ -19,6 +22,8 @@ static inline void setup_release(struct task_struct *t, lt_t release) t->rt_param.job_params.job_no++; } +#define INIT_PHASE_LENGTH_NS (1000000000) + void prepare_for_next_period(struct task_struct *t) { BUG_ON(!t); @@ -30,12 +35,13 @@ void prepare_for_next_period(struct task_struct *t) (long long)litmus_clock() - (long long)t->rt_param.job_params.deadline; + /* Mode 0 is used for initializations * + * Use sporadic releases for all tasks not to overutilize cpus in mode 0 */ if (tsk_rt(t)->sporadic_release) { TRACE_TASK(t, "sporadic release at %llu\n", - tsk_rt(t)->sporadic_release_time); + tsk_rt(t)->sporadic_release_time + INIT_PHASE_LENGTH_NS*(tsk_rt(t)->job_params.job_no)); /* sporadic release */ - setup_release(t, tsk_rt(t)->sporadic_release_time); - tsk_rt(t)->sporadic_release = 0; + setup_release(t, tsk_rt(t)->sporadic_release_time + INIT_PHASE_LENGTH_NS*(tsk_rt(t)->job_params.job_no)); } else { /* periodic release => add period */ setup_release(t, get_release(t) + get_rt_period(t)); diff --git litmus/litmus.c litmus/litmus.c index db5ce0e9..400fd14 100644 --- litmus/litmus.c +++ litmus/litmus.c @@ -14,6 +14,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -21,6 +25,8 @@ #include #include #include +#include +#include #ifdef CONFIG_SCHED_CPU_AFFINITY #include @@ -31,6 +37,8 @@ #include #endif +extern void l2c310_flush_all(void); + /* Number of RT tasks that exist in the system */ atomic_t rt_task_count = ATOMIC_INIT(0); @@ -160,6 +168,14 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param) pid, tp.budget_policy); goto out_unlock; } +#ifdef CONFIG_PGMRT_SUPPORT + if (tp.pgm_type < PGM_NOT_A_NODE || tp.pgm_type > PGM_INTERNAL) { + printk(KERN_INFO "litmus: real-time task %d rejected " + "because of unknown PGM node type specified (%d)\n", + pid, tp.pgm_type); + goto out_unlock; + } +#endif target->rt_param.task_params = tp; @@ -314,6 +330,149 @@ asmlinkage long sys_null_call(cycles_t __user *ts) return ret; } +asmlinkage long sys_reservation_create(int type, void __user *config) +{ + return litmus->reservation_create(type, config); +} + +asmlinkage long sys_reservation_destroy(unsigned int reservation_id, int cpu) +{ + return litmus->reservation_destroy(reservation_id, cpu); +} + +static unsigned long color_mask; + +static inline unsigned long page_color(struct page *page) +{ + return ((page_to_phys(page) & color_mask) >> PAGE_SHIFT); +} + +extern int isolate_lru_page(struct page *page); +extern void putback_movable_page(struct page *page); +extern struct page *new_alloc_page(struct page *page, unsigned long node, int **x); + +/* + * sys_set_page_color + * @cpu: CPU number to assign page colors. + * Syscall for recoloring pages + * Returns -1 on error. + * N on success. N is the number of pages that could not + * be moved. A return of zero means that all pages + * were successfully moved. Currently, two pages + * cannot be moved, signal handler and litmus ctrl + * pages. + * Only mc2 tasks may be configured with this system call. + * Use static linking to isolate all pages. + */ +asmlinkage long sys_set_page_color(int cpu) +{ + long ret = 0; + struct vm_area_struct *vma_itr = NULL; + int nr_pages = 0, nr_failed = 0, nr_not_migrated = 0; + unsigned long node; + enum crit_level lv; + struct mm_struct *mm; + + LIST_HEAD(pagelist); + LIST_HEAD(shared_pagelist); + + migrate_prep(); + + rcu_read_lock(); + get_task_struct(current); + rcu_read_unlock(); + mm = get_task_mm(current); + put_task_struct(current); + + down_read(&mm->mmap_sem); + vma_itr = mm->mmap; + while (vma_itr != NULL) { + unsigned int num_pages = 0, i; + struct page *old_page = NULL; + + num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE; + for (i = 0; i < num_pages; i++) { + old_page = follow_page(vma_itr, vma_itr->vm_start + PAGE_SIZE*i, FOLL_GET|FOLL_SPLIT); + + if (IS_ERR(old_page)) + continue; + if (!old_page) + continue; + + if (PageReserved(old_page)) { + TRACE("Reserved Page!\n"); + put_page(old_page); + continue; + } + ret = isolate_lru_page(old_page); + if (!ret) { + list_add_tail(&old_page->lru, &pagelist); + inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page)); + nr_pages++; + } else { + nr_failed++; + } + put_page(old_page); + } + + vma_itr = vma_itr->vm_next; + } + + ret = 0; + if (tsk_rt(current)->mc2_data) + lv = tsk_rt(current)->mc2_data->crit; + else + BUG(); //lv = 0; + + if (cpu == -1) + node = 8; + else + node = cpu*2 + lv; + + if (!list_empty(&pagelist)) { + ret = migrate_pages(&pagelist, new_alloc_page, NULL, node, MIGRATE_SYNC, MR_SYSCALL); + TRACE_TASK(current, "%ld pages not migrated.\n", ret); + nr_not_migrated = ret; + if (ret) { + putback_movable_pages(&pagelist); + } + } + + up_read(&mm->mmap_sem); + + TRACE_TASK(current, "node = %ld, nr_migrated_pages = %d, nr_pages = %d nr_failed = %d\n", node, nr_pages-nr_not_migrated, nr_pages, nr_failed); + + return ret; +} + +/* sys_test_call() is a test system call for debugging */ +asmlinkage long sys_test_call(unsigned int param) +{ + long ret = 0; + struct vm_area_struct *vma_itr = NULL; + + TRACE_CUR("test_call param = %d\n", param); + + if (param == 0) { + /* Print page information */ + down_read(¤t->mm->mmap_sem); + vma_itr = current->mm->mmap; + while (vma_itr != NULL) { + printk(KERN_INFO "--------------------------------------------\n"); + printk(KERN_INFO "vm_start : %lx\n", vma_itr->vm_start); + printk(KERN_INFO "vm_end : %lx\n", vma_itr->vm_end); + printk(KERN_INFO "vm_flags : %lx\n", vma_itr->vm_flags); + printk(KERN_INFO "vm_prot : %x\n", pgprot_val(vma_itr->vm_page_prot)); + printk(KERN_INFO "VM_SHARED? %ld\n", vma_itr->vm_flags & VM_SHARED); + vma_itr = vma_itr->vm_next; + } + printk(KERN_INFO "--------------------------------------------\n"); + up_read(¤t->mm->mmap_sem); + } + + return ret; +} + /* p is a real-time task. Re-init its state as a best-effort task. */ static void reinit_litmus_state(struct task_struct* p, int restore) { @@ -651,6 +810,12 @@ static int __init _init_litmus(void) * mode change lock is used to enforce single mode change * operation. */ +#if defined(CONFIG_CPU_V7) + unsigned int line_size_log = 5; // 2^5 = 32 byte + unsigned int cache_info_sets = 2048; // 64KB (way_size) / 32B (line_size) = 2048 + printk("LITMIS^RT-ARM kernel\n"); +#endif + printk("Starting LITMUS^RT kernel\n"); register_sched_plugin(&linux_sched_plugin); @@ -665,11 +830,15 @@ static int __init _init_litmus(void) else printk("Could not register kill rt tasks magic sysrq.\n"); #endif - init_litmus_proc(); register_reboot_notifier(&shutdown_notifier); +#if defined(CONFIG_CPU_V7) + color_mask = ((cache_info_sets << line_size_log) - 1) ^ (PAGE_SIZE - 1); + printk("Page color mask %lx\n", color_mask); +#endif + return 0; } diff --git litmus/mc2_common.c litmus/mc2_common.c new file mode 100644 index 0000000..322cc6d --- /dev/null +++ litmus/mc2_common.c @@ -0,0 +1,80 @@ +/* + * litmus/mc2_common.c + * + * Common functions for MC2 plugin. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +long mc2_task_client_init(struct task_client *tc, struct mc2_task *mc2_param, struct task_struct *tsk, struct reservation *res) +{ + task_client_init(tc, tsk, res); + if ((mc2_param->crit < CRIT_LEVEL_A) || + (mc2_param->crit > CRIT_LEVEL_C)) + return -EINVAL; + + TRACE_TASK(tsk, "mc2_task_client_init: mode = %d, crit_level = %d\n", res->mode, mc2_param->crit); + + return 0; +} + +asmlinkage long sys_set_mc2_task_param(pid_t pid, struct mc2_task __user * param) +{ + struct task_struct *target; + int retval = -EINVAL; + struct mc2_task *mp = kzalloc(sizeof(*mp), GFP_KERNEL); + + if (!mp) + return -ENOMEM; + + printk("Setting up mc^2 task parameters for process %d.\n", pid); + + if (pid < 0 || param == 0) { + goto out; + } + if (copy_from_user(mp, param, sizeof(*mp))) { + retval = -EFAULT; + goto out; + } + + /* Task search and manipulation must be protected */ + read_lock_irq(&tasklist_lock); + if (!(target = find_task_by_vpid(pid))) { + retval = -ESRCH; + goto out_unlock; + } + + if (is_realtime(target)) { + /* The task is already a real-time task. + * We cannot not allow parameter changes at this point. + */ + retval = -EBUSY; + goto out_unlock; + } + if (mp->crit < CRIT_LEVEL_A || mp->crit >= NUM_CRIT_LEVELS) { + printk(KERN_INFO "litmus: real-time task %d rejected " + "because of invalid criticality level\n", pid); + goto out_unlock; + } + + mp->init_finished = 0; + //target->rt_param.plugin_state = mp; + target->rt_param.mc2_data = mp; + + retval = 0; + +out_unlock: + read_unlock_irq(&tasklist_lock); +out: + return retval; +} \ No newline at end of file diff --git litmus/polling_reservations.c litmus/polling_reservations.c new file mode 100644 index 0000000..c6e10eb --- /dev/null +++ litmus/polling_reservations.c @@ -0,0 +1,566 @@ +#include + +#include +#include +#include + +//#define TRACE(fmt, args...) do {} while (false) +//#define TRACE_TASK(fmt, args...) do {} while (false) + +static void periodic_polling_client_arrives( + struct reservation* res, + struct reservation_client *client +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + lt_t instances, tmp; + + list_add_tail(&client->list, &res->clients); + + switch (res->state) { + case RESERVATION_INACTIVE: + /* Figure out next replenishment time. */ + if (res->env->time_zero == 0) { + tmp = res->env->current_time - res->env->time_zero; + instances = div64_u64(tmp, pres->period); + res->next_replenishment = + (instances + 1) * pres->period + pres->offset; + } + else { + tmp = res->env->current_time - res->env->time_zero; + instances = div64_u64(tmp, pres->period); + res->next_replenishment = res->env->time_zero + instances * pres->period; + } + + TRACE("ENV_TIME_ZERO %llu in mode %d\n", res->env->time_zero, res->mode); + TRACE("pol-res: mode %d R%d activate tmp=%llu instances=%llu period=%llu nextrp=%llu cur=%llu\n", + res->mode, res->id, tmp, instances, pres->period, res->next_replenishment, + res->env->current_time); + + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + break; + + case RESERVATION_ACTIVE: + case RESERVATION_DEPLETED: + /* do nothing */ + break; + + case RESERVATION_ACTIVE_IDLE: + res->blocked_by_ghost = 0; + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + } +} + + +static void periodic_polling_client_departs( + struct reservation *res, + struct reservation_client *client, + int did_signal_job_completion +) +{ + list_del(&client->list); + + switch (res->state) { + case RESERVATION_INACTIVE: + case RESERVATION_ACTIVE_IDLE: + BUG(); /* INACTIVE or IDLE <=> no client */ + break; + + case RESERVATION_ACTIVE: + if (list_empty(&res->clients)) { + res->env->change_state(res->env, res, +// RESERVATION_ACTIVE_IDLE); + res->cur_budget ? + RESERVATION_ACTIVE_IDLE : + RESERVATION_DEPLETED); +// did_signal_job_completion ? +// RESERVATION_DEPLETED : +// RESERVATION_ACTIVE_IDLE); + } /* else: nothing to do, more clients ready */ + break; + + case RESERVATION_DEPLETED: + /* do nothing */ + break; + } +} + +static void periodic_polling_on_replenishment( + struct reservation *res +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + + /* replenish budget */ + res->cur_budget = pres->max_budget; + res->next_replenishment += pres->period; + res->budget_consumed = 0; + + TRACE("polling_replenish(%u): next_replenishment=%llu\n", res->id, res->next_replenishment); + switch (res->state) { + case RESERVATION_DEPLETED: + case RESERVATION_INACTIVE: + case RESERVATION_ACTIVE_IDLE: + if (list_empty(&res->clients)) + /* no clients => poll again later */ + res->env->change_state(res->env, res, + RESERVATION_INACTIVE); + else + /* we have clients & budget => ACTIVE */ + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + + case RESERVATION_ACTIVE: + /* Replenished while active => tardy? In any case, + * go ahead and stay active. */ + break; + } +} + +static void periodic_polling_on_replenishment_edf( + struct reservation *res +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + + /* update current priority */ + res->priority = res->next_replenishment + pres->deadline; + + /* do common updates */ + periodic_polling_on_replenishment(res); +} + +static void common_drain_budget( + struct reservation *res, + lt_t how_much) +{ + if (how_much >= res->cur_budget) + res->cur_budget = 0; + else + res->cur_budget -= how_much; + + res->budget_consumed += how_much; + res->budget_consumed_total += how_much; + + switch (res->state) { + case RESERVATION_DEPLETED: + case RESERVATION_INACTIVE: + //BUG(); + TRACE("!!!!!!!!!!!!!!!STATE ERROR R%d STATE(%d)\n", res->id, res->state); + break; + + case RESERVATION_ACTIVE_IDLE: + case RESERVATION_ACTIVE: + if (!res->cur_budget) { + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + } /* else: stay in current state */ + break; + } +} + +static struct reservation_ops periodic_polling_ops_fp = { + .dispatch_client = default_dispatch_client, + .client_arrives = periodic_polling_client_arrives, + .client_departs = periodic_polling_client_departs, + .replenish = periodic_polling_on_replenishment, + .drain_budget = common_drain_budget, +}; + +static struct reservation_ops periodic_polling_ops_edf = { + .dispatch_client = default_dispatch_client, + .client_arrives = periodic_polling_client_arrives, + .client_departs = periodic_polling_client_departs, + .replenish = periodic_polling_on_replenishment_edf, + .drain_budget = common_drain_budget, +}; + + + + +static void sporadic_polling_client_arrives_fp( + struct reservation* res, + struct reservation_client *client +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + + list_add_tail(&client->list, &res->clients); + + switch (res->state) { + case RESERVATION_INACTIVE: + /* Replenish now. */ + res->cur_budget = pres->max_budget; + res->next_replenishment = + res->env->current_time + pres->period; + + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + + case RESERVATION_ACTIVE: + case RESERVATION_DEPLETED: + /* do nothing */ + break; + + case RESERVATION_ACTIVE_IDLE: + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + } +} + +static void sporadic_polling_client_arrives_edf( + struct reservation* res, + struct reservation_client *client +) +{ + struct polling_reservation *pres = + container_of(res, struct polling_reservation, res); + + list_add_tail(&client->list, &res->clients); + + switch (res->state) { + case RESERVATION_INACTIVE: + /* Replenish now. */ + res->cur_budget = pres->max_budget; + res->next_replenishment = + res->env->current_time + pres->period; + res->priority = + res->env->current_time + pres->deadline; + + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + + case RESERVATION_ACTIVE: + case RESERVATION_DEPLETED: + /* do nothing */ + break; + + case RESERVATION_ACTIVE_IDLE: + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + } +} + +static struct reservation_ops sporadic_polling_ops_fp = { + .dispatch_client = default_dispatch_client, + .client_arrives = sporadic_polling_client_arrives_fp, + .client_departs = periodic_polling_client_departs, + .replenish = periodic_polling_on_replenishment, + .drain_budget = common_drain_budget, +}; + +static struct reservation_ops sporadic_polling_ops_edf = { + .dispatch_client = default_dispatch_client, + .client_arrives = sporadic_polling_client_arrives_edf, + .client_departs = periodic_polling_client_departs, + .replenish = periodic_polling_on_replenishment_edf, + .drain_budget = common_drain_budget, +}; + +void polling_reservation_init( + struct polling_reservation *pres, + int use_edf_prio, + int use_periodic_polling, + lt_t budget, lt_t period, lt_t deadline, lt_t offset +) +{ + if (!deadline) + deadline = period; + BUG_ON(budget > period); + BUG_ON(budget > deadline); + BUG_ON(offset >= period); + + reservation_init(&pres->res); + pres->max_budget = budget; + pres->period = period; + pres->deadline = deadline; + pres->offset = offset; + TRACE_TASK(current, "polling_reservation_init: periodic %d, use_edf %d\n", use_periodic_polling, use_edf_prio); + if (use_periodic_polling) { + if (use_edf_prio) + pres->res.ops = &periodic_polling_ops_edf; + else + pres->res.ops = &periodic_polling_ops_fp; + } else { + if (use_edf_prio) + pres->res.ops = &sporadic_polling_ops_edf; + else + pres->res.ops = &sporadic_polling_ops_fp; + } +} + + +static lt_t td_cur_major_cycle_start(struct table_driven_reservation *tdres) +{ + lt_t x, tmp; + + tmp = tdres->res.env->current_time - tdres->res.env->time_zero; + x = div64_u64(tmp, tdres->major_cycle); + x *= tdres->major_cycle; + return x; +} + + +static lt_t td_next_major_cycle_start(struct table_driven_reservation *tdres) +{ + lt_t x, tmp; + + tmp = tdres->res.env->current_time - tdres->res.env->time_zero; + x = div64_u64(tmp, tdres->major_cycle) + 1; + x *= tdres->major_cycle; + return x; +} + +static void td_client_arrives( + struct reservation* res, + struct reservation_client *client +) +{ + struct table_driven_reservation *tdres = + container_of(res, struct table_driven_reservation, res); + + list_add_tail(&client->list, &res->clients); + + switch (res->state) { + case RESERVATION_INACTIVE: + /* Figure out first replenishment time. */ + tdres->major_cycle_start = td_next_major_cycle_start(tdres); + res->next_replenishment = tdres->major_cycle_start; + res->next_replenishment += tdres->intervals[0].start; + tdres->next_interval = 0; + + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + break; + + case RESERVATION_ACTIVE: + case RESERVATION_DEPLETED: + /* do nothing */ + break; + + case RESERVATION_ACTIVE_IDLE: + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + } +} + +static void td_client_departs( + struct reservation *res, + struct reservation_client *client, + int did_signal_job_completion +) +{ + list_del(&client->list); + + switch (res->state) { + case RESERVATION_INACTIVE: + case RESERVATION_ACTIVE_IDLE: + //BUG(); /* INACTIVE or IDLE <=> no client */ + break; + + case RESERVATION_ACTIVE: + if (list_empty(&res->clients)) { + res->env->change_state(res->env, res, + RESERVATION_ACTIVE_IDLE); + } /* else: nothing to do, more clients ready */ + break; + + case RESERVATION_DEPLETED: + /* do nothing */ + break; + } +} + +static lt_t td_time_remaining_until_end(struct table_driven_reservation *tdres) +{ + lt_t now = tdres->res.env->current_time; + lt_t end = tdres->cur_interval.end; + //TRACE("td_remaining(%u): start=%llu now=%llu end=%llu state=%d\n", tdres->res.id, tdres->cur_interval.start, now, end, tdres->res.state); + if (now >= end) + return 0; + else + return end - now; +} + +static void td_replenish( + struct reservation *res) +{ + struct table_driven_reservation *tdres = + container_of(res, struct table_driven_reservation, res); + + //TRACE("td_replenish(%u): expected_replenishment=%llu\n", res->id, res->next_replenishment); + + /* figure out current interval */ + tdres->cur_interval.start = tdres->major_cycle_start + + tdres->intervals[tdres->next_interval].start; + tdres->cur_interval.end = tdres->major_cycle_start + + tdres->intervals[tdres->next_interval].end; +/* TRACE("major_cycle_start=%llu => [%llu, %llu]\n", + tdres->major_cycle_start, + tdres->cur_interval.start, + tdres->cur_interval.end); +*/ + /* reset budget */ + res->cur_budget = td_time_remaining_until_end(tdres); + res->budget_consumed = 0; + //TRACE("td_replenish(%u): %s budget=%llu\n", res->id, res->cur_budget ? "" : "WARNING", res->cur_budget); + + /* prepare next slot */ + tdres->next_interval = (tdres->next_interval + 1) % tdres->num_intervals; + if (!tdres->next_interval) + /* wrap to next major cycle */ + tdres->major_cycle_start += tdres->major_cycle; + + /* determine next time this reservation becomes eligible to execute */ + res->next_replenishment = tdres->major_cycle_start; + res->next_replenishment += tdres->intervals[tdres->next_interval].start; + //TRACE("td_replenish(%u): next_replenishment=%llu\n", res->id, res->next_replenishment); + + + switch (res->state) { + case RESERVATION_DEPLETED: + case RESERVATION_ACTIVE: + case RESERVATION_ACTIVE_IDLE: + if (list_empty(&res->clients)) + res->env->change_state(res->env, res, + RESERVATION_ACTIVE_IDLE); + else + /* we have clients & budget => ACTIVE */ + res->env->change_state(res->env, res, + RESERVATION_ACTIVE); + break; + + case RESERVATION_INACTIVE: + BUG(); + break; + } +} + +static void td_drain_budget( + struct reservation *res, + lt_t how_much) +{ + struct table_driven_reservation *tdres = + container_of(res, struct table_driven_reservation, res); + + res->budget_consumed += how_much; + res->budget_consumed_total += how_much; + + /* Table-driven scheduling: instead of tracking the budget, we compute + * how much time is left in this allocation interval. */ + + /* sanity check: we should never try to drain from future slots */ + //TRACE("TD_DRAIN STATE(%d) [%llu,%llu] %llu ?\n", res->state, tdres->cur_interval.start, tdres->cur_interval.end, res->env->current_time); + //BUG_ON(tdres->cur_interval.start > res->env->current_time); + if (tdres->cur_interval.start > res->env->current_time) + TRACE("TD_DRAIN BUG!!!!!!!!!!\n"); + + switch (res->state) { + case RESERVATION_DEPLETED: + case RESERVATION_INACTIVE: + //BUG(); + TRACE("TD_DRAIN!!!!!!!!! RES_STATE = %d\n", res->state); + break; + + case RESERVATION_ACTIVE_IDLE: + case RESERVATION_ACTIVE: + res->cur_budget = td_time_remaining_until_end(tdres); + //TRACE("td_drain_budget(%u): drained to budget=%llu\n", res->id, res->cur_budget); + if (!res->cur_budget) { + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + } else { + /* sanity check budget calculation */ + //BUG_ON(res->env->current_time >= tdres->cur_interval.end); + //BUG_ON(res->env->current_time < tdres->cur_interval.start); + if (res->env->current_time >= tdres->cur_interval.end) + printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING1\n"); + if (res->env->current_time < tdres->cur_interval.start) + printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING2\n"); + } + + break; + } +} + +static struct task_struct* td_dispatch_client( + struct reservation *res, + lt_t *for_at_most) +{ + struct task_struct *t; + struct table_driven_reservation *tdres = + container_of(res, struct table_driven_reservation, res); + + /* usual logic for selecting a client */ + t = default_dispatch_client(res, for_at_most); + + TRACE_TASK(t, "td_dispatch_client(%u): selected, budget=%llu\n", + res->id, res->cur_budget); + + /* check how much budget we have left in this time slot */ + res->cur_budget = td_time_remaining_until_end(tdres); + + TRACE_TASK(t, "td_dispatch_client(%u): updated to budget=%llu next=%d\n", + res->id, res->cur_budget, tdres->next_interval); + + if (unlikely(!res->cur_budget)) { + /* Unlikely case: if we ran out of budget, the user configured + * a broken scheduling table (overlapping table slots). + * Not much we can do about this, but we can't dispatch a job + * now without causing overload. So let's register this reservation + * as depleted and wait for the next allocation. */ + TRACE("td_dispatch_client(%u): budget unexpectedly depleted " + "(check scheduling table for unintended overlap)\n", + res->id); + res->env->change_state(res->env, res, + RESERVATION_DEPLETED); + return NULL; + } else + return t; +} + +static struct reservation_ops td_ops = { + .dispatch_client = td_dispatch_client, + .client_arrives = td_client_arrives, + .client_departs = td_client_departs, + .replenish = td_replenish, + .drain_budget = td_drain_budget, +}; + +void table_driven_reservation_init( + struct table_driven_reservation *tdres, + lt_t major_cycle, + struct lt_interval *intervals, + unsigned int num_intervals) +{ + unsigned int i; + + /* sanity checking */ + BUG_ON(!num_intervals); + for (i = 0; i < num_intervals; i++) + BUG_ON(intervals[i].end <= intervals[i].start); + for (i = 0; i + 1 < num_intervals; i++) + BUG_ON(intervals[i + 1].start <= intervals[i].end); + BUG_ON(intervals[num_intervals - 1].end > major_cycle); + + reservation_init(&tdres->res); + tdres->major_cycle = major_cycle; + tdres->intervals = intervals; + tdres->cur_interval.start = 0; + tdres->cur_interval.end = 0; + tdres->num_intervals = num_intervals; + tdres->res.ops = &td_ops; +} diff --git litmus/reservation.c litmus/reservation.c new file mode 100644 index 0000000..e3381d6 --- /dev/null +++ litmus/reservation.c @@ -0,0 +1,710 @@ +#include +#include + +#include +#include + +//#define TRACE(fmt, args...) do {} while (false) +//#define TRACE_TASK(fmt, args...) do {} while (false) + +#define BUDGET_ENFORCEMENT_AT_C 0 + +void reservation_init(struct reservation *res) +{ + memset(res, sizeof(*res), 0); + res->state = RESERVATION_INACTIVE; + INIT_LIST_HEAD(&res->clients); +} + +struct task_struct* default_dispatch_client( + struct reservation *res, + lt_t *for_at_most) +{ + struct reservation_client *client, *next; + struct task_struct* tsk; + + BUG_ON(res->state != RESERVATION_ACTIVE); + *for_at_most = 0; + + list_for_each_entry_safe(client, next, &res->clients, list) { + tsk = client->dispatch(client); + if (likely(tsk)) { + return tsk; + } + } + return NULL; +} + +static struct task_struct * task_client_dispatch(struct reservation_client *client) +{ + struct task_client *tc = container_of(client, struct task_client, client); + return tc->task; +} + +void task_client_init(struct task_client *tc, struct task_struct *tsk, + struct reservation *res) +{ + memset(&tc->client, sizeof(tc->client), 0); + tc->client.dispatch = task_client_dispatch; + tc->client.reservation = res; + tc->task = tsk; +} + +static void sup_scheduler_update_at( + struct sup_reservation_environment* sup_env, + lt_t when) +{ + //TRACE("SCHEDULER_UPDATE_AT update: %llu > when %llu\n", sup_env->next_scheduler_update, when); + if (sup_env->next_scheduler_update > when) + sup_env->next_scheduler_update = when; +} + +void sup_scheduler_update_after( + struct sup_reservation_environment* sup_env, + lt_t timeout) +{ + sup_scheduler_update_at(sup_env, sup_env->env.current_time + timeout); +} + +static int _sup_queue_depleted( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + struct list_head *pos; + struct reservation *queued; + int passed_earlier = 0; + + list_for_each(pos, &sup_env->depleted_reservations) { + queued = list_entry(pos, struct reservation, list); + if (queued->next_replenishment > res->next_replenishment) { + list_add(&res->list, pos->prev); + return passed_earlier; + } else + passed_earlier = 1; + } + + list_add_tail(&res->list, &sup_env->depleted_reservations); + + return passed_earlier; +} + +static void sup_queue_depleted( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + int passed_earlier = _sup_queue_depleted(sup_env, res); + + /* check for updated replenishment time */ + if (!passed_earlier) + sup_scheduler_update_at(sup_env, res->next_replenishment); +} + +static int _sup_queue_active( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + struct list_head *pos; + struct reservation *queued; + int passed_active = 0; + + list_for_each(pos, &sup_env->active_reservations) { + queued = list_entry(pos, struct reservation, list); + if (queued->priority > res->priority) { + list_add(&res->list, pos->prev); + return passed_active; + } else if (queued->state == RESERVATION_ACTIVE) + passed_active = 1; + } + + list_add_tail(&res->list, &sup_env->active_reservations); + return passed_active; +} + +static void sup_queue_active( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + int passed_active = _sup_queue_active(sup_env, res); + + /* check for possible preemption */ + if (res->state == RESERVATION_ACTIVE && !passed_active) + sup_env->next_scheduler_update = SUP_RESCHEDULE_NOW; + else { + /* Active means this reservation is draining budget => make sure + * the scheduler is called to notice when the reservation budget has been + * drained completely. */ + sup_scheduler_update_after(sup_env, res->cur_budget); + } +} + +static void sup_queue_reservation( + struct sup_reservation_environment* sup_env, + struct reservation *res) +{ + switch (res->state) { + case RESERVATION_INACTIVE: + list_add(&res->list, &sup_env->inactive_reservations); + break; + + case RESERVATION_DEPLETED: + sup_queue_depleted(sup_env, res); + break; + + case RESERVATION_ACTIVE_IDLE: + case RESERVATION_ACTIVE: + sup_queue_active(sup_env, res); + break; + } +} + +void sup_add_new_reservation( + struct sup_reservation_environment* sup_env, + struct reservation* new_res) +{ + new_res->env = &sup_env->env; + sup_queue_reservation(sup_env, new_res); +} + +struct reservation* sup_find_by_id(struct sup_reservation_environment* sup_env, + unsigned int id) +{ + struct reservation *res; + BUG_ON(!sup_env); + list_for_each_entry(res, &sup_env->active_reservations, list) { + if (res->id == id) + return res; + } + list_for_each_entry(res, &sup_env->inactive_reservations, list) { + if (res->id == id) + return res; + } + list_for_each_entry(res, &sup_env->depleted_reservations, list) { + if (res->id == id) + return res; + } + + return NULL; +} + +static void sup_charge_budget( + struct sup_reservation_environment* sup_env, + lt_t delta) +{ + struct list_head *pos, *next; + struct reservation *res; + + int encountered_active = 0; + + list_for_each_safe(pos, next, &sup_env->active_reservations) { + /* charge all ACTIVE_IDLE up to the first ACTIVE reservation */ + res = list_entry(pos, struct reservation, list); + if (res->state == RESERVATION_ACTIVE) { + //TRACE("sup_charge_budget ACTIVE R%u drain %llu\n", res->id, delta); + if (encountered_active == 0 && res->blocked_by_ghost == 0) { + //TRACE("DRAIN !!\n"); + res->ops->drain_budget(res, delta); + encountered_active = 1; + } + } else { + //BUG_ON(res->state != RESERVATION_ACTIVE_IDLE); + TRACE("sup_charge_budget INACTIVE R%u drain %llu\n", res->id, delta); + res->ops->drain_budget(res, delta); + } + if (res->state == RESERVATION_ACTIVE || + res->state == RESERVATION_ACTIVE_IDLE) + { + /* make sure scheduler is invoked when this reservation expires + * its remaining budget */ + //TRACE("requesting scheduler update for reservation %u in %llu nanoseconds\n", res->id, res->cur_budget); + sup_scheduler_update_after(sup_env, res->cur_budget); + } + //if (encountered_active == 2) + /* stop at the first ACTIVE reservation */ + // break; + } + //TRACE("finished charging budgets\n"); +} + +static void sup_replenish_budgets(struct sup_reservation_environment* sup_env) +{ + struct list_head *pos, *next; + struct reservation *res; + + list_for_each_safe(pos, next, &sup_env->depleted_reservations) { + res = list_entry(pos, struct reservation, list); + if (res->next_replenishment <= sup_env->env.current_time) { + res->ops->replenish(res); + } else { + /* list is ordered by increasing depletion times */ + break; + } + } + //TRACE("finished replenishing budgets\n"); + + /* request a scheduler update at the next replenishment instant */ + res = list_first_entry_or_null(&sup_env->depleted_reservations, + struct reservation, list); + if (res) + sup_scheduler_update_at(sup_env, res->next_replenishment); +} + +void sup_update_time( + struct sup_reservation_environment* sup_env, + lt_t now) +{ + lt_t delta; + + /* If the time didn't advance, there is nothing to do. + * This check makes it safe to call sup_advance_time() potentially + * multiple times (e.g., via different code paths. */ + //TRACE("(sup_update_time) now: %llu, current_time: %llu\n", now, sup_env->env.current_time); + if (unlikely(now <= sup_env->env.current_time)) + return; + + delta = now - sup_env->env.current_time; + sup_env->env.current_time = now; + + /* check if future updates are required */ + if (sup_env->next_scheduler_update <= sup_env->env.current_time) + sup_env->next_scheduler_update = SUP_NO_SCHEDULER_UPDATE; + + /* deplete budgets by passage of time */ + //TRACE("CHARGE###\n"); + sup_charge_budget(sup_env, delta); + + /* check if any budgets where replenished */ + //TRACE("REPLENISH###\n"); + sup_replenish_budgets(sup_env); +} + +struct task_struct* sup_dispatch(struct sup_reservation_environment* sup_env) +{ + struct reservation *res, *next; + struct task_struct *tsk = NULL; + lt_t time_slice; + + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) { + if (res->state == RESERVATION_ACTIVE) { + tsk = res->ops->dispatch_client(res, &time_slice); + if (likely(tsk)) { + if (time_slice) + sup_scheduler_update_after(sup_env, time_slice); + sup_scheduler_update_after(sup_env, res->cur_budget); + return tsk; + } + } + } + + return NULL; +} + +static void sup_res_change_state( + struct reservation_environment* env, + struct reservation *res, + reservation_state_t new_state) +{ + struct sup_reservation_environment* sup_env; + + sup_env = container_of(env, struct sup_reservation_environment, env); + + TRACE("reservation R%d state %d->%d at %llu\n", + res->id, res->state, new_state, env->current_time); + + list_del(&res->list); + /* check if we need to reschedule because we lost an active reservation */ + if (res->state == RESERVATION_ACTIVE && !sup_env->will_schedule) + sup_env->next_scheduler_update = SUP_RESCHEDULE_NOW; + res->state = new_state; + sup_queue_reservation(sup_env, res); +} + +void sup_init(struct sup_reservation_environment* sup_env) +{ + memset(sup_env, sizeof(*sup_env), 0); + + INIT_LIST_HEAD(&sup_env->active_reservations); + INIT_LIST_HEAD(&sup_env->depleted_reservations); + INIT_LIST_HEAD(&sup_env->inactive_reservations); + + sup_env->env.change_state = sup_res_change_state; + + sup_env->next_scheduler_update = SUP_NO_SCHEDULER_UPDATE; +} + +struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env, + unsigned int id) +{ + struct reservation *res; + + list_for_each_entry(res, &gmp_env->active_reservations, list) { + if (res->id == id) + return res; + } + list_for_each_entry(res, &gmp_env->inactive_reservations, list) { + if (res->id == id) + return res; + } + list_for_each_entry(res, &gmp_env->depleted_reservations, list) { + if (res->id == id) + return res; + } + + return NULL; +} + + +struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env, + unsigned int id) +{ + struct next_timer_event *event; + + list_for_each_entry(event, &gmp_env->next_events, list) { + if (event->id == id) + return event; + } + + return NULL; +} + + +struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env, + lt_t when) +{ + struct next_timer_event *event; + + list_for_each_entry(event, &gmp_env->next_events, list) { + if (event->next_update == when) + return event; + } + + return NULL; +} + +#define TIMER_RESOLUTION 100000L + +static void gmp_add_event( + struct gmp_reservation_environment* gmp_env, + lt_t when, unsigned int id, event_type_t type) +{ + struct next_timer_event *nevent, *queued; + struct list_head *pos; + int found = 0, update = 0; + + //when = div64_u64(when, TIMER_RESOLUTION); + //when *= TIMER_RESOLUTION; +//printk(KERN_ALERT "GMP_ADD id=%d type=%d when=%llu\n", id, type, when); + nevent = gmp_find_event_by_id(gmp_env, id); + + if (nevent) + TRACE("EVENT R%d update prev = %llu, new = %llu\n", nevent->id, nevent->next_update, when); + + if (nevent && nevent->next_update > when) { + list_del(&nevent->list); + update = 1; + + } + + if (!nevent || nevent->type != type || update == 1) { + if (update == 0) + nevent = kzalloc(sizeof(*nevent), GFP_ATOMIC); + BUG_ON(!nevent); + nevent->next_update = when; + nevent->id = id; + nevent->type = type; + nevent->timer_armed_on = NO_CPU; + + list_for_each(pos, &gmp_env->next_events) { + queued = list_entry(pos, struct next_timer_event, list); + if (queued->next_update > nevent->next_update) { + list_add(&nevent->list, pos->prev); + found = 1; + TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at before %llu\n", nevent->id, nevent->type, nevent->next_update, queued->next_update); + break; + } + } + + if (!found) { + list_add_tail(&nevent->list, &gmp_env->next_events); + TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at TAIL\n", nevent->id, nevent->type, nevent->next_update); + } + } else { + //TRACE("EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when); +; //printk(KERN_ALERT "EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when); + } + + TRACE("======START PRINTING EVENT LIST======\n"); + gmp_print_events(gmp_env, litmus_clock()); + TRACE("======FINISH PRINTING EVENT LIST======\n"); +} + +void gmp_add_event_after( + struct gmp_reservation_environment* gmp_env, lt_t timeout, unsigned int id, event_type_t type) +{ + //printk(KERN_ALERT "ADD_EVENT_AFTER id = %d\n", id); + gmp_add_event(gmp_env, gmp_env->env.current_time + timeout, id, type); +} + +static void gmp_queue_depleted( + struct gmp_reservation_environment* gmp_env, + struct reservation *res) +{ + struct list_head *pos; + struct reservation *queued; + int found = 0; + +//printk(KERN_ALERT "R%d request to enqueue depleted_list\n", res->id); + + list_for_each(pos, &gmp_env->depleted_reservations) { + queued = list_entry(pos, struct reservation, list); + if (queued && (queued->next_replenishment > res->next_replenishment)) { +//printk(KERN_ALERT "QUEUED R%d %llu\n", queued->id, queued->next_replenishment); + list_add(&res->list, pos->prev); + found = 1; + break; + } + } + + if (!found) + list_add_tail(&res->list, &gmp_env->depleted_reservations); + + TRACE("R%d queued to depleted_list\n", res->id); +//printk(KERN_ALERT "R%d queued to depleted_list\n", res->id); + gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH); +} + +static void gmp_queue_active( + struct gmp_reservation_environment* gmp_env, + struct reservation *res) +{ + struct list_head *pos; + struct reservation *queued; + int check_preempt = 1, found = 0; + + TRACE("R%d has been queued on active\n", res->id); + + list_for_each(pos, &gmp_env->active_reservations) { + queued = list_entry(pos, struct reservation, list); + if (queued->priority > res->priority) { + list_add(&res->list, pos->prev); + found = 1; + break; + } else if (queued->scheduled_on == NO_CPU) + check_preempt = 0; + } + + if (!found) + list_add_tail(&res->list, &gmp_env->active_reservations); + + /* check for possible preemption */ + if (res->state == RESERVATION_ACTIVE && check_preempt) + gmp_env->schedule_now++; + +#if BUDGET_ENFORCEMENT_AT_C + gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN); +#endif + res->event_added = 1; +} + +static void gmp_queue_reservation( + struct gmp_reservation_environment* gmp_env, + struct reservation *res) +{ + +//printk(KERN_ALERT "DEBUG: Passed %s %d %p R%d STATE %d\n",__FUNCTION__,__LINE__, gmp_env, res->id, res->state); + switch (res->state) { + case RESERVATION_INACTIVE: + list_add(&res->list, &gmp_env->inactive_reservations); + break; + + case RESERVATION_DEPLETED: + gmp_queue_depleted(gmp_env, res); + break; + + case RESERVATION_ACTIVE_IDLE: + case RESERVATION_ACTIVE: + gmp_queue_active(gmp_env, res); + break; + } +} + +void gmp_add_new_reservation( + struct gmp_reservation_environment* gmp_env, + struct reservation* new_res) +{ + new_res->env = &gmp_env->env; + gmp_queue_reservation(gmp_env, new_res); +} + +#if BUDGET_ENFORCEMENT_AT_C +static void gmp_charge_budget( + struct gmp_reservation_environment* gmp_env, + lt_t delta) +{ + struct list_head *pos, *next; + struct reservation *res; + + list_for_each_safe(pos, next, &gmp_env->active_reservations) { + int drained = 0; + /* charge all ACTIVE_IDLE up to the first ACTIVE reservation */ + res = list_entry(pos, struct reservation, list); + if (res->state == RESERVATION_ACTIVE) { + TRACE("gmp_charge_budget ACTIVE R%u scheduled_on=%d drain %llu\n", res->id, res->scheduled_on, delta); + if (res->scheduled_on != NO_CPU && res->blocked_by_ghost == 0) { + TRACE("DRAIN !!\n"); + drained = 1; + res->ops->drain_budget(res, delta); + } else { + TRACE("NO DRAIN (not scheduled)!!\n"); + } + } else { + //BUG_ON(res->state != RESERVATION_ACTIVE_IDLE); + if (res->state != RESERVATION_ACTIVE_IDLE) + TRACE("BUG!!!!!!!!!!!! gmp_charge_budget()\n"); + TRACE("gmp_charge_budget INACTIVE R%u drain %llu\n", res->id, delta); + //if (res->is_ghost != NO_CPU) { + TRACE("DRAIN !!\n"); + drained = 1; + res->ops->drain_budget(res, delta); + //} + } + if ((res->state == RESERVATION_ACTIVE || + res->state == RESERVATION_ACTIVE_IDLE) && (drained == 1)) + { + /* make sure scheduler is invoked when this reservation expires + * its remaining budget */ + TRACE("requesting gmp_scheduler update for reservation %u in %llu nanoseconds\n", res->id, res->cur_budget); + gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN); + res->event_added = 1; + } + //if (encountered_active == 2) + /* stop at the first ACTIVE reservation */ + // break; + } + //TRACE("finished charging budgets\n"); +} +#else + +static void gmp_charge_budget( + struct gmp_reservation_environment* gmp_env, + lt_t delta) +{ + return; +} + +#endif + +static void gmp_replenish_budgets(struct gmp_reservation_environment* gmp_env) +{ + struct list_head *pos, *next; + struct reservation *res; + + list_for_each_safe(pos, next, &gmp_env->depleted_reservations) { + res = list_entry(pos, struct reservation, list); + if (res->next_replenishment <= gmp_env->env.current_time) { + res->ops->replenish(res); + if (res->is_ghost != NO_CPU) { + TRACE("R%d replenished! scheduled_on=%d\n", res->id, res->scheduled_on); + } + } else { + /* list is ordered by increasing depletion times */ + break; + } + } + //TRACE("finished replenishing budgets\n"); +} + +#define EPSILON 50 + +/* return schedule_now */ +int gmp_update_time( + struct gmp_reservation_environment* gmp_env, + lt_t now) +{ + struct next_timer_event *event, *next; + lt_t delta, ret; + + /* If the time didn't advance, there is nothing to do. + * This check makes it safe to call sup_advance_time() potentially + * multiple times (e.g., via different code paths. */ + //TRACE("(gmp_update_time) now: %llu, current_time: %llu\n", now, gmp_env->env.current_time); + if (unlikely(now <= gmp_env->env.current_time + EPSILON)) + return 0; + + delta = now - gmp_env->env.current_time; + gmp_env->env.current_time = now; + + + //gmp_print_events(gmp_env, now); + /* deplete budgets by passage of time */ + //TRACE("CHARGE###\n"); + gmp_charge_budget(gmp_env, delta); + + /* check if any budgets where replenished */ + //TRACE("REPLENISH###\n"); + gmp_replenish_budgets(gmp_env); + + + list_for_each_entry_safe(event, next, &gmp_env->next_events, list) { + if (event->next_update < now) { + list_del(&event->list); + //TRACE("EVENT at %llu IS DELETED\n", event->next_update); + kfree(event); + } else { + break; + } + } + + //gmp_print_events(gmp_env, litmus_clock()); + + ret = min(gmp_env->schedule_now, NR_CPUS); + gmp_env->schedule_now = 0; + + return ret; +} + +void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now) +{ + struct next_timer_event *event, *next; + + TRACE("GLOBAL EVENTS now=%llu\n", now); + list_for_each_entry_safe(event, next, &gmp_env->next_events, list) { + TRACE("at %llu type=%d id=%d armed_on=%d\n", event->next_update, event->type, event->id, event->timer_armed_on); + } +} + +static void gmp_res_change_state( + struct reservation_environment* env, + struct reservation *res, + reservation_state_t new_state) +{ + struct gmp_reservation_environment* gmp_env; + + gmp_env = container_of(env, struct gmp_reservation_environment, env); + + TRACE("GMP reservation R%d state %d->%d at %llu\n", + res->id, res->state, new_state, env->current_time); + + list_del(&res->list); + /* check if we need to reschedule because we lost an active reservation */ + if (res->state == RESERVATION_ACTIVE) + gmp_env->schedule_now++; + res->state = new_state; + gmp_queue_reservation(gmp_env, res); +} + +void gmp_init(struct gmp_reservation_environment* gmp_env) +{ + memset(gmp_env, sizeof(*gmp_env), 0); + + INIT_LIST_HEAD(&gmp_env->active_reservations); + INIT_LIST_HEAD(&gmp_env->depleted_reservations); + INIT_LIST_HEAD(&gmp_env->inactive_reservations); + INIT_LIST_HEAD(&gmp_env->next_events); + + gmp_env->env.change_state = gmp_res_change_state; + + gmp_env->schedule_now = 0; + gmp_env->will_schedule = false; + + //raw_spin_lock_init(&gmp_env->lock); +} diff --git litmus/sched_mc2.c litmus/sched_mc2.c new file mode 100644 index 0000000..d7cf3fb --- /dev/null +++ litmus/sched_mc2.c @@ -0,0 +1,2150 @@ +/* + * litmus/sched_mc2.c + * + * Implementation of the Mixed-Criticality on MultiCore scheduler + * + * This plugin implements a scheduling algorithm proposed in + * "Mixed-Criticality Real-Time Scheduling for Multicore System" paper. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define BUDGET_ENFORCEMENT_AT_C 0 + +extern atomic_t num_sync_released; +extern void do_partition(enum crit_level lv, int cpu); + +/* _global_env - reservation container for level-C tasks */ +struct gmp_reservation_environment _global_env_modes[NR_MODES]; +struct gmp_reservation_environment *_global_env; +raw_spinlock_t global_lock; + +/* cpu_entry - keep track of a running task on a cpu + * This state is used to decide the lowest priority cpu + */ +struct cpu_entry { + struct task_struct *scheduled; + lt_t deadline; + int cpu; + enum crit_level lv; + /* if will_schedule is true, this cpu is already selected and + mc2_schedule() will be executed soon. */ + bool will_schedule; +}; + +/* cpu_priority - a global state for choosing the lowest priority CPU */ +struct cpu_priority { + raw_spinlock_t lock; + struct cpu_entry cpu_entries[NR_CPUS]; +}; + +struct cpu_priority _lowest_prio_cpu; + +/* mc2_task_state - a task state structure */ +struct mc2_task_state { + /* A task can be shared by multiple modes */ + struct task_client res_info[NR_MODES]; + /* if cpu == -1, this task is a global task (level C) */ + int cpu; + bool has_departed; + struct mc2_task mc2_param; +}; + +/* mc2_cpu_state - maintain the scheduled state and ghost jobs + * timer : timer for partitioned tasks (level A and B) + */ +struct mc2_cpu_state { + raw_spinlock_t lock; + + struct sup_reservation_environment sup_env_modes[NR_MODES]; + struct sup_reservation_environment *sup_env; + struct hrtimer timer; + + int cpu; + struct task_struct* scheduled; +}; + +static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state); + +static int resched_cpu[NR_CPUS]; +static DEFINE_PER_CPU(struct mc2_cpu_state, mc2_cpu_state); + +#define cpu_state_for(cpu_id) (&per_cpu(mc2_cpu_state, cpu_id)) +#define local_cpu_state() (this_cpu_ptr(&mc2_cpu_state)) + +unsigned int mode; //currently executing mode, from 0 to NR_MODES-1 +unsigned int requested_mode; //The pending mode +/* Prevent multiple requests from entering and prevent request from entering while old + * is being enacted */ +raw_spinlock_t mode_lock; + +unsigned int mode_sizes[NR_MODES]; +unsigned int res_reported; +bool cpu_0_spin_flag; +bool seen_once; +bool cpu_0_task_exist; +bool mode_changed; +bool mode_poll_exited; +static DEFINE_PER_CPU(unsigned long, mode_counter); + +/* Mode change macros */ +#define local_mode_counter() (this_cpu_ptr(&mode_counter)) +#define cpu_0_mode_counter() (&per_cpu(mode_counter, 0)) +#define in_mode(t, modenum) (tsk_mc2_data(t)->mode_mask & (1 << modenum)) +#define pending (mode != requested_mode) +#define ready (!res_reported) + +/* + * To be called from level A task's with period equal to + * A and B hyperperiod + */ + +asmlinkage long sys_enact_mode(void) +{ + struct mc2_cpu_state *state = local_cpu_state(); + struct reservation *res; + struct list_head *pos; + unsigned long flags; + if (state->cpu == 0 && !mode_poll_exited){ + unsigned long *other_cpu_counter; + unsigned long cpu0_val = this_cpu_read(mode_counter); + int i; + for(i = 1; i < NR_CPUS; i++){ + other_cpu_counter = &per_cpu(mode_counter, i); + while(cpu0_val == *other_cpu_counter && !mode_poll_exited){ + udelay(1); + } + } + mode_changed = false; + if (pending){ //MCR has entered + raw_spin_lock_irqsave(&state->lock, flags); + raw_spin_lock(&global_lock); + raw_spin_lock(&mode_lock); + + if (!seen_once){ + TRACE_TASK(current, "REQUEST\n"); + sched_trace_request_mode(current); + //clean up jobs that are already done + //after this jobs report themselves + list_for_each(pos, &_global_env->active_reservations){ + res = list_entry(pos, struct reservation, list); + if (tsk_rt(res->tsk)->completed && res->mode == mode && !res->reported){ + res->reported = 1; + TRACE_TASK(res->tsk,"R%d RES_REPORTED_ACTIVE = %d mode %d\n", res->id, res_reported, res->mode); + res_reported--; + } + } + list_for_each(pos, &_global_env->depleted_reservations){ + res = list_entry(pos, struct reservation, list); + if (tsk_rt(res->tsk)->completed && res->mode == mode && !res->reported){ + res->reported = 1; + TRACE_TASK(res->tsk,"R%d RES_REPORTED_DEPLETED = %d mode %d\n",res->id, res_reported, res->mode); + res_reported--; + } + + } + list_for_each(pos, &_global_env->inactive_reservations){ + res = list_entry(pos, struct reservation, list); + if (tsk_rt(res->tsk)->completed && res->mode == mode && !res->reported){ + res->reported = 1; + TRACE_TASK(res->tsk,"R%d RES_REPORTED_INACTIVE = %d mode %d\n", res->id, res_reported, res->mode); + res_reported--; + } + } + seen_once = true; + } + if (seen_once && !(ready)){ + TRACE("Mode change waiting on tasks\n"); + list_for_each(pos, &_global_env->active_reservations){ + res = list_entry(pos, struct reservation, list); + if (!res->reported){ + TRACE_TASK(res->tsk, "R%d is ACTIVE LIST with state %d scheduled on %d and hasn't reported\n", res->id, res->state, res->scheduled_on); + } + } + list_for_each(pos, &_global_env->depleted_reservations){ + res = list_entry(pos, struct reservation, list); + if (!res->reported){ + TRACE_TASK(res->tsk, "R%d is DEPLETED LIST and hasn't reported\n", res->id); + } + } + list_for_each(pos, &_global_env->inactive_reservations){ + res = list_entry(pos, struct reservation, list); + if (!res->reported){ + TRACE_TASK(res->tsk, "R%d is INACTIVE LIST and hasn't reported\n", res->id); + } + } + } + if( ready ){ //C is throttled + lt_t new_mode_basetime = get_release(current); + mode = requested_mode; + TRACE("Mode has been changed.\n"); + mode_changed = true; + _global_env = &_global_env_modes[mode]; + /* set res->reported for new global tasks */ + list_for_each(pos, &_global_env->active_reservations){ + res = list_entry(pos, struct reservation, list); + release_at(res->tsk, new_mode_basetime); + res->reported = 0; + } + list_for_each(pos, &_global_env->depleted_reservations){ + res = list_entry(pos, struct reservation, list); + release_at(res->tsk, new_mode_basetime); + res->reported = 0; + } + list_for_each(pos, &_global_env->inactive_reservations){ + res = list_entry(pos, struct reservation, list); + release_at(res->tsk, new_mode_basetime); + res->reported = 0; + } + gmp_update_time(_global_env, litmus_clock()); + + state->sup_env = &state->sup_env_modes[mode]; + list_for_each(pos, &state->sup_env->active_reservations){ + res = list_entry(pos, struct reservation, list); + release_at(res->tsk, new_mode_basetime); + } + list_for_each(pos, &state->sup_env->depleted_reservations){ + res = list_entry(pos, struct reservation, list); + release_at(res->tsk, new_mode_basetime); + } + list_for_each(pos, &state->sup_env->inactive_reservations){ + res = list_entry(pos, struct reservation, list); + release_at(res->tsk, new_mode_basetime); + } + sup_update_time(state->sup_env, litmus_clock()); + sched_trace_enact_mode(current); + TRACE("ENACT\n"); + } + raw_spin_unlock(&mode_lock); + raw_spin_unlock(&global_lock); + raw_spin_unlock_irqrestore(&state->lock, flags); + + raw_spin_lock(&state->lock); + mc2_update_timer_and_unlock(state); + } + this_cpu_inc(mode_counter); + } + else if (!mode_poll_exited) { + unsigned long *cpu0_counter = cpu_0_mode_counter(); + unsigned long my_val; + this_cpu_inc(mode_counter); + my_val = this_cpu_read(mode_counter); + //spin, wait for CPU 0 to stabilize mode decision + //before scheduling next hyperperiod + + while (*cpu0_counter < my_val && !mode_poll_exited){ + udelay(1); + } + TRACE("CPU%d counter check. %d\n",state->cpu, this_cpu_read(mode_counter)); + if (mode_changed) { + lt_t new_mode_basetime = get_release(current); + TRACE("CPU%d mode changed\n",state->cpu); + hrtimer_cancel(&state->timer); //stop listening to old mode timers + TRACE("Timer is cancelled at %llu. mode-change\n", litmus_clock()); + raw_spin_lock_irqsave(&state->lock, flags); + state->sup_env = &state->sup_env_modes[mode]; + list_for_each(pos, &state->sup_env->active_reservations){ + res = list_entry(pos, struct reservation, list); + release_at(res->tsk, new_mode_basetime); + } + list_for_each(pos, &state->sup_env->depleted_reservations){ + res = list_entry(pos, struct reservation, list); + release_at(res->tsk, new_mode_basetime); + } + list_for_each(pos, &state->sup_env->inactive_reservations){ + res = list_entry(pos, struct reservation, list); + release_at(res->tsk, new_mode_basetime); + } + sup_update_time(state->sup_env, litmus_clock()); + raw_spin_unlock_irqrestore(&state->lock, flags); + + raw_spin_lock(&state->lock); + mc2_update_timer_and_unlock(state); + + } + } + else { + TRACE("CPU%d exits sys_enact_mode(). No cpu_0_task_exist.%d\n",state->cpu, mode_changed); + return 0; + } + + return 0; +} + + +/* + * Called from non-real time program + * Protect by exclusive lock to prevent from occuring while mode change is enacted + */ + +#define GET_MODE -1 +#define GET_REP -2 +#define GET_SIZE -3 + +asmlinkage long sys_request_mode(int new_mode){ + TRACE("Requesting mode %d\n", new_mode); + preempt_disable(); + raw_spin_lock(&mode_lock); + if (new_mode == GET_MODE){ + int tmp_mode = mode; + raw_spin_unlock(&mode_lock); + preempt_enable(); + return tmp_mode; + } + if (new_mode == GET_REP){ + int tmp_rep = res_reported; + raw_spin_unlock(&mode_lock); + preempt_enable(); + return tmp_rep; + } + if (new_mode == GET_SIZE){ + int tmp_size = mode_sizes[mode]; + raw_spin_unlock(&mode_lock); + preempt_enable(); + return tmp_size; + } + if (pending){ + TRACE("Request to %d denied due to pending to %d\n", new_mode, requested_mode); + raw_spin_unlock(&mode_lock); + preempt_enable(); + TRACE("MCR rejected because the previous MCR is pedning.\n"); + return -EAGAIN; + } + if (mode == new_mode){ + TRACE("Request to %d denied becuase I am in that mode already\n", new_mode); + raw_spin_unlock(&mode_lock); + preempt_enable(); + return -EINVAL; + } + requested_mode = new_mode; + TRACE("MCR received: %d, old:%d\n",requested_mode,mode); + res_reported = mode_sizes[mode]; + TRACE_CUR("RES_REPORTED = %d\n",res_reported); + seen_once = false; + raw_spin_unlock(&mode_lock); + preempt_enable(); + return 0; +} + + +/* get_mc2_state - get the task's state */ +static struct mc2_task_state* get_mc2_state(struct task_struct *tsk) +{ + struct mc2_task_state* tinfo; + + tinfo = (struct mc2_task_state*)tsk_rt(tsk)->plugin_state; + + if (tinfo) + return tinfo; + else + return NULL; +} + +/* get_task_crit_level - return the criticaility level of a task */ +static enum crit_level get_task_crit_level(struct task_struct *tsk) +{ + struct mc2_task *mp; + + if (!tsk || !is_realtime(tsk)) + return NUM_CRIT_LEVELS; + + mp = tsk_rt(tsk)->mc2_data; + + if (!mp) + return NUM_CRIT_LEVELS; + else + return mp->crit; +} + +/* task_depart - remove a task from its reservation + * If the job has remaining budget, convert it to a ghost job + * and update crit_entries[] + * + * @job_complete indicate whether job completes or not + */ +static void task_departs(struct task_struct *tsk, int job_complete) +{ + struct mc2_task_state* tinfo = get_mc2_state(tsk); + struct reservation* res = NULL; + struct reservation_client *client = NULL; + int i; + BUG_ON(!is_realtime(tsk)); + + for(i = 0; i < NR_MODES; i++){ + if (! in_mode(tsk, i) && i != 0) + continue; + res = tinfo->res_info[i].client.reservation; + client = &tinfo->res_info[i].client; + BUG_ON(!res); + BUG_ON(!client); + + if (job_complete) + res->cur_budget = 0; + + res->ops->client_departs(res, client, job_complete); + } + + tinfo->has_departed = true; + TRACE_TASK(tsk, "CLIENT DEPART with budget %llu at %llu\n", res->cur_budget, litmus_clock()); +} + +/* task_arrive - put a task into its reservation + * If the job was a ghost job, remove it from crit_entries[] + */ +static void task_arrives(struct mc2_cpu_state *state, struct task_struct *tsk) +{ + struct mc2_task_state* tinfo = get_mc2_state(tsk); + struct reservation* res; + struct reservation_client *client; + enum crit_level lv = get_task_crit_level(tsk); + int i; + + switch(lv) { + case CRIT_LEVEL_A: + case CRIT_LEVEL_B: + TS_RELEASE_START; + break; + case CRIT_LEVEL_C: + TS_RELEASE_C_START; + break; + default: + break; + } + + tinfo->has_departed = false; + + TRACE_TASK(tsk, "CLIENT ARRIVES at %llu\n", litmus_clock()); + + for(i = 0; i < NR_MODES; i++){ + if (! in_mode(tsk, i) && i != 0) + continue; + res = tinfo->res_info[i].client.reservation; + client = &tinfo->res_info[i].client; + + res->ops->client_arrives(res, client); + } + + switch(lv) { + case CRIT_LEVEL_A: + case CRIT_LEVEL_B: + TS_RELEASE_END; + break; + case CRIT_LEVEL_C: + TS_RELEASE_C_END; + break; + default: + break; + } +} + +/* get_lowest_prio_cpu - return the lowest priority cpu + * This will be used for scheduling level-C tasks. + * If all CPUs are running tasks which has + * higher priority than level C, return NO_CPU. + */ +static int get_lowest_prio_cpu(lt_t priority) +{ + struct cpu_entry *ce; + int cpu, ret = NO_CPU; + lt_t latest_deadline = 0; + + if (priority == LITMUS_NO_PRIORITY) + return ret; + + ce = &_lowest_prio_cpu.cpu_entries[local_cpu_state()->cpu]; + if (!ce->will_schedule && !ce->scheduled) { + TRACE("CPU %d (local) is the lowest (Idle)!\n", ce->cpu); + return ce->cpu; + } else { + TRACE("Local CPU will_schedule=%d, scheduled=(%s/%d)\n", ce->will_schedule, ce->scheduled ? (ce->scheduled)->comm : "null", ce->scheduled ? (ce->scheduled)->pid : 0); + } + + for_each_online_cpu(cpu) { + ce = &_lowest_prio_cpu.cpu_entries[cpu]; + /* If a CPU will call schedule() in the near future, we don't + return that CPU. */ + + if (!ce->will_schedule) { + if (!ce->scheduled) { + /* Idle cpu, return this. */ + TRACE("CPU %d is the lowest (Idle)!\n", ce->cpu); + return ce->cpu; + } else if (ce->lv == CRIT_LEVEL_C && + ce->deadline > latest_deadline) { + latest_deadline = ce->deadline; + ret = ce->cpu; + } + } + } + + TRACE("CPU %d is the lowest! deadline = %llu, my priority = %llu\n", ret, latest_deadline, priority); + + if (priority >= latest_deadline) { + TRACE("CPU %d is running a higher-priority task. return NO_CPU\n", ret); + ret = NO_CPU; + } + + return ret; +} + +/* NOTE: drops state->lock */ +/* mc2_update_timer_and_unlock - set a timer and unlock state->lock. + * Whenever res_env.current_time is updated, + * we check next_scheduler_update and set + * a timer. + * If there exist a global event which is + * not armed on any CPU and g_timer is not + * active, set a g_timer for that event. + */ +static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state) +{ + int local, cpus; + lt_t update, now; + struct next_timer_event *event, *next; + int reschedule[NR_CPUS]; + unsigned long flags; + + local_irq_save(flags); + + for (cpus = 0; cpussup_env->next_scheduler_update; + now = state->sup_env->env.current_time; + + /* Be sure we're actually running on the right core, + * as pres_update_timer() is also called from pres_task_resume(), + * which might be called on any CPU when a thread resumes. + */ + local = local_cpu_state() == state; + + raw_spin_lock(&global_lock); + + list_for_each_entry_safe(event, next, &_global_env->next_events, list) { + /* If the event time is already passed, we call schedule() on + the lowest priority cpu */ + if (event->next_update >= update) { + break; + } + + if (event->next_update < litmus_clock()) { + if (event->timer_armed_on == NO_CPU) { + struct reservation *res = gmp_find_by_id(_global_env, event->id); + int cpu = get_lowest_prio_cpu(res?res->priority:LITMUS_NO_PRIORITY); + list_del(&event->list); + kfree(event); + if (cpu != NO_CPU) { + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; + reschedule[cpu] = 1; + } + } + } else if (event->next_update < update && (event->timer_armed_on == NO_CPU || event->timer_armed_on == state->cpu)) { + event->timer_armed_on = state->cpu; + update = event->next_update; + break; + } + } + + /* Must drop state lock before calling into hrtimer_start(), which + * may raise a softirq, which in turn may wake ksoftirqd. */ + + raw_spin_unlock(&global_lock); + local_irq_restore(flags); + raw_spin_unlock(&state->lock); + + if ((update <= now) || reschedule[state->cpu]) { + reschedule[state->cpu] = 0; + litmus_reschedule(state->cpu); + } else if (likely(local && update != SUP_NO_SCHEDULER_UPDATE)) { + /* Reprogram only if not already set correctly. */ + if (!hrtimer_active(&state->timer) || + ktime_to_ns(hrtimer_get_expires(&state->timer)) != update) { + TRACE("canceling timer...at %llu\n", + ktime_to_ns(hrtimer_get_expires(&state->timer))); + hrtimer_cancel(&state->timer); + TRACE("setting scheduler timer for %llu\n", update); + /* We cannot use hrtimer_start() here because the + * wakeup flag must be set to zero. */ + __hrtimer_start_range_ns(&state->timer, + ns_to_ktime(update), + 0 /* timer coalescing slack */, + HRTIMER_MODE_ABS_PINNED, + 0 /* wakeup */); + if (update < litmus_clock()) { + /* uh oh, timer expired while trying to set it */ + TRACE("timer expired during setting " + "update:%llu now:%llu actual:%llu\n", + update, now, litmus_clock()); + /* The timer HW may not have been reprogrammed + * correctly; force rescheduling now. */ + litmus_reschedule(state->cpu); + } + } + } else if (unlikely(!local && update != SUP_NO_SCHEDULER_UPDATE)) { + /* Poke remote core only if timer needs to be set earlier than + * it is currently set. + */ + TRACE("mc2_update_timer for remote CPU %d (update=%llu, " + "active:%d, set:%llu)\n", + state->cpu, update, hrtimer_active(&state->timer), + ktime_to_ns(hrtimer_get_expires(&state->timer))); + if (!hrtimer_active(&state->timer) || + ktime_to_ns(hrtimer_get_expires(&state->timer)) > update) { + TRACE("poking CPU %d so that it can update its " + "scheduling timer (active:%d, set:%llu)\n", + state->cpu, + hrtimer_active(&state->timer), + ktime_to_ns(hrtimer_get_expires(&state->timer))); + } + } +} + +/* update_cpu_prio - Update cpu's priority + * When a cpu picks a new task, call this function + * to update cpu priorities. + */ +static void update_cpu_prio(struct mc2_cpu_state *state) +{ + struct cpu_entry *ce = &_lowest_prio_cpu.cpu_entries[state->cpu]; + enum crit_level lv = get_task_crit_level(state->scheduled); + + if (!state->scheduled) { + /* cpu is idle. */ + ce->scheduled = NULL; + ce->deadline = ULLONG_MAX; + ce->lv = NUM_CRIT_LEVELS; + } else if (lv == CRIT_LEVEL_C) { + ce->scheduled = state->scheduled; + ce->deadline = get_deadline(state->scheduled); + ce->lv = lv; + } else if (lv < CRIT_LEVEL_C) { + /* If cpu is running level A or B tasks, it is not eligible + to run level-C tasks */ + ce->scheduled = state->scheduled; + ce->deadline = 0; + ce->lv = lv; + } +}; + +/* on_scheduling_timer - timer event for partitioned tasks + */ +static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer) +{ + unsigned long flags; + enum hrtimer_restart restart = HRTIMER_NORESTART; + struct mc2_cpu_state *state; + lt_t update, now; + int global_schedule_now; + int reschedule[NR_CPUS]; + int cpus; + + if (mode_changed) + return restart; + + for (cpus = 0; cpuscpu != raw_smp_processor_id()); + + TS_ISR_START; + + TRACE("Timer fired at %llu\n", litmus_clock()); + raw_spin_lock_irqsave(&state->lock, flags); + now = litmus_clock(); + sup_update_time(state->sup_env, now); + + update = state->sup_env->next_scheduler_update; + now = state->sup_env->env.current_time; + + if (update <= now) { + litmus_reschedule_local(); + } else if (update != SUP_NO_SCHEDULER_UPDATE) { + hrtimer_set_expires(timer, ns_to_ktime(update)); + restart = HRTIMER_RESTART; + } + + raw_spin_lock(&global_lock); + global_schedule_now = gmp_update_time(_global_env, litmus_clock()); + BUG_ON(global_schedule_now < 0 || global_schedule_now > 4); + + /* Find the lowest cpu, and call reschedule */ + while (global_schedule_now--) { + int cpu = get_lowest_prio_cpu(0); + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) { + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; + if (cpu == state->cpu && update > now) + ; + else + reschedule[cpu] = 1; + } + } + raw_spin_unlock(&global_lock); + raw_spin_unlock_irqrestore(&state->lock, flags); + + TS_ISR_END; + + for (cpus = 0; cpuscompleted = 1; + raw_spin_unlock(&mode_lock); + + if (atomic_read(&num_sync_released) == 0 && mode != 0) { + tsk_rt(current)->sporadic_release = 0; + TRACE_CUR("num_sync_released is 0\n"); + } + + /* If this the first job instance, we need to reset replenish + time to the next release time */ + if (tsk_rt(current)->sporadic_release) { + struct mc2_cpu_state *state; + struct mc2_task_state *tinfo; + struct reservation *res = NULL; + unsigned long flags; + + local_irq_save(flags); + + tinfo = get_mc2_state(current); + + if (lv < CRIT_LEVEL_C) { + int i; + state = cpu_state_for(tinfo->cpu); + raw_spin_lock(&state->lock); + for (i = 0; isup_env_modes[i].env.time_zero = tsk_rt(current)->sporadic_release_time + INIT_PHASE_LENGTH_NS*(tsk_rt(current)->job_params.job_no); + } + } + res = sup_find_by_id(state->sup_env, tinfo->mc2_param.res_id); + + } + else if (lv == CRIT_LEVEL_C) { + int i; + state = local_cpu_state(); + raw_spin_lock(&state->lock); + raw_spin_lock(&global_lock); + for (i = 0; i < NR_MODES; i++) { + if (in_mode(current,i) || i == 0) { + _global_env_modes[i].env.time_zero = tsk_rt(current)->sporadic_release_time + INIT_PHASE_LENGTH_NS*(tsk_rt(current)->job_params.job_no); + } + } + res = gmp_find_by_id(_global_env, tinfo->mc2_param.res_id); + } + else + BUG(); + + /* set next_replenish to synchronous release time */ + BUG_ON(!res); + res->next_replenishment = tsk_rt(current)->sporadic_release_time + INIT_PHASE_LENGTH_NS*(tsk_rt(current)->job_params.job_no); +/* + if (get_task_crit_level(current) == CRIT_LEVEL_A) { + struct table_driven_reservation *tdres; + tdres = container_of(res, struct table_driven_reservation, res); + tdres->next_interval = 0; + tdres->major_cycle_start = tsk_rt(current)->sporadic_release_time; + res->next_replenishment += tdres->intervals[0].start; + } +*/ + res->cur_budget = 0; + res->env->change_state(res->env, res, RESERVATION_DEPLETED); + + TRACE_CUR("CHANGE NEXT_REP = %llu NEXT_UPDATE = %llu\n", res->next_replenishment, state->sup_env->next_scheduler_update); + if (lv == CRIT_LEVEL_C){ + raw_spin_unlock(&global_lock); + } + raw_spin_unlock(&state->lock); + local_irq_restore(flags); + } + + sched_trace_task_completion(current, 0); + /* update the next release time and deadline */ + prepare_for_next_period(current); + sched_trace_task_release(current); + next_release = ns_to_ktime(get_release(current)); + preempt_disable(); + TRACE_CUR("next_release=%llu\n", get_release(current)); + + /* + * Changed logic for mode switch case + * In case of mode switch, do not want to release + * new job even if release time has passed + */ + + if (lv == CRIT_LEVEL_C && pending) { + struct reservation *res = NULL; + + raw_spin_lock(&mode_lock); + res = gmp_find_by_id(_global_env, tsk_mc2_data(current)->res_id); + if (res && !res->reported){ + res_reported--; + TRACE_CUR("RES_REPORTED = %d\n", res_reported); + res->reported = 1; + } + raw_spin_unlock(&mode_lock); + } + + + if (get_release(current) > litmus_clock()) { + /* sleep until next_release */ + set_current_state(TASK_INTERRUPTIBLE); + preempt_enable_no_resched(); + TRACE_CUR("Sleep until %llu\n", next_release); + err = schedule_hrtimeout(&next_release, HRTIMER_MODE_ABS); + } else { + /* release the next job immediately */ + err = 0; + TRACE_CUR("TARDY: release=%llu now=%llu\n", get_release(current), litmus_clock()); + preempt_enable(); + } + + TRACE_CUR("mc2_complete_job returns at %llu\n", litmus_clock()); + + raw_spin_lock(&mode_lock); + tsk_rt(current)->completed = 0; + raw_spin_unlock(&mode_lock); + + return err; +} + +/* mc2_dispatch - Select the next task to schedule. + */ +struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, struct mc2_cpu_state* state) +{ + struct reservation *res, *next; + struct task_struct *tsk = NULL; + enum crit_level lv; + lt_t time_slice; + + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) { + if (res->state == RESERVATION_ACTIVE) { + tsk = res->ops->dispatch_client(res, &time_slice); + if (likely(tsk)) { + lv = get_task_crit_level(tsk); + if (lv == NUM_CRIT_LEVELS) { + sup_scheduler_update_after(sup_env, res->cur_budget); + return tsk; + } else { + sup_scheduler_update_after(sup_env, res->cur_budget); + res->blocked_by_ghost = 0; + res->is_ghost = NO_CPU; + return tsk; + } + } + } + } + + return NULL; +} + +struct task_struct* mc2_global_dispatch(struct mc2_cpu_state* state) +{ + struct reservation *res, *next; + struct task_struct *tsk = NULL; + enum crit_level lv; + lt_t time_slice; + + list_for_each_entry_safe(res, next, &_global_env->active_reservations, list) { + BUG_ON(!res); + if (res->state == RESERVATION_ACTIVE && res->scheduled_on == NO_CPU) { + tsk = res->ops->dispatch_client(res, &time_slice); + if (pending && res->reported && !in_mode(tsk, requested_mode)){ + TRACE_TASK(tsk, "Rejected because task not in requested mode %d\n", requested_mode); + continue; + } + if (likely(tsk)) { + lv = get_task_crit_level(tsk); + if (lv != CRIT_LEVEL_C) + BUG(); +#if BUDGET_ENFORCEMENT_AT_C + gmp_add_event_after(_global_env, res->cur_budget, res->id, EVENT_DRAIN); +#endif + res->event_added = 1; + res->blocked_by_ghost = 0; + res->is_ghost = NO_CPU; + TRACE_TASK(res->tsk, "R%d global dispatched on %d\n", res->id, state->cpu); + res->scheduled_on = state->cpu; + return tsk; + } + } + } + return NULL; +} + +static inline void pre_schedule(struct task_struct *prev, int cpu) +{ + TS_SCHED_A_START; + TS_SCHED_C_START; + + if (!prev || !is_realtime(prev)) + return; + + do_partition(CRIT_LEVEL_C, cpu); +} + +static inline void post_schedule(struct task_struct *next, int cpu) +{ + enum crit_level lev; + if ((!next) || !is_realtime(next)) + return; + + lev = get_task_crit_level(next); + if (is_mode_poll_task(next)) { + lev = MODE_POLL_TASK; + } + + do_partition(lev, cpu); + + switch(lev) { + case CRIT_LEVEL_A: + case CRIT_LEVEL_B: + case MODE_POLL_TASK: + TS_SCHED_A_END(next); + break; + case CRIT_LEVEL_C: + TS_SCHED_C_END(next); + break; + default: + break; + } + +} + +/* mc2_schedule - main scheduler function. pick the next task to run + */ +static struct task_struct* mc2_schedule(struct task_struct * prev) +{ + int np, blocks, exists; + /* next == NULL means "schedule background work". */ + lt_t now = litmus_clock(); + struct mc2_cpu_state *state = local_cpu_state(); + + raw_spin_lock(&state->lock); + + pre_schedule(prev, state->cpu); + + BUG_ON(state->scheduled && state->scheduled != prev); + BUG_ON(state->scheduled && !is_realtime(prev)); + + /* (0) Determine state */ + exists = state->scheduled != NULL; + blocks = exists && !is_current_running(); + np = exists && is_np(state->scheduled); + + /* update time */ + state->sup_env->will_schedule = true; + sup_update_time(state->sup_env, now); + + if (is_realtime(current) && blocks) { + if (get_task_crit_level(current) == CRIT_LEVEL_C){ + raw_spin_lock(&global_lock); + } + task_departs(current, is_completed(current)); + if (get_task_crit_level(current) == CRIT_LEVEL_C){ + raw_spin_unlock(&global_lock); + } + } + + /* figure out what to schedule next */ + if (!np) + state->scheduled = mc2_dispatch(state->sup_env, state); + + if (!state->scheduled) { + raw_spin_lock(&global_lock); + if (is_realtime(prev)) + gmp_update_time(_global_env, now); + state->scheduled = mc2_global_dispatch(state); + _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false; + update_cpu_prio(state); + raw_spin_unlock(&global_lock); + } else { + raw_spin_lock(&global_lock); + _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false; + update_cpu_prio(state); + raw_spin_unlock(&global_lock); + } + + /* Notify LITMUS^RT core that we've arrived at a scheduling decision. */ + sched_state_task_picked(); + + /* program scheduler timer */ + state->sup_env->will_schedule = false; + + /* NOTE: drops state->lock */ + mc2_update_timer_and_unlock(state); + + raw_spin_lock(&state->lock); + if (prev != state->scheduled && is_realtime(prev)) { + struct mc2_task_state* tinfo = get_mc2_state(prev); + struct reservation* res = tinfo->res_info[mode].client.reservation; + if (res) { + res->scheduled_on = NO_CPU; + } + TRACE_TASK(prev, "descheduled at %llu.\n", litmus_clock()); + /* if prev is preempted and a global task, find the lowest cpu and reschedule */ + if (tinfo->has_departed == false && get_task_crit_level(prev) == CRIT_LEVEL_C) { + int cpu; + raw_spin_lock(&global_lock); + cpu = get_lowest_prio_cpu(res?res->priority:LITMUS_NO_PRIORITY); + if (cpu != NO_CPU && _lowest_prio_cpu.cpu_entries[cpu].will_schedule == false) { + _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true; + resched_cpu[cpu] = 1; + } + raw_spin_unlock(&global_lock); + } + } + + post_schedule(state->scheduled, state->cpu); + + raw_spin_unlock(&state->lock); + if (state->scheduled) { + TRACE_TASK(state->scheduled, "scheduled.\n"); + } + + return state->scheduled; +} + +static void resume_legacy_task_model_updates(struct task_struct *tsk) +{ + lt_t now; + if (is_sporadic(tsk)) { + /* If this sporadic task was gone for a "long" time and woke up past + * its deadline, then give it a new budget by triggering a job + * release. This is purely cosmetic and has no effect on the + * MC2 scheduler. */ + + now = litmus_clock(); + if (is_tardy(tsk, now)) { + release_at(tsk, now); + } + } +} + +/* mc2_task_resume - Called when the state of tsk changes back to + * TASK_RUNNING. We need to requeue the task. + */ +static void mc2_task_resume(struct task_struct *tsk) +{ + unsigned long flags; + struct mc2_task_state* tinfo; + struct mc2_cpu_state *state; + + TRACE_TASK(tsk, "thread wakes up at %llu\n", litmus_clock()); + + preempt_disable(); + tinfo = get_mc2_state(tsk); + if (tinfo->cpu != -1) + state = cpu_state_for(tinfo->cpu); + else + state = local_cpu_state(); + preempt_enable(); + + /* Requeue only if self-suspension was already processed. */ + if (tinfo->has_departed) + { +#ifdef CONFIG_SCHED_OVERHEAD_TRACE + switch(get_task_crit_level(tsk)) { + case CRIT_LEVEL_A: + TS_RELEASE_LATENCY_A(get_release(tsk)); + break; + case CRIT_LEVEL_B: + TS_RELEASE_LATENCY_B(get_release(tsk)); + break; + case CRIT_LEVEL_C: + TS_RELEASE_LATENCY_C(get_release(tsk)); + break; + default: + break; + } +#endif + /* We don't want to consider jobs in the initialization mode */ + if (tsk_rt(tsk)->job_params.job_no == 2) { + tsk_mc2_data(tsk)->init_finished = 1; + atomic_dec(&num_sync_released); + + if (atomic_read(&num_sync_released) == 0) { + lt_t start = tsk_rt(tsk)->sporadic_release_time + INIT_PHASE_LENGTH_NS*(tsk_rt(tsk)->job_params.job_no); + sys_request_mode(1); + sched_trace_sys_start(&start); + } + } + + raw_spin_lock_irqsave(&state->lock, flags); + + /* Assumption: litmus_clock() is synchronized across cores, + * since we might not actually be executing on tinfo->cpu + * at the moment. */ + if (tinfo->cpu != -1) { + sup_update_time(state->sup_env, litmus_clock()); + task_arrives(state, tsk); + } else { + raw_spin_lock(&global_lock); + gmp_update_time(_global_env, litmus_clock()); + task_arrives(state, tsk); + raw_spin_unlock(&global_lock); + } + + /* NOTE: drops state->lock */ + TRACE_TASK(tsk, "mc2_resume()\n"); + raw_spin_unlock_irqrestore(&state->lock, flags); + + raw_spin_lock(&state->lock); + mc2_update_timer_and_unlock(state); + } else { + TRACE_TASK(tsk, "resume event ignored, still scheduled\n"); + } + + resume_legacy_task_model_updates(tsk); +} + + +/* mc2_admit_task - Setup mc2 task parameters + */ +static long mc2_admit_task(struct task_struct *tsk) +{ + long err = 0; + unsigned long flags; + struct reservation *res; + struct mc2_cpu_state *state; + struct mc2_task_state *tinfo = kzalloc(sizeof(*tinfo), GFP_ATOMIC); + struct mc2_task *mp = tsk_rt(tsk)->mc2_data; + enum crit_level lv; + int i; + + TRACE_TASK(tsk, "MC2 admitting task\n"); + if (!tinfo) + return -ENOMEM; + + if (!mp) { + TRACE("mc2_admit_task: criticality level has not been set\n"); + return -ESRCH; + } + + lv = mp->crit; + + if (lv < CRIT_LEVEL_C) { + state = cpu_state_for(task_cpu(tsk)); + raw_spin_lock_irqsave(&state->lock, flags); + + tinfo->mc2_param.crit = mp->crit; + tinfo->cpu = task_cpu(tsk); + tinfo->has_departed = true; + tinfo->mc2_param.res_id = mp->res_id; + tinfo->mc2_param.mode_mask = mp->mode_mask; + tinfo->mc2_param.init_finished = 0; + + res = sup_find_by_id(&(state->sup_env_modes[0]), mp->res_id); + + /* found the appropriate reservation */ + if (res) { + /* initial values */ + err = err? err:mc2_task_client_init(&tinfo->res_info[0], &tinfo->mc2_param, tsk, res); + } + else { + //failed to find an expected reservation + err = -ESRCH; + } + + for(i = 1; i < NR_MODES; i++){ + if (!in_mode(tsk, i)){ + // task not present in mode + continue; + } + res = sup_find_by_id(&(state->sup_env_modes[i]), mp->res_id); + + /* found the appropriate reservation */ + if (res) { + /* initial values */ + err = err? err:mc2_task_client_init(&tinfo->res_info[i], &tinfo->mc2_param, tsk, res); + } + else{ + //failed to find an expected reservation + err = -ESRCH; + } + } + + if (!err){ + /* disable LITMUS^RT's per-thread budget enforcement */ + tsk_rt(tsk)->plugin_state = tinfo; + tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT; + } + + if (is_mode_poll_task(tsk) && tinfo->cpu == 0) { + cpu_0_task_exist = true; + } + atomic_inc(&num_sync_released); + raw_spin_unlock_irqrestore(&state->lock, flags); + } else if (lv == CRIT_LEVEL_C) { + state = local_cpu_state(); + raw_spin_lock_irqsave(&state->lock, flags); + + tinfo->mc2_param.crit = mp->crit; + tinfo->cpu = -1; + tinfo->has_departed = true; + tinfo->mc2_param.res_id = mp->res_id; + tinfo->mc2_param.mode_mask = mp->mode_mask; + tinfo->mc2_param.init_finished = 0; + + raw_spin_lock(&global_lock); + res = gmp_find_by_id(&(_global_env_modes[0]), mp->res_id); + + /* found the appropriate reservation */ + if (res) { + /* initial values */ + err = err? err:mc2_task_client_init(&tinfo->res_info[0], &tinfo->mc2_param, tsk, res); + } + else { + //failed to find an expected reservation + err = -ESRCH; + } + + for(i = 1; i < NR_MODES; i++){ + if (!in_mode(tsk, i)) + continue; + res = gmp_find_by_id(&(_global_env_modes[i]), mp->res_id); + + /* found the appropriate reservation (or vCPU) */ + if (res) { + /* initial values */ + err = err? err:mc2_task_client_init(&tinfo->res_info[i], &tinfo->mc2_param, tsk, res); + + } + } + raw_spin_unlock(&global_lock); + + if (!err){ + /* disable LITMUS^RT's per-thread budget enforcement */ + tsk_rt(tsk)->plugin_state = tinfo; + tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT; + raw_spin_lock(&mode_lock); + for(i = 1; i < NR_MODES; i++){ + if (in_mode(tsk, i)){ + mode_sizes[i]++; + } + } + mode_sizes[0]++; + raw_spin_unlock(&mode_lock); + + } + + atomic_inc(&num_sync_released); + raw_spin_unlock_irqrestore(&state->lock, flags); + } + + if (err) + kfree(tinfo); + + TRACE_TASK(tsk, "MC2 task admitted %d\n", err); + return err; +} + +/* mc2_task_new - A new real-time job is arrived. Release the next job + * at the next reservation replenish time + */ +static void mc2_task_new(struct task_struct *tsk, int on_runqueue, + int is_running) +{ + unsigned long flags; + struct mc2_task_state* tinfo = get_mc2_state(tsk); + struct mc2_cpu_state *state; + struct reservation *res; + enum crit_level lv = get_task_crit_level(tsk); + lt_t release = 0; + + BUG_ON(lv < CRIT_LEVEL_A || lv > CRIT_LEVEL_C); + + TRACE_TASK(tsk, "new RT task %llu (on_rq:%d, running:%d)\n", + litmus_clock(), on_runqueue, is_running); + + if (tinfo->cpu == -1) + state = local_cpu_state(); + else + state = cpu_state_for(tinfo->cpu); + + + if (is_running) { + state->scheduled = tsk; + /* make sure this task should actually be running */ + litmus_reschedule_local(); + } + + /* acquire the lock protecting the state and disable interrupts */ + local_irq_save(flags); + raw_spin_lock(&state->lock); + + if (lv == CRIT_LEVEL_C) { + raw_spin_lock(&global_lock); + res = gmp_find_by_id(_global_env, tinfo->mc2_param.res_id); + } + else { + res = sup_find_by_id(state->sup_env, tinfo->mc2_param.res_id); + } + BUG_ON(!res); + + // the current mode doesn't have this task. + // do not update timer and set the next release time. + + if (on_runqueue || is_running) { + /* Assumption: litmus_clock() is synchronized across cores + * [see comment in pres_task_resume()] */ + if (lv == CRIT_LEVEL_C) { + gmp_update_time(_global_env, litmus_clock()); + } + else + sup_update_time(state->sup_env, litmus_clock()); + + task_arrives(state, tsk); + if (lv == CRIT_LEVEL_C){ + raw_spin_unlock(&global_lock); + } + /* NOTE: drops state->lock */ + raw_spin_unlock(&state->lock); + local_irq_restore(flags); + + raw_spin_lock(&state->lock); + mc2_update_timer_and_unlock(state); + } else { + if (lv == CRIT_LEVEL_C){ + raw_spin_unlock(&global_lock); + } + raw_spin_unlock(&state->lock); + local_irq_restore(flags); + } + release = res->next_replenishment; + + if (!release) { + BUG(); + } + else + TRACE_TASK(tsk, "mc2_task_new() next_release = %llu\n", release); +} + +/* mc2_reservation_destroy - reservation_destroy system call backend + */ +static long mc2_reservation_destroy(unsigned int reservation_id, int cpu) +{ + long ret = -EINVAL; + struct mc2_cpu_state *state; + struct reservation *res = NULL, *next; + struct sup_reservation_environment *sup_env; + int found = 0; + unsigned long flags; + int i; + + if (cpu == -1) { + struct next_timer_event *event, *e_next; + local_irq_save(flags); + raw_spin_lock(&global_lock); + + /* if the reservation is global reservation, + * delete reservation id in all modes */ + for(i = 0; i < NR_MODES; i++) { + list_for_each_entry_safe(res, next, &_global_env_modes[i].depleted_reservations, list) { + if (res->id == reservation_id) { + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + if (!found) { + list_for_each_entry_safe(res, next, &_global_env_modes[i].inactive_reservations, list) { + if (res->id == reservation_id) { + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + } + if (!found) { + list_for_each_entry_safe(res, next, &_global_env_modes[i].active_reservations, list) { + if (res->id == reservation_id) { + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + } + + list_for_each_entry_safe(event, e_next, &_global_env_modes[i].next_events, list) { + if (event->id == reservation_id) { + list_del(&event->list); + kfree(event); + } + } + } + + raw_spin_unlock(&global_lock); + local_irq_restore(flags); + } else { + /* if the reservation is partitioned reservation */ + state = cpu_state_for(cpu); + for (i = 0; i < NR_MODES; i++){ + local_irq_save(flags); + raw_spin_lock(&state->lock); + sup_env = &(state->sup_env_modes[i]); + list_for_each_entry_safe(res, next, &sup_env->depleted_reservations, list) { + if (res->id == reservation_id) { +/* + if (lv == CRIT_LEVEL_A) { + struct table_driven_reservation *tdres; + tdres = container_of(res, struct table_driven_reservation, res); + kfree(tdres->intervals); + } +*/ + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + if (!found) { + list_for_each_entry_safe(res, next, &sup_env->inactive_reservations, list) { + if (res->id == reservation_id) { +/* if (lv == CRIT_LEVEL_A) { + struct table_driven_reservation *tdres; + tdres = container_of(res, struct table_driven_reservation, res); + kfree(tdres->intervals); + } +*/ + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + } + if (!found) { + list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) { + if (res->id == reservation_id) { +/* if (lv == CRIT_LEVEL_A) { + struct table_driven_reservation *tdres; + tdres = container_of(res, struct table_driven_reservation, res); + kfree(tdres->intervals); + } +*/ + list_del(&res->list); + kfree(res); + found = 1; + ret = 0; + } + } + } + + raw_spin_unlock(&state->lock); + local_irq_restore(flags); + } + } + + TRACE("Rerservation destroyed ret = %d\n", ret); + return ret; +} + +/* mc2_task_exit - Task became a normal task (not real-time task) + */ +static void mc2_task_exit(struct task_struct *tsk) +{ + unsigned long flags; + struct mc2_task_state* tinfo = get_mc2_state(tsk); + struct mc2_cpu_state *state; + enum crit_level lv = tinfo->mc2_param.crit; + int cpu, i; + + local_irq_save(flags); + if (tinfo->cpu != -1) + state = cpu_state_for(tinfo->cpu); + else + state = local_cpu_state(); + + raw_spin_lock(&state->lock); + + if (state->scheduled == tsk) + state->scheduled = NULL; + + /* remove from queues */ + if (is_running(tsk)) { + /* Assumption: litmus_clock() is synchronized across cores + * [see comment in pres_task_resume()] */ + + /* update both global and partitioned */ + if (lv < CRIT_LEVEL_C) { + sup_update_time(state->sup_env, litmus_clock()); + } + else if (lv == CRIT_LEVEL_C) { + raw_spin_lock(&global_lock); + gmp_update_time(_global_env, litmus_clock()); + } + task_departs(tsk, 0); + if (lv == CRIT_LEVEL_C){ + raw_spin_unlock(&global_lock); + } + atomic_dec(&num_sync_released); + + /* NOTE: drops state->lock */ + mc2_update_timer_and_unlock(state); + } else { + raw_spin_unlock(&state->lock); + } + + if (lv == CRIT_LEVEL_C) { + raw_spin_lock(&mode_lock); + for(i = 1; i < NR_MODES; i++){ + if ( !(tsk_mc2_data(tsk)->mode_mask & (1<lock); + + if (state->scheduled == tsk) + state->scheduled = NULL; + + raw_spin_unlock(&state->lock); + } + } + + local_irq_restore(flags); + + if (is_mode_poll_task(tsk)) { + mode_poll_exited = true; + } + + kfree(tsk_rt(tsk)->plugin_state); + tsk_rt(tsk)->plugin_state = NULL; + kfree(tsk_rt(tsk)->mc2_data); + tsk_rt(tsk)->mc2_data = NULL; +} + +/* create_polling_reservation - create a new polling reservation + */ +static long create_polling_reservation( + int res_type, + struct reservation_config *config) +{ + struct mc2_cpu_state *state = NULL; + struct polling_reservation *pres; + unsigned long flags; + int use_edf = config->priority == LITMUS_NO_PRIORITY; + int periodic = res_type == PERIODIC_POLLING; + long err = -EINVAL; + bool resExist = false; + + /* sanity checks */ + if (config->polling_params.budget > + config->polling_params.period) { + printk(KERN_ERR "invalid polling reservation (%u): " "budget > period\n", config->id); + return -EINVAL; + } + if (config->polling_params.budget > + config->polling_params.relative_deadline + && config->polling_params.relative_deadline) { + printk(KERN_ERR "invalid polling reservation (%u): " + "budget > deadline\n", config->id); + return -EINVAL; + } + if (config->polling_params.offset > + config->polling_params.period) { + printk(KERN_ERR "invalid polling reservation (%u): " + "offset > period\n", config->id); + return -EINVAL; + } + //Added sanity check for mode + if (config->mode < 0 || config->mode >= NR_MODES){ + printk(KERN_ERR "invalid polling reservation (%u): " + "Mode outside range\n", config->id); + return -EINVAL; + } + + /* Allocate before we grab a spin lock. + * Todo: would be nice to use a core-local allocation. + */ + pres = kzalloc(sizeof(*pres), GFP_KERNEL); + if (!pres) + return -ENOMEM; + + TRACE("CREATE_POLLING_RESERVATION id %d mode %d\n", config->id, config->mode); + if (config->cpu != -1) { + int i, is_exist = 0; + + state = cpu_state_for(config->cpu); + raw_spin_lock_irqsave(&state->lock, flags); + + /* check if it is the first creation of reservartion */ + for (i = 0; i < NR_MODES; i++) { + if( sup_find_by_id(&(state->sup_env_modes[i]), config->id) ) + is_exist = 1; + } + if (!is_exist && config->mode != 0) { + /* create mode 0 reservation first */ + struct polling_reservation *pres_0 = kzalloc(sizeof(*pres_0), GFP_ATOMIC); + + TRACE_CUR("The first mode_num = %d\n",config->mode); + + if (!pres_0) { + raw_spin_unlock_irqrestore(&state->lock, flags); + kfree(pres); + return -ENOMEM; + } + polling_reservation_init(pres_0, use_edf, periodic, + config->polling_params.budget, + config->polling_params.period, + config->polling_params.relative_deadline, + config->polling_params.offset); + pres_0->res.id = config->id; + pres_0->res.blocked_by_ghost = 0; + pres_0->res.is_ghost = NO_CPU; + pres_0->res.mode = config->mode; + + if (!use_edf) + pres_0->res.priority = config->priority; + sup_add_new_reservation(&(state->sup_env_modes[0]), &pres_0->res); + TRACE_CUR("SUP reservation created R%d for mode 0 priority : %llu\n", config->id, pres_0->res.priority); + pres_0->res.reported = 0; + pres_0->res.tsk = current; + } + + //force reservation id unique inside of res_config->mode + if( sup_find_by_id(&(state->sup_env_modes[config->mode]), config->id) ){ + resExist = true; + } + if (!resExist) { + polling_reservation_init(pres, use_edf, periodic, + config->polling_params.budget, + config->polling_params.period, + config->polling_params.relative_deadline, + config->polling_params.offset); + pres->res.id = config->id; + pres->res.blocked_by_ghost = 0; + pres->res.is_ghost = NO_CPU; + pres->res.mode = config->mode; + if (!use_edf) + pres->res.priority = config->priority; + sup_add_new_reservation(&(state->sup_env_modes[config->mode]), &pres->res); + err = config->id; + TRACE_CUR("SUP reservation created R%d for mode %d priority : %llu\n", config->id, config->mode, pres->res.priority); + } else { + err = -EEXIST; + } + + raw_spin_unlock_irqrestore(&state->lock, flags); + + } else { + int i, is_exist = 0; + raw_spin_lock_irqsave(&global_lock, flags); + + /* check if it is the first creation of reservartion */ + for (i = 0; i < NR_MODES; i++) { + if(gmp_find_by_id(&(_global_env_modes[i]), config->id)) + is_exist = 1; + } + if (!is_exist && config->mode != 0) { + /* create mode 0 reservation first */ + struct polling_reservation *pres_0 = kzalloc(sizeof(*pres_0), GFP_ATOMIC); + + TRACE_CUR("The first mode_num = %d\n",config->mode); + + if (!pres_0) { + raw_spin_unlock_irqrestore(&global_lock, flags); + kfree(pres); + return -ENOMEM; + } + polling_reservation_init(pres_0, use_edf, periodic, + config->polling_params.budget, + config->polling_params.period, + config->polling_params.relative_deadline, + config->polling_params.offset); + pres_0->res.id = config->id; + pres_0->res.blocked_by_ghost = 0; + pres_0->res.scheduled_on = NO_CPU; + pres_0->res.is_ghost = NO_CPU; + pres_0->res.mode = config->mode; + + if (!use_edf) + pres_0->res.priority = config->priority; + gmp_add_new_reservation(&(_global_env_modes[0]), &pres_0->res); + TRACE_CUR("GMP reservation created R%d for mode 0 priority : %llu\n", config->id, pres_0->res.priority); + pres_0->res.reported = 0; + pres_0->res.tsk = current; + } + + //force id's unique within desired mode + if (gmp_find_by_id(&(_global_env_modes[config->mode]), config->id)){ + resExist = true; + } + if (!resExist) { + polling_reservation_init(pres, use_edf, periodic, + config->polling_params.budget, + config->polling_params.period, + config->polling_params.relative_deadline, + config->polling_params.offset); + pres->res.id = config->id; + pres->res.blocked_by_ghost = 0; + pres->res.scheduled_on = NO_CPU; + pres->res.is_ghost = NO_CPU; + pres->res.mode = config->mode; + if (!use_edf) + pres->res.priority = config->priority; + gmp_add_new_reservation(&(_global_env_modes[config->mode]), &pres->res); + TRACE_CUR("GMP reservation created R%d for mode %d priority : %llu\n", config->id, config->mode, pres->res.priority); + err = config->id; + } else { + err = -EEXIST; + } + raw_spin_unlock_irqrestore(&global_lock, flags); + } + + + pres->res.reported = 0; + pres->res.tsk = current; + + if (err < 0) + kfree(pres); + + return err; +} + +#define MAX_INTERVALS 1024 + +/* create_table_driven_reservation - create a table_driven reservation + */ +static long create_table_driven_reservation( + struct reservation_config *config) +{ + struct mc2_cpu_state *state; + //struct reservation* res = NULL; + struct table_driven_reservation *td_res = NULL; + struct lt_interval *slots = NULL; + size_t slots_size; + unsigned int i, num_slots; + unsigned long flags; + long err = -EINVAL; + bool resExist = false; + + if (!config->table_driven_params.num_intervals) { + printk(KERN_ERR "invalid table-driven reservation (%u): " + "no intervals\n", config->id); + return -EINVAL; + } + + if (config->table_driven_params.num_intervals > MAX_INTERVALS) { + printk(KERN_ERR "invalid table-driven reservation (%u): " + "too many intervals (max: %d)\n", config->id, MAX_INTERVALS); + return -EINVAL; + } + + if (config->mode >= NR_MODES || config->mode < 0){ + printk(KERN_ERR "invalid table-driven reservation (%u): " + "mode outside of range\n", config->id); + return -EINVAL; + } + + num_slots = config->table_driven_params.num_intervals; + slots_size = sizeof(slots[0]) * num_slots; + slots = kzalloc(slots_size, GFP_KERNEL); + if (!slots) + return -ENOMEM; + + td_res = kzalloc(sizeof(*td_res), GFP_KERNEL); + if (!td_res) + err = -ENOMEM; + else + err = copy_from_user(slots, + config->table_driven_params.intervals, slots_size); + + if (!err) { + /* sanity checks */ + for (i = 0; !err && i < num_slots; i++) + if (slots[i].end <= slots[i].start) { + printk(KERN_ERR + "invalid table-driven reservation (%u): " + "invalid interval %u => [%llu, %llu]\n", + config->id, i, + slots[i].start, slots[i].end); + err = -EINVAL; + } + + for (i = 0; !err && i + 1 < num_slots; i++) + if (slots[i + 1].start <= slots[i].end) { + printk(KERN_ERR + "invalid table-driven reservation (%u): " + "overlapping intervals %u, %u\n", + config->id, i, i + 1); + err = -EINVAL; + } + + if (slots[num_slots - 1].end > + config->table_driven_params.major_cycle_length) { + printk(KERN_ERR + "invalid table-driven reservation (%u): last " + "interval ends past major cycle %llu > %llu\n", + config->id, + slots[num_slots - 1].end, + config->table_driven_params.major_cycle_length); + err = -EINVAL; + } + } + + if (!err) { + state = cpu_state_for(config->cpu); + raw_spin_lock_irqsave(&state->lock, flags); + + //force unique id's across all modes + for(i = 0; i < NR_MODES; i++){ + if (sup_find_by_id(&(state->sup_env_modes[i]), config->id)){ + resExist = true; + break; + } + } + if (!resExist) { + table_driven_reservation_init(td_res, + config->table_driven_params.major_cycle_length, + slots, num_slots); + td_res->res.id = config->id; + td_res->res.priority = config->priority; + td_res->res.blocked_by_ghost = 0; + td_res->res.mode = config->mode; + sup_add_new_reservation(&(state->sup_env_modes[config->mode]), &td_res->res); + err = config->id; + } else { + err = -EEXIST; + } + + raw_spin_unlock_irqrestore(&state->lock, flags); + } + + td_res->res.reported = 0; + td_res->res.tsk = current; + + if (err < 0) { + kfree(slots); + kfree(td_res); + } + + return err; +} + +/* mc2_reservation_create - reservation_create system call backend + */ +static long mc2_reservation_create(int res_type, void* __user _config) +{ + long ret = -EINVAL; + struct reservation_config config; + + TRACE("Attempt to create reservation (%d)\n", res_type); + + if (copy_from_user(&config, _config, sizeof(config))) + return -EFAULT; + + TRACE("Attempt to create reservation id %d mode %d\n", config.id, config.mode); + + if (config.cpu != -1) { + if (config.cpu < 0 || !cpu_online(config.cpu)) { + printk(KERN_ERR "invalid polling reservation (%u): " + "CPU %d offline\n", config.id, config.cpu); + return -EINVAL; + } + } + + switch (res_type) { + case PERIODIC_POLLING: + case SPORADIC_POLLING: + ret = create_polling_reservation(res_type, &config); + break; + + case TABLE_DRIVEN: + ret = create_table_driven_reservation(&config); + break; + + default: + return -EINVAL; + }; + + return ret; +} + +static struct domain_proc_info mc2_domain_proc_info; + +static long mc2_get_domain_proc_info(struct domain_proc_info **ret) +{ + *ret = &mc2_domain_proc_info; + return 0; +} + +static void mc2_setup_domain_proc(void) +{ + int i, cpu; + int num_rt_cpus = num_online_cpus(); + + struct cd_mapping *cpu_map, *domain_map; + + memset(&mc2_domain_proc_info, sizeof(mc2_domain_proc_info), 0); + init_domain_proc_info(&mc2_domain_proc_info, num_rt_cpus, num_rt_cpus); + mc2_domain_proc_info.num_cpus = num_rt_cpus; + mc2_domain_proc_info.num_domains = num_rt_cpus; + + i = 0; + for_each_online_cpu(cpu) { + cpu_map = &mc2_domain_proc_info.cpu_to_domains[i]; + domain_map = &mc2_domain_proc_info.domain_to_cpus[i]; + + cpu_map->id = cpu; + domain_map->id = i; + cpumask_set_cpu(i, cpu_map->mask); + cpumask_set_cpu(cpu, domain_map->mask); + ++i; + } +} + +static long mc2_activate_plugin(void) +{ + int cpu; + struct mc2_cpu_state *state; + struct cpu_entry *ce; + int i; + + for(i = 0; i < NR_MODES; i++){ + gmp_init(&(_global_env_modes[i])); + } + _global_env = &_global_env_modes[0]; + + raw_spin_lock_init(&mode_lock); + raw_spin_lock_init(&global_lock); + + seen_once = false; + + for_each_online_cpu(cpu) { + TRACE("Initializing CPU%d...\n", cpu); + + resched_cpu[cpu] = 0; + this_cpu_write(mode_counter, 0); + + state = cpu_state_for(cpu); + ce = &_lowest_prio_cpu.cpu_entries[cpu]; + + ce->cpu = cpu; + ce->scheduled = NULL; + ce->deadline = ULLONG_MAX; + ce->lv = NUM_CRIT_LEVELS; + ce->will_schedule = false; + + raw_spin_lock_init(&state->lock); + printk(KERN_ALERT "CPU%d state->lock %p\n", cpu, &state->lock); + state->cpu = cpu; + state->scheduled = NULL; + + for(i = 0; i < NR_MODES; i++){ + sup_init(&(state->sup_env_modes[i])); + } + state->sup_env = &(state->sup_env_modes[0]); + + hrtimer_init(&state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); + state->timer.function = on_scheduling_timer; + this_cpu_write(mode_counter, 0); + } + + mc2_setup_domain_proc(); + + mode_poll_exited = false; + + mode = 0; + requested_mode = 0; + + for(i = 0; i < NR_MODES; i++){ + mode_sizes[i] = 0; + } + res_reported = 0; + + return 0; +} + +static void mc2_finish_switch(struct task_struct *prev) +{ + int cpus; + enum crit_level lv = get_task_crit_level(prev); + struct mc2_cpu_state *state = local_cpu_state(); + + state->scheduled = is_realtime(current) ? current : NULL; + if (lv == CRIT_LEVEL_C) { + for (cpus = 0; cpuslock); + + hrtimer_cancel(&state->timer); + + ce = &_lowest_prio_cpu.cpu_entries[cpu]; + + ce->cpu = cpu; + ce->scheduled = NULL; + ce->deadline = ULLONG_MAX; + ce->lv = NUM_CRIT_LEVELS; + ce->will_schedule = false; + + + for(i = 0; i < NR_MODES; i++){ + /* Delete all reservations --- assumes struct reservation + * is prefix of containing struct. */ + state->sup_env = &(state->sup_env_modes[i]); + while (!list_empty(&state->sup_env->active_reservations)) { + res = list_first_entry( + &state->sup_env->active_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&state->sup_env->inactive_reservations)) { + res = list_first_entry( + &state->sup_env->inactive_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&state->sup_env->depleted_reservations)) { + res = list_first_entry( + &state->sup_env->depleted_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + } + + raw_spin_unlock(&state->lock); + } + + raw_spin_lock(&global_lock); + for(i = 0; i < NR_MODES; i++){ + _global_env = &_global_env_modes[i]; + while (!list_empty(&_global_env->active_reservations)) { + res = list_first_entry( + &_global_env->active_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&_global_env->inactive_reservations)) { + res = list_first_entry( + &_global_env->inactive_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&_global_env->depleted_reservations)) { + res = list_first_entry( + &_global_env->depleted_reservations, + struct reservation, list); + list_del(&res->list); + kfree(res); + } + + while (!list_empty(&_global_env->next_events)) { + event = list_first_entry( + &_global_env->next_events, + struct next_timer_event, list); + list_del(&event->list); + kfree(event); + } + + } + raw_spin_unlock(&global_lock); + + atomic_set(&num_sync_released, 0); + destroy_domain_proc_info(&mc2_domain_proc_info); + return 0; +} + +static struct sched_plugin mc2_plugin = { + .plugin_name = "MC2", + .schedule = mc2_schedule, + .finish_switch = mc2_finish_switch, + .task_wake_up = mc2_task_resume, + .admit_task = mc2_admit_task, + .task_new = mc2_task_new, + .task_exit = mc2_task_exit, + .complete_job = mc2_complete_job, + .get_domain_proc_info = mc2_get_domain_proc_info, + .activate_plugin = mc2_activate_plugin, + .deactivate_plugin = mc2_deactivate_plugin, + .reservation_create = mc2_reservation_create, + .reservation_destroy = mc2_reservation_destroy, +}; + +static int __init init_mc2(void) +{ + return register_sched_plugin(&mc2_plugin); +} + +module_init(init_mc2); diff --git litmus/sched_plugin.c litmus/sched_plugin.c index edd91e9..7b1eba0 100644 --- litmus/sched_plugin.c +++ litmus/sched_plugin.c @@ -13,6 +13,7 @@ #include #include #include +#include /* * Generic function to trigger preemption on either local or remote cpu @@ -197,6 +198,9 @@ int register_sched_plugin(struct sched_plugin* plugin) if (!plugin->wait_for_release_at) plugin->wait_for_release_at = default_wait_for_release_at; + if (!plugin->current_budget) + plugin->current_budget = litmus_current_budget; + raw_spin_lock(&sched_plugins_lock); list_add(&plugin->list, &sched_plugins); raw_spin_unlock(&sched_plugins_lock); diff --git litmus/sched_psn_edf.c litmus/sched_psn_edf.c index 2549a3f..76a57af 100644 --- litmus/sched_psn_edf.c +++ litmus/sched_psn_edf.c @@ -243,7 +243,7 @@ static struct task_struct* psnedf_schedule(struct task_struct * prev) if (next) { TRACE_TASK(next, "scheduled at %llu\n", litmus_clock()); } else { - TRACE("becoming idle at %llu\n", litmus_clock()); + ; //TRACE("becoming idle at %llu\n", litmus_clock()); } pedf->scheduled = next; @@ -644,10 +644,14 @@ static long psnedf_admit_task(struct task_struct* tsk) /* don't allow tasks on release master CPU */ && task_cpu(tsk) != remote_edf(task_cpu(tsk))->release_master #endif - ) + ) { + TRACE_TASK(tsk, "admitted\n"); return 0; - else + } + else { + TRACE_TASK(tsk, "not admitted\n"); return -EINVAL; + } } /* Plugin object */ diff --git litmus/sched_task_trace.c litmus/sched_task_trace.c index c154ec4..d844180 100644 --- litmus/sched_task_trace.c +++ litmus/sched_task_trace.c @@ -189,6 +189,7 @@ feather_callback void do_sched_trace_task_completion(unsigned long id, if (rec) { rec->data.completion.when = now(); rec->data.completion.forced = forced; + rec->data.completion.exec_time = get_exec_time(t); put_record(rec); } } @@ -240,3 +241,39 @@ feather_callback void do_sched_trace_action(unsigned long id, put_record(rec); } } + +feather_callback void do_sched_trace_enact_mode(unsigned long id, + unsigned long _task) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_ENACT_MODE, t); + + if (rec) { + rec->data.enact_mode.when = now(); + put_record(rec); + } +} + +feather_callback void do_sched_trace_request_mode(unsigned long id, + unsigned long _task) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_REQUEST_MODE, t); + + if (rec) { + rec->data.request_mode.when = now(); + put_record(rec); + } +} + +feather_callback void do_sched_trace_sys_start(unsigned long id, + unsigned long _start) +{ + lt_t *start = (lt_t*) _start; + struct st_event_record* rec = get_record(ST_SYS_START, NULL); + if (rec) { + rec->data.sys_start.when = now(); + rec->data.sys_start.start = *start; + put_record(rec); + } +} \ No newline at end of file diff --git litmus/sync.c litmus/sync.c index 5d18060..7733f67 100644 --- litmus/sync.c +++ litmus/sync.c @@ -16,6 +16,8 @@ #include +atomic_t num_sync_released; + struct ts_release_wait { struct list_head list; struct completion completion; @@ -147,6 +149,6 @@ asmlinkage long sys_release_ts(lt_t __user *__delay) start_time *= ONE_MS; ret = do_release_ts(start_time + delay); } - + atomic_set(&num_sync_released, ret); return ret; }