Attachment 'gpu-klmirqd-litmus-rt-ecrts12.patch'

Download

   1 diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
   2 index 91fd0c7..433cd15 100644
   3 --- a/arch/x86/kernel/irq.c
   4 +++ b/arch/x86/kernel/irq.c
   5 @@ -8,6 +8,10 @@
   6  #include <linux/smp.h>
   7  #include <linux/ftrace.h>
   8  
   9 +#ifdef CONFIG_LITMUS_NVIDIA
  10 +#include <litmus/sched_trace.h>
  11 +#endif
  12 +
  13  #include <asm/apic.h>
  14  #include <asm/io_apic.h>
  15  #include <asm/irq.h>
  16 diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S
  17 index 3770290..b5ddae4 100644
  18 --- a/arch/x86/kernel/syscall_table_32.S
  19 +++ b/arch/x86/kernel/syscall_table_32.S
  20 @@ -352,3 +352,4 @@ ENTRY(sys_call_table)
  21  	.long sys_wait_for_ts_release
  22  	.long sys_release_ts
  23  	.long sys_null_call
  24 +	.long sys_register_nv_device
  25 diff --git a/include/linux/completion.h b/include/linux/completion.h
  26 index c63950e..3ce20dd 100644
  27 --- a/include/linux/completion.h
  28 +++ b/include/linux/completion.h
  29 @@ -76,6 +76,7 @@ static inline void init_completion(struct completion *x)
  30  	init_waitqueue_head(&x->wait);
  31  }
  32  
  33 +extern void __wait_for_completion_locked(struct completion *);
  34  extern void wait_for_completion(struct completion *);
  35  extern int wait_for_completion_interruptible(struct completion *x);
  36  extern int wait_for_completion_killable(struct completion *x);
  37 diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
  38 index a0384a4..a2f2880 100644
  39 --- a/include/linux/interrupt.h
  40 +++ b/include/linux/interrupt.h
  41 @@ -459,6 +459,10 @@ struct tasklet_struct
  42  	atomic_t count;
  43  	void (*func)(unsigned long);
  44  	unsigned long data;
  45 +
  46 +#if defined(CONFIG_LITMUS_SOFTIRQD) || defined(CONFIG_LITMUS_PAI_SOFTIRQD)
  47 +	struct task_struct *owner;
  48 +#endif
  49  };
  50  
  51  #define DECLARE_TASKLET(name, func, data) \
  52 @@ -496,6 +500,7 @@ static inline void tasklet_unlock_wait(struct tasklet_struct *t)
  53  #define tasklet_unlock(t) do { } while (0)
  54  #endif
  55  
  56 +extern void ___tasklet_schedule(struct tasklet_struct *t);
  57  extern void __tasklet_schedule(struct tasklet_struct *t);
  58  
  59  static inline void tasklet_schedule(struct tasklet_struct *t)
  60 @@ -504,6 +509,7 @@ static inline void tasklet_schedule(struct tasklet_struct *t)
  61  		__tasklet_schedule(t);
  62  }
  63  
  64 +extern void ___tasklet_hi_schedule(struct tasklet_struct *t);
  65  extern void __tasklet_hi_schedule(struct tasklet_struct *t);
  66  
  67  static inline void tasklet_hi_schedule(struct tasklet_struct *t)
  68 @@ -512,6 +518,7 @@ static inline void tasklet_hi_schedule(struct tasklet_struct *t)
  69  		__tasklet_hi_schedule(t);
  70  }
  71  
  72 +extern void ___tasklet_hi_schedule_first(struct tasklet_struct *t);
  73  extern void __tasklet_hi_schedule_first(struct tasklet_struct *t);
  74  
  75  /*
  76 @@ -541,7 +548,7 @@ static inline void tasklet_disable(struct tasklet_struct *t)
  77  }
  78  
  79  static inline void tasklet_enable(struct tasklet_struct *t)
  80 -{
  81 +{	
  82  	smp_mb__before_atomic_dec();
  83  	atomic_dec(&t->count);
  84  }
  85 diff --git a/include/linux/mutex.h b/include/linux/mutex.h
  86 index f363bc8..9f31995 100644
  87 --- a/include/linux/mutex.h
  88 +++ b/include/linux/mutex.h
  89 @@ -126,6 +126,15 @@ static inline int mutex_is_locked(struct mutex *lock)
  90  	return atomic_read(&lock->count) != 1;
  91  }
  92  
  93 +/* return non-zero to abort.  only pre-side-effects may abort */
  94 +typedef int (*side_effect_t)(unsigned long);
  95 +extern void mutex_lock_sfx(struct mutex *lock,
  96 +						   side_effect_t pre, unsigned long pre_arg,
  97 +						   side_effect_t post, unsigned long post_arg);
  98 +extern void mutex_unlock_sfx(struct mutex *lock,
  99 +							 side_effect_t pre, unsigned long pre_arg,
 100 +							 side_effect_t post, unsigned long post_arg);
 101 +
 102  /*
 103   * See kernel/mutex.c for detailed documentation of these APIs.
 104   * Also see Documentation/mutex-design.txt.
 105 @@ -145,6 +154,7 @@ extern void mutex_lock(struct mutex *lock);
 106  extern int __must_check mutex_lock_interruptible(struct mutex *lock);
 107  extern int __must_check mutex_lock_killable(struct mutex *lock);
 108  
 109 +
 110  # define mutex_lock_nested(lock, subclass) mutex_lock(lock)
 111  # define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock)
 112  # define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock)
 113 diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h
 114 index 5310d27..69e3f57 100644
 115 --- a/include/linux/semaphore.h
 116 +++ b/include/linux/semaphore.h
 117 @@ -49,4 +49,13 @@ extern int __must_check down_trylock(struct semaphore *sem);
 118  extern int __must_check down_timeout(struct semaphore *sem, long jiffies);
 119  extern void up(struct semaphore *sem);
 120  
 121 +extern void __down(struct semaphore *sem);
 122 +extern void __up(struct semaphore *sem);
 123 +
 124 +struct semaphore_waiter {
 125 +	struct list_head list;
 126 +	struct task_struct *task;
 127 +	int up;
 128 +};
 129 +
 130  #endif /* __LINUX_SEMAPHORE_H */
 131 diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
 132 index 25e02c9..5fecfb3 100644
 133 --- a/include/linux/workqueue.h
 134 +++ b/include/linux/workqueue.h
 135 @@ -83,6 +83,9 @@ struct work_struct {
 136  #ifdef CONFIG_LOCKDEP
 137  	struct lockdep_map lockdep_map;
 138  #endif
 139 +#ifdef CONFIG_LITMUS_SOFTIRQD
 140 +	struct task_struct *owner;
 141 +#endif
 142  };
 143  
 144  #define WORK_DATA_INIT()	ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU)
 145 @@ -115,11 +118,25 @@ struct execute_work {
 146  #define __WORK_INIT_LOCKDEP_MAP(n, k)
 147  #endif
 148  
 149 +#ifdef CONFIG_LITMUS_SOFTIRQD
 150 +#define __WORK_INIT_OWNER() \
 151 +	.owner = NULL,
 152 +
 153 +#define PREPARE_OWNER(_work, _owner) \
 154 +	do { \
 155 +		(_work)->owner = (_owner); \
 156 +	} while(0)
 157 +#else
 158 +#define __WORK_INIT_OWNER()
 159 +#define PREPARE_OWNER(_work, _owner)
 160 +#endif
 161 +
 162  #define __WORK_INITIALIZER(n, f) {				\
 163  	.data = WORK_DATA_STATIC_INIT(),			\
 164  	.entry	= { &(n).entry, &(n).entry },			\
 165  	.func = (f),						\
 166  	__WORK_INIT_LOCKDEP_MAP(#n, &(n))			\
 167 +	__WORK_INIT_OWNER() \
 168  	}
 169  
 170  #define __DELAYED_WORK_INITIALIZER(n, f) {			\
 171 @@ -327,6 +344,7 @@ extern void flush_workqueue(struct workqueue_struct *wq);
 172  extern void flush_scheduled_work(void);
 173  extern void flush_delayed_work(struct delayed_work *work);
 174  
 175 +extern int __schedule_work(struct work_struct *work);
 176  extern int schedule_work(struct work_struct *work);
 177  extern int schedule_work_on(int cpu, struct work_struct *work);
 178  extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay);
 179 diff --git a/include/litmus/affinity.h b/include/litmus/affinity.h
 180 new file mode 100644
 181 index 0000000..877b409
 182 --- /dev/null
 183 +++ b/include/litmus/affinity.h
 184 @@ -0,0 +1,78 @@
 185 +#ifndef __LITMUS_AFFINITY_H
 186 +#define __LITMUS_AFFINITY_H
 187 +
 188 +#include <linux/cpumask.h>
 189 +
 190 +/*
 191 +  L1 (instr) = depth 0
 192 +  L1 (data)  = depth 1
 193 +  L2 = depth 2
 194 +  L3 = depth 3
 195 + */
 196 +#define NUM_CACHE_LEVELS 4
 197 +
 198 +struct neighborhood
 199 +{
 200 +	unsigned int size[NUM_CACHE_LEVELS];
 201 +	cpumask_var_t neighbors[NUM_CACHE_LEVELS];
 202 +};
 203 +
 204 +/* topology info is stored redundently in a big array for fast lookups */
 205 +extern struct neighborhood neigh_info[NR_CPUS];
 206 +
 207 +void init_topology(void); /* called by Litmus module's _init_litmus() */
 208 +
 209 +/* Works like:
 210 +void get_nearest_available_cpu(cpu_entry_t* nearest, cpu_entry_t* start, cpu_entry_t* entries, int release_master)
 211 +
 212 +Set release_master = -1 for no RM.
 213 + */
 214 +#define get_nearest_available_cpu(nearest, start, entries, release_master) \
 215 +{ \
 216 +	(nearest) = NULL; \
 217 +	if(!(start)->linked) \
 218 +	{ \
 219 +		(nearest) = (start); \
 220 +	} \
 221 +	else \
 222 +	{ \
 223 +		int __level; \
 224 +		int __cpu; \
 225 +		struct neighborhood* __neighbors = &neigh_info[(start)->cpu]; \
 226 +		\
 227 +		for(__level = 0; (__level < NUM_CACHE_LEVELS) && !(nearest); ++__level) \
 228 +		{ \
 229 +			if(__neighbors->size[__level] > 1) \
 230 +			{ \
 231 +				for_each_cpu(__cpu, __neighbors->neighbors[__level]) \
 232 +				{ \
 233 +					if(__cpu != (release_master)) \
 234 +					{ \
 235 +						cpu_entry_t* __entry = &per_cpu((entries), __cpu); \
 236 +						if(!__entry->linked) \
 237 +						{ \
 238 +							(nearest) = __entry; \
 239 +							break; \
 240 +						} \
 241 +					} \
 242 +				} \
 243 +			} \
 244 +			else if(__neighbors->size[__level] == 0) \
 245 +			{ \
 246 +				break; \
 247 +			} \
 248 +		} \
 249 +	} \
 250 +	\
 251 +	if((nearest)) \
 252 +	{ \
 253 +		TRACE("P%d is closest available CPU to P%d\n", (nearest)->cpu, (start)->cpu); \
 254 +	} \
 255 +	else \
 256 +	{ \
 257 +		TRACE("Could not find an available CPU close to P%d\n", \
 258 +						(start)->cpu); \
 259 +	} \
 260 +}
 261 +
 262 +#endif
 263 diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h
 264 index caf2a1e..c740e8f 100644
 265 --- a/include/litmus/fdso.h
 266 +++ b/include/litmus/fdso.h
 267 @@ -18,9 +18,10 @@ typedef enum  {
 268  	MIN_OBJ_TYPE 	= 0,
 269  
 270  	FMLP_SEM	= 0,
 271 -	SRP_SEM		= 1,
 272 +	KFMLP_SEM	= 1,
 273 +	SRP_SEM		= 2,
 274  
 275 -	MAX_OBJ_TYPE	= 1
 276 +	MAX_OBJ_TYPE	= SRP_SEM
 277  } obj_type_t;
 278  
 279  struct inode_obj_id {
 280 @@ -64,6 +65,7 @@ static inline void* od_lookup(int od, obj_type_t type)
 281  }
 282  
 283  #define lookup_fmlp_sem(od)((struct pi_semaphore*)  od_lookup(od, FMLP_SEM))
 284 +#define lookup_kfmlp_sem(od)((struct pi_semaphore*)  od_lookup(od, KFMLP_SEM))
 285  #define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM))
 286  #define lookup_ics(od)     ((struct ics*)           od_lookup(od, ICS_ID))
 287  
 288 diff --git a/include/litmus/fifo_common.h b/include/litmus/fifo_common.h
 289 new file mode 100644
 290 index 0000000..12cfbfe
 291 --- /dev/null
 292 +++ b/include/litmus/fifo_common.h
 293 @@ -0,0 +1,25 @@
 294 +/*
 295 + * EDF common data structures and utility functions shared by all EDF
 296 + * based scheduler plugins
 297 + */
 298 +
 299 +/* CLEANUP: Add comments and make it less messy.
 300 + *
 301 + */
 302 +
 303 +#ifndef __UNC_FIFO_COMMON_H__
 304 +#define __UNC_FIFO_COMMON_H__
 305 +
 306 +#include <litmus/rt_domain.h>
 307 +
 308 +void fifo_domain_init(rt_domain_t* rt, check_resched_needed_t resched,
 309 +		     release_jobs_t release);
 310 +
 311 +int fifo_higher_prio(struct task_struct* first,
 312 +		    struct task_struct* second);
 313 +
 314 +int fifo_ready_order(struct bheap_node* a, struct bheap_node* b);
 315 +
 316 +int fifo_preemption_needed(rt_domain_t* rt, struct task_struct *t);
 317 +
 318 +#endif
 319 diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
 320 index e7769ca..829c1c5 100644
 321 --- a/include/litmus/litmus.h
 322 +++ b/include/litmus/litmus.h
 323 @@ -26,6 +26,7 @@ static inline int in_list(struct list_head* list)
 324  		);
 325  }
 326  
 327 +
 328  struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq);
 329  
 330  #define NO_CPU			0xffffffff
 331 @@ -117,7 +118,9 @@ static inline lt_t litmus_clock(void)
 332  #define earlier_release(a, b)  (lt_before(\
 333  	(a)->rt_param.job_params.release,\
 334  	(b)->rt_param.job_params.release))
 335 -
 336 +#define shorter_period(a, b)   (lt_before(\
 337 +    (a)->rt_param.task_params.period,\
 338 +    (b)->rt_param.task_params.period))
 339  void preempt_if_preemptable(struct task_struct* t, int on_cpu);
 340  
 341  #ifdef CONFIG_LITMUS_LOCKING
 342 diff --git a/include/litmus/litmus_softirq.h b/include/litmus/litmus_softirq.h
 343 new file mode 100644
 344 index 0000000..34287f3
 345 --- /dev/null
 346 +++ b/include/litmus/litmus_softirq.h
 347 @@ -0,0 +1,199 @@
 348 +#ifndef __LITMUS_SOFTIRQ_H
 349 +#define __LITMUS_SOFTIRQ_H
 350 +
 351 +#include <linux/interrupt.h>
 352 +#include <linux/workqueue.h>
 353 +
 354 +/*
 355 +   Threaded tasklet handling for Litmus.  Tasklets
 356 +   are scheduled with the priority of the tasklet's
 357 +   owner---that is, the RT task on behalf the tasklet
 358 +   runs.
 359 + 
 360 +   Tasklets are current scheduled in FIFO order with
 361 +   NO priority inheritance for "blocked" tasklets.
 362 + 
 363 +   klitirqd assumes the priority of the owner of the
 364 +   tasklet when the tasklet is next to execute.
 365 + 
 366 +   Currently, hi-tasklets are scheduled before
 367 +   low-tasklets, regardless of priority of low-tasklets.
 368 +   And likewise, low-tasklets are scheduled before work
 369 +   queue objects.  This priority inversion probably needs
 370 +   to be fixed, though it is not an issue if our work with
 371 +   GPUs as GPUs are owned (and associated klitirqds) for
 372 +   exclusive time periods, thus no inversions can
 373 +   occur.
 374 + */
 375 +
 376 +
 377 +
 378 +#define NR_LITMUS_SOFTIRQD CONFIG_NR_LITMUS_SOFTIRQD
 379 +
 380 +/* Spawns NR_LITMUS_SOFTIRQD klitirqd daemons.
 381 +   Actual launch of threads is deffered to kworker's
 382 +   workqueue, so daemons will likely not be immediately
 383 +   running when this function returns, though the required
 384 +   data will be initialized.
 385 + 
 386 +   @affinity_set: an array expressing the processor affinity
 387 +    for each of the NR_LITMUS_SOFTIRQD daemons.  May be set
 388 +    to NULL for global scheduling.
 389 + 
 390 +	- Examples -
 391 +	8-CPU system with two CPU clusters:
 392 +		affinity[] = {0, 0, 0, 0, 3, 3, 3, 3}
 393 +		NOTE: Daemons not actually bound to specified CPU, but rather
 394 +		cluster in which the CPU resides.
 395 + 
 396 +	8-CPU system, partitioned:
 397 +		affinity[] = {0, 1, 2, 3, 4, 5, 6, 7}
 398 + 
 399 +	FIXME: change array to a CPU topology or array of cpumasks
 400 + 
 401 + */
 402 +void spawn_klitirqd(int* affinity);
 403 +
 404 +
 405 +/* Raises a flag to tell klitirqds to terminate.
 406 +   Termination is async, so some threads may be running
 407 +   after function return. */
 408 +void kill_klitirqd(void);
 409 +
 410 +
 411 +/* Returns 1 if all NR_LITMUS_SOFTIRQD klitirqs are ready
 412 +   to handle tasklets. 0, otherwise.*/
 413 +int klitirqd_is_ready(void);
 414 +
 415 +/* Returns 1 if no NR_LITMUS_SOFTIRQD klitirqs are ready
 416 +   to handle tasklets. 0, otherwise.*/
 417 +int klitirqd_is_dead(void);
 418 +
 419 +/* Flushes all pending work out to the OS for regular
 420 + * tasklet/work processing of the specified 'owner'
 421 + *
 422 + * PRECOND: klitirqd_thread must have a clear entry
 423 + * in the GPU registry, otherwise this call will become
 424 + * a no-op as work will loop back to the klitirqd_thread.
 425 + *
 426 + * Pass NULL for owner to flush ALL pending items.
 427 + */
 428 +void flush_pending(struct task_struct* klitirqd_thread,
 429 +				   struct task_struct* owner);
 430 +
 431 +struct task_struct* get_klitirqd(unsigned int k_id);
 432 +
 433 +
 434 +extern int __litmus_tasklet_schedule(
 435 +        struct tasklet_struct *t,
 436 +        unsigned int k_id);
 437 +
 438 +/* schedule a tasklet on klitirqd #k_id */
 439 +static inline int litmus_tasklet_schedule(
 440 +    struct tasklet_struct *t,
 441 +    unsigned int k_id)
 442 +{
 443 +	int ret = 0;
 444 +	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
 445 +		ret = __litmus_tasklet_schedule(t, k_id);
 446 +	return(ret);
 447 +}
 448 +
 449 +/* for use by __tasklet_schedule() */
 450 +static inline int _litmus_tasklet_schedule(
 451 +    struct tasklet_struct *t,
 452 +    unsigned int k_id)
 453 +{
 454 +    return(__litmus_tasklet_schedule(t, k_id));
 455 +}
 456 +
 457 +
 458 +
 459 +
 460 +extern int __litmus_tasklet_hi_schedule(struct tasklet_struct *t,
 461 +                                         unsigned int k_id);
 462 +
 463 +/* schedule a hi tasklet on klitirqd #k_id */
 464 +static inline int litmus_tasklet_hi_schedule(struct tasklet_struct *t,
 465 +                                              unsigned int k_id)
 466 +{
 467 +	int ret = 0;
 468 +	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
 469 +		ret = __litmus_tasklet_hi_schedule(t, k_id);
 470 +	return(ret);
 471 +}
 472 +
 473 +/* for use by __tasklet_hi_schedule() */
 474 +static inline int _litmus_tasklet_hi_schedule(struct tasklet_struct *t,
 475 +                                               unsigned int k_id)
 476 +{
 477 +    return(__litmus_tasklet_hi_schedule(t, k_id));
 478 +}
 479 +
 480 +
 481 +
 482 +
 483 +
 484 +extern int __litmus_tasklet_hi_schedule_first(
 485 +    struct tasklet_struct *t,
 486 +    unsigned int k_id);
 487 +
 488 +/* schedule a hi tasklet on klitirqd #k_id on next go-around */
 489 +/* PRECONDITION: Interrupts must be disabled. */
 490 +static inline int litmus_tasklet_hi_schedule_first(
 491 +    struct tasklet_struct *t,
 492 +    unsigned int k_id)
 493 +{
 494 +	int ret = 0;
 495 +	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
 496 +		ret = __litmus_tasklet_hi_schedule_first(t, k_id);
 497 +	return(ret);
 498 +}
 499 +
 500 +/* for use by __tasklet_hi_schedule_first() */
 501 +static inline int _litmus_tasklet_hi_schedule_first(
 502 +    struct tasklet_struct *t,
 503 +    unsigned int k_id)
 504 +{
 505 +    return(__litmus_tasklet_hi_schedule_first(t, k_id));
 506 +}
 507 +
 508 +
 509 +
 510 +//////////////
 511 +
 512 +extern int __litmus_schedule_work(
 513 +	struct work_struct* w,
 514 +	unsigned int k_id);
 515 +
 516 +static inline int litmus_schedule_work(
 517 +	struct work_struct* w,
 518 +	unsigned int k_id)
 519 +{
 520 +	return(__litmus_schedule_work(w, k_id));
 521 +}
 522 +
 523 +
 524 +
 525 +///////////// mutex operations for client threads.
 526 + 
 527 +void down_and_set_stat(struct task_struct* t,
 528 +					 enum klitirqd_sem_status to_set,
 529 +					 struct mutex* sem);
 530 +
 531 +void __down_and_reset_and_set_stat(struct task_struct* t,
 532 +				enum klitirqd_sem_status to_reset,
 533 +				enum klitirqd_sem_status to_set,
 534 +				struct mutex* sem);
 535 +
 536 +void up_and_set_stat(struct task_struct* t,
 537 +					enum klitirqd_sem_status to_set,
 538 +					struct mutex* sem);
 539 +
 540 +
 541 +
 542 +void release_klitirqd_lock(struct task_struct* t);
 543 +
 544 +int reacquire_klitirqd_lock(struct task_struct* t);
 545 +
 546 +#endif
 547 diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h
 548 new file mode 100644
 549 index 0000000..9e07a27
 550 --- /dev/null
 551 +++ b/include/litmus/nvidia_info.h
 552 @@ -0,0 +1,38 @@
 553 +#ifndef __LITMUS_NVIDIA_H
 554 +#define __LITMUS_NVIDIA_H
 555 +
 556 +#include <linux/interrupt.h>
 557 +
 558 +
 559 +#include <litmus/litmus_softirq.h>
 560 +
 561 +
 562 +//#define NV_DEVICE_NUM NR_LITMUS_SOFTIRQD
 563 +#define NV_DEVICE_NUM CONFIG_NV_DEVICE_NUM
 564 +
 565 +int init_nvidia_info(void);
 566 +
 567 +int is_nvidia_func(void* func_addr);
 568 +
 569 +void dump_nvidia_info(const struct tasklet_struct *t);
 570 +
 571 +
 572 +// Returns the Nvidia device # associated with provided tasklet and work_struct.
 573 +u32 get_tasklet_nv_device_num(const struct tasklet_struct *t);
 574 +u32 get_work_nv_device_num(const struct work_struct *t);
 575 +
 576 +
 577 +int init_nv_device_reg(void);
 578 +//int get_nv_device_id(struct task_struct* owner);
 579 +
 580 +
 581 +int reg_nv_device(int reg_device_id, int register_device);
 582 +
 583 +struct task_struct* get_nv_device_owner(u32 target_device_id);
 584 +
 585 +void lock_nv_registry(u32 reg_device_id, unsigned long* flags);
 586 +void unlock_nv_registry(u32 reg_device_id, unsigned long* flags);
 587 +
 588 +void increment_nv_int_count(u32 device);
 589 +
 590 +#endif
 591 diff --git a/include/litmus/preempt.h b/include/litmus/preempt.h
 592 index 260c6fe..244924f 100644
 593 --- a/include/litmus/preempt.h
 594 +++ b/include/litmus/preempt.h
 595 @@ -26,6 +26,7 @@ const char* sched_state_name(int s);
 596  				    (x), #x, __FUNCTION__);		\
 597  	} while (0);
 598  
 599 +//#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) /* ignore */
 600  #define TRACE_SCHED_STATE_CHANGE(x, y, cpu)				\
 601  	TRACE_STATE("[P%d] 0x%x (%s) -> 0x%x (%s)\n",			\
 602  		    cpu,  (x), sched_state_name(x),			\
 603 diff --git a/include/litmus/rm_common.h b/include/litmus/rm_common.h
 604 new file mode 100644
 605 index 0000000..5991b0b
 606 --- /dev/null
 607 +++ b/include/litmus/rm_common.h
 608 @@ -0,0 +1,25 @@
 609 +/*
 610 + * EDF common data structures and utility functions shared by all EDF
 611 + * based scheduler plugins
 612 + */
 613 +
 614 +/* CLEANUP: Add comments and make it less messy.
 615 + *
 616 + */
 617 +
 618 +#ifndef __UNC_RM_COMMON_H__
 619 +#define __UNC_RM_COMMON_H__
 620 +
 621 +#include <litmus/rt_domain.h>
 622 +
 623 +void rm_domain_init(rt_domain_t* rt, check_resched_needed_t resched,
 624 +		     release_jobs_t release);
 625 +
 626 +int rm_higher_prio(struct task_struct* first,
 627 +		    struct task_struct* second);
 628 +
 629 +int rm_ready_order(struct bheap_node* a, struct bheap_node* b);
 630 +
 631 +int rm_preemption_needed(rt_domain_t* rt, struct task_struct *t);
 632 +
 633 +#endif
 634 diff --git a/include/litmus/rm_srt_common.h b/include/litmus/rm_srt_common.h
 635 new file mode 100644
 636 index 0000000..78aa287
 637 --- /dev/null
 638 +++ b/include/litmus/rm_srt_common.h
 639 @@ -0,0 +1,25 @@
 640 +/*
 641 + * EDF common data structures and utility functions shared by all EDF
 642 + * based scheduler plugins
 643 + */
 644 +
 645 +/* CLEANUP: Add comments and make it less messy.
 646 + *
 647 + */
 648 +
 649 +#ifndef __UNC_RM_SRT_COMMON_H__
 650 +#define __UNC_RM_SRT_COMMON_H__
 651 +
 652 +#include <litmus/rt_domain.h>
 653 +
 654 +void rm_srt_domain_init(rt_domain_t* rt, check_resched_needed_t resched,
 655 +		     release_jobs_t release);
 656 +
 657 +int rm_srt_higher_prio(struct task_struct* first,
 658 +		    struct task_struct* second);
 659 +
 660 +int rm_srt_ready_order(struct bheap_node* a, struct bheap_node* b);
 661 +
 662 +int rm_srt_preemption_needed(rt_domain_t* rt, struct task_struct *t);
 663 +
 664 +#endif
 665 diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
 666 index 5de422c..53af3ce 100644
 667 --- a/include/litmus/rt_param.h
 668 +++ b/include/litmus/rt_param.h
 669 @@ -69,6 +69,8 @@ struct control_page {
 670  /* don't export internal data structures to user space (liblitmus) */
 671  #ifdef __KERNEL__
 672  
 673 +#include <linux/semaphore.h>
 674 +
 675  struct _rt_domain;
 676  struct bheap_node;
 677  struct release_heap;
 678 @@ -94,6 +96,14 @@ struct rt_job {
 679  
 680  struct pfair_param;
 681  
 682 +enum klitirqd_sem_status
 683 +{
 684 +	NEED_TO_REACQUIRE,
 685 +	REACQUIRING,
 686 +	NOT_HELD,
 687 +	HELD
 688 +};
 689 +
 690  /*	RT task parameters for scheduling extensions
 691   *	These parameters are inherited during clone and therefore must
 692   *	be explicitly set up before the task set is launched.
 693 @@ -108,6 +118,38 @@ struct rt_param {
 694  	/* is the task present? (true if it can be scheduled) */
 695  	unsigned int		present:1;
 696  
 697 +#ifdef CONFIG_LITMUS_SOFTIRQD
 698 +    /* proxy threads have minimum priority by default */
 699 +    unsigned int        is_proxy_thread:1;
 700 +    
 701 +	/* pointer to klitirqd currently working on this
 702 +	   task_struct's behalf.  only set by the task pointed
 703 +	   to by klitirqd.
 704 +	 
 705 +	   ptr only valid if is_proxy_thread == 0
 706 +	 */
 707 +	struct task_struct* cur_klitirqd;
 708 +
 709 +	/* Used to implement mutual execution exclusion between
 710 +	 * job and klitirqd execution.  Job must always hold
 711 +	 * it's klitirqd_sem to execute.  klitirqd instance
 712 +	 * must hold the semaphore before executing on behalf
 713 +	 * of a job.
 714 +	 */
 715 +	//struct semaphore			klitirqd_sem;
 716 +	struct mutex				klitirqd_sem;
 717 +
 718 +	/* status of held klitirqd_sem, even if the held klitirqd_sem is from
 719 +	   another task (only proxy threads do this though).
 720 +	 */
 721 +	atomic_t					klitirqd_sem_stat;
 722 +#endif
 723 +
 724 +#ifdef CONFIG_LITMUS_NVIDIA
 725 +	/* number of top-half interrupts handled on behalf of current job */
 726 +	atomic_t					nv_int_count;
 727 +#endif
 728 +
 729  #ifdef CONFIG_LITMUS_LOCKING
 730  	/* Is the task being priority-boosted by a locking protocol? */
 731  	unsigned int		priority_boosted:1;
 732 @@ -128,7 +170,7 @@ struct rt_param {
 733  	 * an increased task priority.
 734  	 */
 735  	 struct task_struct*	inh_task;
 736 -
 737 +    
 738  #ifdef CONFIG_NP_SECTION
 739  	/* For the FMLP under PSN-EDF, it is required to make the task
 740  	 * non-preemptive from kernel space. In order not to interfere with
 741 diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h
 742 index 6e7cabd..8fdf05d 100644
 743 --- a/include/litmus/sched_plugin.h
 744 +++ b/include/litmus/sched_plugin.h
 745 @@ -11,6 +11,8 @@
 746  #include <litmus/locking.h>
 747  #endif
 748  
 749 +#include <linux/interrupt.h>
 750 +
 751  /************************ setup/tear down ********************/
 752  
 753  typedef long (*activate_plugin_t) (void);
 754 @@ -29,7 +31,6 @@ typedef struct task_struct* (*schedule_t)(struct task_struct * prev);
 755   */
 756  typedef void (*finish_switch_t)(struct task_struct *prev);
 757  
 758 -
 759  /********************* task state changes ********************/
 760  
 761  /* Called to setup a new real-time task.
 762 @@ -58,6 +59,21 @@ typedef void (*task_exit_t)    (struct task_struct *);
 763  typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type,
 764  				 void* __user config);
 765  
 766 +/* Called to change inheritance levels of given task */
 767 +typedef void (*set_prio_inh_t)(struct task_struct* t,
 768 +                               struct task_struct* prio_inh);
 769 +typedef void (*clear_prio_inh_t)(struct task_struct* t);
 770 +
 771 +
 772 +typedef void (*set_prio_inh_klitirq_t)(struct task_struct* klitirqd,
 773 +                                       struct task_struct* old_owner,
 774 +                                       struct task_struct* new_owner);
 775 +typedef void (*clear_prio_inh_klitirqd_t)(struct task_struct* klitirqd,
 776 +                                          struct task_struct* old_owner);
 777 +
 778 +
 779 +typedef int (*enqueue_pai_tasklet_t)(struct tasklet_struct* tasklet);
 780 +typedef void (*run_tasklets_t)(struct task_struct* next);
 781  
 782  /********************* sys call backends  ********************/
 783  /* This function causes the caller to sleep until the next release */
 784 @@ -88,7 +104,7 @@ struct sched_plugin {
 785  	/*	task state changes 	*/
 786  	admit_task_t		admit_task;
 787  
 788 -        task_new_t 		task_new;
 789 +    task_new_t			task_new;
 790  	task_wake_up_t		task_wake_up;
 791  	task_block_t		task_block;
 792  	task_exit_t 		task_exit;
 793 @@ -96,6 +112,19 @@ struct sched_plugin {
 794  #ifdef CONFIG_LITMUS_LOCKING
 795  	/*	locking protocols	*/
 796  	allocate_lock_t		allocate_lock;
 797 +    
 798 +    set_prio_inh_t      set_prio_inh;
 799 +    clear_prio_inh_t    clear_prio_inh;
 800 +#endif
 801 +    
 802 +#ifdef CONFIG_LITMUS_SOFTIRQD
 803 +    set_prio_inh_klitirq_t		set_prio_inh_klitirqd;
 804 +    clear_prio_inh_klitirqd_t	clear_prio_inh_klitirqd;
 805 +#endif
 806 +	
 807 +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
 808 +	enqueue_pai_tasklet_t		enqueue_pai_tasklet;
 809 +	run_tasklets_t				run_tasklets;
 810  #endif
 811  } __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
 812  
 813 diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h
 814 index 7ca34cb..232c758 100644
 815 --- a/include/litmus/sched_trace.h
 816 +++ b/include/litmus/sched_trace.h
 817 @@ -11,12 +11,12 @@ struct st_trace_header {
 818  	u8	cpu;		/* On which CPU was it recorded? */
 819  	u16	pid;		/* PID of the task.              */
 820  	u32	job;		/* The job sequence number.      */
 821 -};
 822 +} __attribute__((packed));
 823  
 824  #define ST_NAME_LEN 16
 825  struct st_name_data {
 826  	char	cmd[ST_NAME_LEN];/* The name of the executable of this process. */
 827 -};
 828 +} __attribute__((packed));
 829  
 830  struct st_param_data {		/* regular params */
 831  	u32	wcet;
 832 @@ -25,30 +25,29 @@ struct st_param_data {		/* regular params */
 833  	u8	partition;
 834  	u8	class;
 835  	u8	__unused[2];
 836 -};
 837 +} __attribute__((packed));
 838  
 839  struct st_release_data {	/* A job is was/is going to be released. */
 840  	u64	release;	/* What's the release time?              */
 841  	u64	deadline;	/* By when must it finish?		 */
 842 -};
 843 +} __attribute__((packed));
 844  
 845  struct st_assigned_data {	/* A job was asigned to a CPU. 		 */
 846  	u64	when;
 847  	u8	target;		/* Where should it execute?	         */
 848  	u8	__unused[7];
 849 -};
 850 +} __attribute__((packed));
 851  
 852  struct st_switch_to_data {	/* A process was switched to on a given CPU.   */
 853  	u64	when;		/* When did this occur?                        */
 854  	u32	exec_time;	/* Time the current job has executed.          */
 855  	u8	__unused[4];
 856 -
 857 -};
 858 +} __attribute__((packed));
 859  
 860  struct st_switch_away_data {	/* A process was switched away from on a given CPU. */
 861  	u64	when;
 862  	u64	exec_time;
 863 -};
 864 +} __attribute__((packed));
 865  
 866  struct st_completion_data {	/* A job completed. */
 867  	u64	when;
 868 @@ -56,35 +55,92 @@ struct st_completion_data {	/* A job completed. */
 869  				 * next task automatically; set to 0 otherwise.
 870  				 */
 871  	u8	__uflags:7;
 872 -	u8	__unused[7];
 873 -};
 874 +	u16 nv_int_count;
 875 +	u8	__unused[5];
 876 +} __attribute__((packed));
 877  
 878  struct st_block_data {		/* A task blocks. */
 879  	u64	when;
 880  	u64	__unused;
 881 -};
 882 +} __attribute__((packed));
 883  
 884  struct st_resume_data {		/* A task resumes. */
 885  	u64	when;
 886  	u64	__unused;
 887 -};
 888 +} __attribute__((packed));
 889  
 890  struct st_action_data {
 891  	u64	when;
 892  	u8	action;
 893  	u8	__unused[7];
 894 -};
 895 +} __attribute__((packed));
 896  
 897  struct st_sys_release_data {
 898  	u64	when;
 899  	u64	release;
 900 -};
 901 +} __attribute__((packed));
 902 +
 903 +
 904 +struct st_tasklet_release_data {
 905 +	u64 when;
 906 +	u64 __unused;
 907 +} __attribute__((packed));
 908 +
 909 +struct st_tasklet_begin_data {
 910 +	u64 when;
 911 +	u16 exe_pid;
 912 +	u8  __unused[6];
 913 +} __attribute__((packed));
 914 +
 915 +struct st_tasklet_end_data {
 916 +	u64 when;
 917 +	u16 exe_pid;
 918 +	u8	flushed;
 919 +	u8	__unused[5];
 920 +} __attribute__((packed));
 921 +
 922 +
 923 +struct st_work_release_data {
 924 +	u64 when;
 925 +	u64 __unused;
 926 +} __attribute__((packed));
 927 +
 928 +struct st_work_begin_data {
 929 +	u64 when;
 930 +	u16 exe_pid;
 931 +	u8	__unused[6];
 932 +} __attribute__((packed));
 933 +
 934 +struct st_work_end_data {
 935 +	u64 when;
 936 +	u16 exe_pid;
 937 +	u8	flushed;
 938 +	u8	__unused[5];
 939 +} __attribute__((packed));
 940 +
 941 +struct st_effective_priority_change_data {
 942 +	u64 when;
 943 +	u16 inh_pid;
 944 +	u8	__unused[6];
 945 +} __attribute__((packed));
 946 +
 947 +struct st_nv_interrupt_begin_data {
 948 +	u64 when;
 949 +	u32 device;
 950 +	u32 serialNumber;
 951 +} __attribute__((packed));
 952 +
 953 +struct st_nv_interrupt_end_data {
 954 +	u64 when;
 955 +	u32 device;
 956 +	u32 serialNumber;
 957 +} __attribute__((packed));
 958  
 959  #define DATA(x) struct st_ ## x ## _data x;
 960  
 961  typedef enum {
 962 -        ST_NAME = 1,		/* Start at one, so that we can spot
 963 -				 * uninitialized records. */
 964 +    ST_NAME = 1, /* Start at one, so that we can spot
 965 +				  * uninitialized records. */
 966  	ST_PARAM,
 967  	ST_RELEASE,
 968  	ST_ASSIGNED,
 969 @@ -94,7 +150,16 @@ typedef enum {
 970  	ST_BLOCK,
 971  	ST_RESUME,
 972  	ST_ACTION,
 973 -	ST_SYS_RELEASE
 974 +	ST_SYS_RELEASE,
 975 +	ST_TASKLET_RELEASE,
 976 +	ST_TASKLET_BEGIN,
 977 +	ST_TASKLET_END,
 978 +	ST_WORK_RELEASE,
 979 +	ST_WORK_BEGIN,
 980 +	ST_WORK_END,
 981 +	ST_EFF_PRIO_CHANGE,
 982 +	ST_NV_INTERRUPT_BEGIN,
 983 +	ST_NV_INTERRUPT_END,
 984  } st_event_record_type_t;
 985  
 986  struct st_event_record {
 987 @@ -113,8 +178,17 @@ struct st_event_record {
 988  		DATA(resume);
 989  		DATA(action);
 990  		DATA(sys_release);
 991 +		DATA(tasklet_release);
 992 +		DATA(tasklet_begin);
 993 +		DATA(tasklet_end);
 994 +		DATA(work_release);
 995 +		DATA(work_begin);
 996 +		DATA(work_end);
 997 +		DATA(effective_priority_change);
 998 +		DATA(nv_interrupt_begin);
 999 +		DATA(nv_interrupt_end);
1000  	} data;
1001 -};
1002 +} __attribute__((packed));
1003  
1004  #undef DATA
1005  
1006 @@ -129,6 +203,8 @@ struct st_event_record {
1007  	ft_event1(id, callback, task)
1008  #define SCHED_TRACE2(id, callback, task, xtra) \
1009  	ft_event2(id, callback, task, xtra)
1010 +#define SCHED_TRACE3(id, callback, task, xtra1, xtra2) \
1011 +	ft_event3(id, callback, task, xtra1, xtra2)
1012  
1013  /* provide prototypes; needed on sparc64 */
1014  #ifndef NO_TASK_TRACE_DECLS
1015 @@ -155,12 +231,45 @@ feather_callback void do_sched_trace_action(unsigned long id,
1016  feather_callback void do_sched_trace_sys_release(unsigned long id,
1017  						 lt_t* start);
1018  
1019 +
1020 +feather_callback void do_sched_trace_tasklet_release(unsigned long id,
1021 +												   struct task_struct* owner);
1022 +feather_callback void do_sched_trace_tasklet_begin(unsigned long id,
1023 +												  struct task_struct* owner);
1024 +feather_callback void do_sched_trace_tasklet_end(unsigned long id,
1025 +												 struct task_struct* owner,
1026 +												 unsigned long flushed);
1027 +
1028 +feather_callback void do_sched_trace_work_release(unsigned long id,
1029 +													 struct task_struct* owner);
1030 +feather_callback void do_sched_trace_work_begin(unsigned long id,
1031 +												struct task_struct* owner,
1032 +												struct task_struct* exe);
1033 +feather_callback void do_sched_trace_work_end(unsigned long id,
1034 +											  struct task_struct* owner,
1035 +											  struct task_struct* exe,
1036 +											  unsigned long flushed);
1037 +
1038 +feather_callback void do_sched_trace_eff_prio_change(unsigned long id,
1039 +											  struct task_struct* task,
1040 +											  struct task_struct* inh);
1041 +
1042 +feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id,
1043 +												u32 device);
1044 +feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id,
1045 +												unsigned long unused);
1046 +
1047 +
1048 +/* returns true if we're tracing an interrupt on current CPU */
1049 +/* int is_interrupt_tracing_active(void); */
1050 +
1051  #endif
1052  
1053  #else
1054  
1055  #define SCHED_TRACE(id, callback, task)        /* no tracing */
1056  #define SCHED_TRACE2(id, callback, task, xtra) /* no tracing */
1057 +#define SCHED_TRACE3(id, callback, task, xtra1, xtra2)
1058  
1059  #endif
1060  
1061 @@ -193,6 +302,35 @@ feather_callback void do_sched_trace_sys_release(unsigned long id,
1062  	SCHED_TRACE(SCHED_TRACE_BASE_ID + 10, do_sched_trace_sys_release, when)
1063  
1064  
1065 +#define sched_trace_tasklet_release(t) \
1066 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 11, do_sched_trace_tasklet_release, t)
1067 +
1068 +#define sched_trace_tasklet_begin(t) \
1069 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, do_sched_trace_tasklet_begin, t)
1070 +
1071 +#define sched_trace_tasklet_end(t, flushed) \
1072 +	SCHED_TRACE2(SCHED_TRACE_BASE_ID + 13, do_sched_trace_tasklet_end, t, flushed)
1073 +
1074 +
1075 +#define sched_trace_work_release(t) \
1076 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 14, do_sched_trace_work_release, t)
1077 +
1078 +#define sched_trace_work_begin(t, e) \
1079 +	SCHED_TRACE2(SCHED_TRACE_BASE_ID + 15, do_sched_trace_work_begin, t, e)
1080 +
1081 +#define sched_trace_work_end(t, e, flushed) \
1082 +	SCHED_TRACE3(SCHED_TRACE_BASE_ID + 16, do_sched_trace_work_end, t, e, flushed)
1083 +
1084 +
1085 +#define sched_trace_eff_prio_change(t, inh) \
1086 +	SCHED_TRACE2(SCHED_TRACE_BASE_ID + 17, do_sched_trace_eff_prio_change, t, inh)
1087 +
1088 +
1089 +#define sched_trace_nv_interrupt_begin(d) \
1090 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 18, do_sched_trace_nv_interrupt_begin, d)
1091 +#define sched_trace_nv_interrupt_end(d) \
1092 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 19, do_sched_trace_nv_interrupt_end, d)
1093 +
1094  #define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */
1095  
1096  #endif /* __KERNEL__ */
1097 diff --git a/include/litmus/sched_trace_external.h b/include/litmus/sched_trace_external.h
1098 new file mode 100644
1099 index 0000000..e70e45e
1100 --- /dev/null
1101 +++ b/include/litmus/sched_trace_external.h
1102 @@ -0,0 +1,78 @@
1103 +/*
1104 + * sched_trace.h -- record scheduler events to a byte stream for offline analysis.
1105 + */
1106 +#ifndef _LINUX_SCHED_TRACE_EXTERNAL_H_
1107 +#define _LINUX_SCHED_TRACE_EXTERNAL_H_
1108 +
1109 +
1110 +#ifdef CONFIG_SCHED_TASK_TRACE
1111 +extern void __sched_trace_tasklet_begin_external(struct task_struct* t);
1112 +static inline void sched_trace_tasklet_begin_external(struct task_struct* t)
1113 +{
1114 +	__sched_trace_tasklet_begin_external(t);
1115 +}
1116 +
1117 +extern void __sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed);
1118 +static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed)
1119 +{
1120 +	__sched_trace_tasklet_end_external(t, flushed);
1121 +}
1122 +
1123 +extern void __sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e);
1124 +static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e)
1125 +{
1126 +	__sched_trace_work_begin_external(t, e);
1127 +}
1128 +
1129 +extern void __sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f);
1130 +static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f)
1131 +{
1132 +	__sched_trace_work_end_external(t, e, f);
1133 +}
1134 +
1135 +#ifdef CONFIG_LITMUS_NVIDIA
1136 +extern void __sched_trace_nv_interrupt_begin_external(u32 device);
1137 +static inline void sched_trace_nv_interrupt_begin_external(u32 device)
1138 +{
1139 +	__sched_trace_nv_interrupt_begin_external(device);
1140 +}
1141 +
1142 +extern void __sched_trace_nv_interrupt_end_external(u32 device);
1143 +static inline void sched_trace_nv_interrupt_end_external(u32 device)
1144 +{
1145 +	__sched_trace_nv_interrupt_end_external(device);
1146 +}
1147 +#endif
1148 +
1149 +#else
1150 +
1151 +// no tracing.
1152 +static inline void sched_trace_tasklet_begin_external(struct task_struct* t){}
1153 +static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed){}
1154 +static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e){}
1155 +static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f){}
1156 +
1157 +#ifdef CONFIG_LITMUS_NVIDIA
1158 +static inline void sched_trace_nv_interrupt_begin_external(u32 device){}
1159 +static inline void sched_trace_nv_interrupt_end_external(u32 device){}
1160 +#endif
1161 +
1162 +#endif
1163 +
1164 +
1165 +#ifdef CONFIG_LITMUS_NVIDIA
1166 +
1167 +#define EX_TS(evt) \
1168 +extern void __##evt(void); \
1169 +static inline void EX_##evt(void) { __##evt(); }
1170 +
1171 +EX_TS(TS_NV_TOPISR_START)
1172 +EX_TS(TS_NV_TOPISR_END)
1173 +EX_TS(TS_NV_BOTISR_START)
1174 +EX_TS(TS_NV_BOTISR_END)
1175 +EX_TS(TS_NV_RELEASE_BOTISR_START)
1176 +EX_TS(TS_NV_RELEASE_BOTISR_END)
1177 +
1178 +#endif
1179 +
1180 +#endif
1181 diff --git a/include/litmus/trace.h b/include/litmus/trace.h
1182 index 05f4872..09d409b 100644
1183 --- a/include/litmus/trace.h
1184 +++ b/include/litmus/trace.h
1185 @@ -100,4 +100,24 @@ feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu)
1186  #define TS_SEND_RESCHED_END		DTIMESTAMP(191, TSK_UNKNOWN)
1187  
1188  
1189 +
1190 +#ifdef CONFIG_LITMUS_NVIDIA
1191 +
1192 +#define TS_NV_TOPISR_START		TIMESTAMP(200)
1193 +#define TS_NV_TOPISR_END		TIMESTAMP(201)
1194 +
1195 +#define TS_NV_BOTISR_START		TIMESTAMP(202)
1196 +#define TS_NV_BOTISR_END		TIMESTAMP(203)
1197 +
1198 +#define TS_NV_RELEASE_BOTISR_START	TIMESTAMP(204)
1199 +#define TS_NV_RELEASE_BOTISR_END	TIMESTAMP(205)
1200 +
1201 +#endif
1202 +
1203 +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1204 +#define TS_NV_SCHED_BOTISR_START	TIMESTAMP(206)
1205 +#define TS_NV_SCHED_BOTISR_END		TIMESTAMP(207)
1206 +#endif
1207 +
1208 +
1209  #endif /* !_SYS_TRACE_H_ */
1210 diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h
1211 index 94264c2..c6efc4c 100644
1212 --- a/include/litmus/unistd_32.h
1213 +++ b/include/litmus/unistd_32.h
1214 @@ -17,5 +17,6 @@
1215  #define __NR_wait_for_ts_release __LSC(9)
1216  #define __NR_release_ts		__LSC(10)
1217  #define __NR_null_call		__LSC(11)
1218 +#define __NR_register_nv_device			__LSC(12)
1219  
1220 -#define NR_litmus_syscalls 12
1221 +#define NR_litmus_syscalls 13
1222 diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h
1223 index d5ced0d..b44a7c3 100644
1224 --- a/include/litmus/unistd_64.h
1225 +++ b/include/litmus/unistd_64.h
1226 @@ -29,5 +29,8 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release)
1227  __SYSCALL(__NR_release_ts, sys_release_ts)
1228  #define __NR_null_call				__LSC(11)
1229  __SYSCALL(__NR_null_call, sys_null_call)
1230 +#define __NR_register_nv_device			__LSC(12)
1231 +__SYSCALL(__NR_register_nv_device, sys_register_nv_device)
1232  
1233 -#define NR_litmus_syscalls 12
1234 +
1235 +#define NR_litmus_syscalls 13
1236 diff --git a/kernel/lockdep.c b/kernel/lockdep.c
1237 index f2852a5..ebff2cf 100644
1238 --- a/kernel/lockdep.c
1239 +++ b/kernel/lockdep.c
1240 @@ -530,7 +530,7 @@ static void print_lock(struct held_lock *hlock)
1241  	print_ip_sym(hlock->acquire_ip);
1242  }
1243  
1244 -static void lockdep_print_held_locks(struct task_struct *curr)
1245 +void lockdep_print_held_locks(struct task_struct *curr)
1246  {
1247  	int i, depth = curr->lockdep_depth;
1248  
1249 @@ -546,6 +546,7 @@ static void lockdep_print_held_locks(struct task_struct *curr)
1250  		print_lock(curr->held_locks + i);
1251  	}
1252  }
1253 +EXPORT_SYMBOL(lockdep_print_held_locks);
1254  
1255  static void print_kernel_version(void)
1256  {
1257 diff --git a/kernel/mutex.c b/kernel/mutex.c
1258 index 200407c..435685e 100644
1259 --- a/kernel/mutex.c
1260 +++ b/kernel/mutex.c
1261 @@ -496,3 +496,144 @@ int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
1262  	return 1;
1263  }
1264  EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
1265 +
1266 +
1267 +
1268 +
1269 +
1270 +
1271 +
1272 +
1273 +//__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, _RET_IP_);
1274 +
1275 +void mutex_lock_sfx(struct mutex *lock,
1276 +				   side_effect_t pre, unsigned long pre_arg,
1277 +				   side_effect_t post, unsigned long post_arg)
1278 +{
1279 +	long state = TASK_UNINTERRUPTIBLE;
1280 +	unsigned int subclass = 0;
1281 +	unsigned long ip = _RET_IP_;
1282 +	
1283 +
1284 +	struct task_struct *task = current;
1285 +	struct mutex_waiter waiter;
1286 +	unsigned long flags;
1287 +	
1288 +	preempt_disable();
1289 +	mutex_acquire(&lock->dep_map, subclass, 0, ip);
1290 +
1291 +	spin_lock_mutex(&lock->wait_lock, flags);
1292 +	
1293 +	if(pre)
1294 +	{
1295 +		if(unlikely(pre(pre_arg)))
1296 +		{
1297 +			// this will fuck with lockdep's CONFIG_PROVE_LOCKING...
1298 +			spin_unlock_mutex(&lock->wait_lock, flags);
1299 +			preempt_enable();
1300 +			return;
1301 +		}
1302 +	}
1303 +
1304 +	debug_mutex_lock_common(lock, &waiter);
1305 +	debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
1306 +	
1307 +	/* add waiting tasks to the end of the waitqueue (FIFO): */
1308 +	list_add_tail(&waiter.list, &lock->wait_list);
1309 +	waiter.task = task;
1310 +	
1311 +	if (atomic_xchg(&lock->count, -1) == 1)
1312 +		goto done;
1313 +	
1314 +	lock_contended(&lock->dep_map, ip);
1315 +	
1316 +	for (;;) {
1317 +		/*
1318 +		 * Lets try to take the lock again - this is needed even if
1319 +		 * we get here for the first time (shortly after failing to
1320 +		 * acquire the lock), to make sure that we get a wakeup once
1321 +		 * it's unlocked. Later on, if we sleep, this is the
1322 +		 * operation that gives us the lock. We xchg it to -1, so
1323 +		 * that when we release the lock, we properly wake up the
1324 +		 * other waiters:
1325 +		 */
1326 +		if (atomic_xchg(&lock->count, -1) == 1)
1327 +			break;
1328 +		
1329 +		__set_task_state(task, state);
1330 +		
1331 +		/* didnt get the lock, go to sleep: */
1332 +		spin_unlock_mutex(&lock->wait_lock, flags);
1333 +		preempt_enable_no_resched();
1334 +		schedule();
1335 +		preempt_disable();
1336 +		spin_lock_mutex(&lock->wait_lock, flags);
1337 +	}
1338 +	
1339 +done:
1340 +	lock_acquired(&lock->dep_map, ip);
1341 +	/* got the lock - rejoice! */
1342 +	mutex_remove_waiter(lock, &waiter, current_thread_info());
1343 +	mutex_set_owner(lock);
1344 +	
1345 +	/* set it to 0 if there are no waiters left: */
1346 +	if (likely(list_empty(&lock->wait_list)))
1347 +		atomic_set(&lock->count, 0);
1348 +	
1349 +	if(post)
1350 +		post(post_arg);	
1351 +	
1352 +	spin_unlock_mutex(&lock->wait_lock, flags);
1353 +	
1354 +	debug_mutex_free_waiter(&waiter);
1355 +	preempt_enable();
1356 +	
1357 +	//return 0;	
1358 +}
1359 +EXPORT_SYMBOL(mutex_lock_sfx);
1360 +
1361 +
1362 +
1363 +//__mutex_unlock_common_slowpath(lock_count, 1);
1364 +
1365 +void mutex_unlock_sfx(struct mutex *lock,
1366 +					side_effect_t pre, unsigned long pre_arg,
1367 +					side_effect_t post, unsigned long post_arg)
1368 +{
1369 +	//struct mutex *lock = container_of(lock_count, struct mutex, count);
1370 +	unsigned long flags;
1371 +	
1372 +	spin_lock_mutex(&lock->wait_lock, flags);
1373 +	
1374 +	if(pre)
1375 +		pre(pre_arg);
1376 +	
1377 +	//mutex_release(&lock->dep_map, nested, _RET_IP_);
1378 +	mutex_release(&lock->dep_map, 1, _RET_IP_);
1379 +	debug_mutex_unlock(lock);
1380 +	
1381 +	/*
1382 +	 * some architectures leave the lock unlocked in the fastpath failure
1383 +	 * case, others need to leave it locked. In the later case we have to
1384 +	 * unlock it here
1385 +	 */
1386 +	if (__mutex_slowpath_needs_to_unlock())
1387 +		atomic_set(&lock->count, 1);
1388 +	
1389 +	if (!list_empty(&lock->wait_list)) {
1390 +		/* get the first entry from the wait-list: */
1391 +		struct mutex_waiter *waiter =
1392 +		list_entry(lock->wait_list.next,
1393 +				   struct mutex_waiter, list);
1394 +		
1395 +		debug_mutex_wake_waiter(lock, waiter);
1396 +		
1397 +		wake_up_process(waiter->task);
1398 +	}
1399 +	
1400 +	if(post)
1401 +		post(post_arg);
1402 +	
1403 +	spin_unlock_mutex(&lock->wait_lock, flags);	
1404 +}
1405 +EXPORT_SYMBOL(mutex_unlock_sfx);
1406 diff --git a/kernel/sched.c b/kernel/sched.c
1407 index c5d7750..08b725c 100644
1408 --- a/kernel/sched.c
1409 +++ b/kernel/sched.c
1410 @@ -82,6 +82,10 @@
1411  #include <litmus/sched_trace.h>
1412  #include <litmus/trace.h>
1413  
1414 +#ifdef CONFIG_LITMUS_SOFTIRQD
1415 +#include <litmus/litmus_softirq.h>
1416 +#endif
1417 +
1418  static void litmus_tick(struct rq*, struct task_struct*);
1419  
1420  #define CREATE_TRACE_POINTS
1421 @@ -2879,6 +2883,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
1422  	struct mm_struct *mm, *oldmm;
1423  
1424  	prepare_task_switch(rq, prev, next);
1425 +
1426  	trace_sched_switch(prev, next);
1427  	mm = next->mm;
1428  	oldmm = prev->active_mm;
1429 @@ -3789,6 +3794,7 @@ pick_next_task(struct rq *rq)
1430  	}
1431  }
1432  
1433 +
1434  /*
1435   * schedule() is the main scheduler function.
1436   */
1437 @@ -3807,6 +3813,10 @@ need_resched:
1438  	rcu_note_context_switch(cpu);
1439  	prev = rq->curr;
1440  
1441 +#ifdef CONFIG_LITMUS_SOFTIRQD
1442 +	release_klitirqd_lock(prev);
1443 +#endif
1444 +	
1445  	release_kernel_lock(prev);
1446  need_resched_nonpreemptible:
1447  	TS_SCHED_START;
1448 @@ -3882,15 +3892,26 @@ need_resched_nonpreemptible:
1449  
1450  	if (sched_state_validate_switch() || unlikely(reacquire_kernel_lock(prev)))
1451  		goto need_resched_nonpreemptible;
1452 -
1453 +	
1454  	preempt_enable_no_resched();
1455 +
1456  	if (need_resched())
1457  		goto need_resched;
1458  
1459 +#ifdef LITMUS_SOFTIRQD
1460 +	reacquire_klitirqd_lock(prev);
1461 +#endif
1462 +
1463 +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1464 +	litmus->run_tasklets(prev);
1465 +#endif	
1466 +	
1467  	srp_ceiling_block();
1468  }
1469  EXPORT_SYMBOL(schedule);
1470  
1471 +
1472 +
1473  #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
1474  /*
1475   * Look out! "owner" is an entirely speculative pointer
1476 @@ -4051,6 +4072,7 @@ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
1477  	}
1478  }
1479  
1480 +
1481  /**
1482   * __wake_up - wake up threads blocked on a waitqueue.
1483   * @q: the waitqueue
1484 @@ -4236,6 +4258,12 @@ void __sched wait_for_completion(struct completion *x)
1485  }
1486  EXPORT_SYMBOL(wait_for_completion);
1487  
1488 +void __sched __wait_for_completion_locked(struct completion *x)
1489 +{
1490 +	do_wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
1491 +}
1492 +EXPORT_SYMBOL(__wait_for_completion_locked);
1493 +
1494  /**
1495   * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
1496   * @x:  holds the state of this particular completion
1497 diff --git a/kernel/semaphore.c b/kernel/semaphore.c
1498 index 94a62c0..c947a04 100644
1499 --- a/kernel/semaphore.c
1500 +++ b/kernel/semaphore.c
1501 @@ -33,11 +33,11 @@
1502  #include <linux/spinlock.h>
1503  #include <linux/ftrace.h>
1504  
1505 -static noinline void __down(struct semaphore *sem);
1506 +noinline void __down(struct semaphore *sem);
1507  static noinline int __down_interruptible(struct semaphore *sem);
1508  static noinline int __down_killable(struct semaphore *sem);
1509  static noinline int __down_timeout(struct semaphore *sem, long jiffies);
1510 -static noinline void __up(struct semaphore *sem);
1511 +noinline void __up(struct semaphore *sem);
1512  
1513  /**
1514   * down - acquire the semaphore
1515 @@ -190,11 +190,13 @@ EXPORT_SYMBOL(up);
1516  
1517  /* Functions for the contended case */
1518  
1519 +/*
1520  struct semaphore_waiter {
1521  	struct list_head list;
1522  	struct task_struct *task;
1523  	int up;
1524  };
1525 + */
1526  
1527  /*
1528   * Because this function is inlined, the 'state' parameter will be
1529 @@ -233,10 +235,12 @@ static inline int __sched __down_common(struct semaphore *sem, long state,
1530  	return -EINTR;
1531  }
1532  
1533 -static noinline void __sched __down(struct semaphore *sem)
1534 +noinline void __sched __down(struct semaphore *sem)
1535  {
1536  	__down_common(sem, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
1537  }
1538 +EXPORT_SYMBOL(__down);
1539 +
1540  
1541  static noinline int __sched __down_interruptible(struct semaphore *sem)
1542  {
1543 @@ -253,7 +257,7 @@ static noinline int __sched __down_timeout(struct semaphore *sem, long jiffies)
1544  	return __down_common(sem, TASK_UNINTERRUPTIBLE, jiffies);
1545  }
1546  
1547 -static noinline void __sched __up(struct semaphore *sem)
1548 +noinline void __sched __up(struct semaphore *sem)
1549  {
1550  	struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list,
1551  						struct semaphore_waiter, list);
1552 @@ -261,3 +265,4 @@ static noinline void __sched __up(struct semaphore *sem)
1553  	waiter->up = 1;
1554  	wake_up_process(waiter->task);
1555  }
1556 +EXPORT_SYMBOL(__up);
1557 \ No newline at end of file
1558 diff --git a/kernel/softirq.c b/kernel/softirq.c
1559 index 07b4f1b..7a6f500 100644
1560 --- a/kernel/softirq.c
1561 +++ b/kernel/softirq.c
1562 @@ -29,6 +29,15 @@
1563  #include <trace/events/irq.h>
1564  
1565  #include <asm/irq.h>
1566 +
1567 +#include <litmus/litmus.h>
1568 +#include <litmus/sched_trace.h>
1569 +
1570 +#ifdef CONFIG_LITMUS_NVIDIA
1571 +#include <litmus/nvidia_info.h>
1572 +#include <litmus/trace.h>
1573 +#endif
1574 +
1575  /*
1576     - No shared variables, all the data are CPU local.
1577     - If a softirq needs serialization, let it serialize itself
1578 @@ -54,7 +63,7 @@ EXPORT_SYMBOL(irq_stat);
1579  
1580  static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
1581  
1582 -static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
1583 +static DEFINE_PER_CPU(struct task_struct *, ksoftirqd) = NULL;
1584  
1585  char *softirq_to_name[NR_SOFTIRQS] = {
1586  	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
1587 @@ -177,6 +186,7 @@ void local_bh_enable_ip(unsigned long ip)
1588  }
1589  EXPORT_SYMBOL(local_bh_enable_ip);
1590  
1591 +
1592  /*
1593   * We restart softirq processing MAX_SOFTIRQ_RESTART times,
1594   * and we fall back to softirqd after that.
1595 @@ -187,34 +197,30 @@ EXPORT_SYMBOL(local_bh_enable_ip);
1596   * should not be able to lock up the box.
1597   */
1598  #define MAX_SOFTIRQ_RESTART 10
1599 -
1600 -asmlinkage void __do_softirq(void)
1601 +static void ____do_softirq(void)
1602  {
1603 -	struct softirq_action *h;
1604  	__u32 pending;
1605 -	int max_restart = MAX_SOFTIRQ_RESTART;
1606 +	
1607 +	struct softirq_action *h;
1608  	int cpu;
1609 -
1610 +	
1611  	pending = local_softirq_pending();
1612 +	
1613  	account_system_vtime(current);
1614 -
1615 -	__local_bh_disable((unsigned long)__builtin_return_address(0));
1616 -	lockdep_softirq_enter();
1617 -
1618 +	
1619  	cpu = smp_processor_id();
1620 -restart:
1621 -	/* Reset the pending bitmask before enabling irqs */
1622 -	set_softirq_pending(0);
1623  
1624 +	set_softirq_pending(0);
1625 +	
1626  	local_irq_enable();
1627 -
1628 +	
1629  	h = softirq_vec;
1630 -
1631 +	
1632  	do {
1633  		if (pending & 1) {
1634  			int prev_count = preempt_count();
1635  			kstat_incr_softirqs_this_cpu(h - softirq_vec);
1636 -
1637 +			
1638  			trace_softirq_entry(h, softirq_vec);
1639  			h->action(h);
1640  			trace_softirq_exit(h, softirq_vec);
1641 @@ -226,26 +232,70 @@ restart:
1642  				       h->action, prev_count, preempt_count());
1643  				preempt_count() = prev_count;
1644  			}
1645 -
1646 +			
1647  			rcu_bh_qs(cpu);
1648  		}
1649  		h++;
1650  		pending >>= 1;
1651  	} while (pending);
1652 -
1653 +	
1654  	local_irq_disable();
1655 +}
1656 +
1657 +static void ___do_softirq(void)
1658 +{
1659 +	__u32 pending;
1660 +
1661 +	//struct softirq_action *h;
1662 +	int max_restart = MAX_SOFTIRQ_RESTART;
1663 +	//int cpu;
1664 +
1665 +	pending = local_softirq_pending();
1666 +
1667 +restart:
1668 +	____do_softirq();
1669  
1670  	pending = local_softirq_pending();
1671  	if (pending && --max_restart)
1672  		goto restart;
1673  
1674  	if (pending)
1675 +	{
1676  		wakeup_softirqd();
1677 +	}
1678 +}
1679  
1680 +asmlinkage void __do_softirq(void)
1681 +{
1682 +#ifdef LITMUS_THREAD_ALL_SOFTIRQ
1683 +	/* Skip straight to wakeup_softirqd() if we're using 
1684 +	 LITMUS_THREAD_ALL_SOFTIRQ (unless there's really high prio-stuff waiting.). */
1685 +	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
1686 +	
1687 +	if(tsk)
1688 +	{
1689 +		__u32 pending = local_softirq_pending();
1690 +		const __u32 high_prio_softirq = (1<<HI_SOFTIRQ) | (1<<TIMER_SOFTIRQ) | (1<<HRTIMER_SOFTIRQ);
1691 +		if(pending && !(pending & high_prio_softirq))
1692 +		{
1693 +			wakeup_softirqd();
1694 +			return;
1695 +		}
1696 +	}
1697 +#endif
1698 +	
1699 +	/*
1700 +	 * 'immediate' softirq execution:
1701 +	 */
1702 +	__local_bh_disable((unsigned long)__builtin_return_address(0));
1703 +	lockdep_softirq_enter();
1704 +	
1705 +	___do_softirq();
1706 +	
1707  	lockdep_softirq_exit();
1708 -
1709 +	
1710  	account_system_vtime(current);
1711 -	_local_bh_enable();
1712 +	_local_bh_enable();	
1713  }
1714  
1715  #ifndef __ARCH_HAS_DO_SOFTIRQ
1716 @@ -357,8 +407,65 @@ struct tasklet_head
1717  static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
1718  static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
1719  
1720 +
1721  void __tasklet_schedule(struct tasklet_struct *t)
1722  {
1723 +#ifdef CONFIG_LITMUS_NVIDIA
1724 +	if(is_nvidia_func(t->func))
1725 +	{
1726 +		u32 nvidia_device = get_tasklet_nv_device_num(t);	
1727 +		//		TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
1728 +		//			  __FUNCTION__, nvidia_device,litmus_clock());
1729 +
1730 +		unsigned long flags;
1731 +		struct task_struct* device_owner;
1732 +
1733 +		lock_nv_registry(nvidia_device, &flags);
1734 +
1735 +		device_owner = get_nv_device_owner(nvidia_device);
1736 +
1737 +		if(device_owner==NULL)
1738 +		{
1739 +			t->owner = NULL;
1740 +		}
1741 +		else
1742 +		{
1743 +			if(is_realtime(device_owner))
1744 +			{
1745 +				TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
1746 +					  __FUNCTION__, nvidia_device,litmus_clock());				
1747 +				TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
1748 +					  __FUNCTION__,device_owner->pid,nvidia_device);
1749 +
1750 +				t->owner = device_owner;
1751 +				sched_trace_tasklet_release(t->owner);
1752 +
1753 +				if(likely(_litmus_tasklet_schedule(t,nvidia_device)))
1754 +				{
1755 +					unlock_nv_registry(nvidia_device, &flags);
1756 +					return;
1757 +				}
1758 +				else
1759 +				{
1760 +					t->owner = NULL; /* fall through to normal scheduling */
1761 +				}
1762 +			}
1763 +			else
1764 +			{
1765 +				t->owner = NULL;
1766 +			}
1767 +		}
1768 +		unlock_nv_registry(nvidia_device, &flags);
1769 +	}
1770 +#endif
1771 +
1772 +	___tasklet_schedule(t);
1773 +}
1774 +EXPORT_SYMBOL(__tasklet_schedule);
1775 +
1776 +
1777 +void ___tasklet_schedule(struct tasklet_struct *t)
1778 +{
1779  	unsigned long flags;
1780  
1781  	local_irq_save(flags);
1782 @@ -368,11 +475,65 @@ void __tasklet_schedule(struct tasklet_struct *t)
1783  	raise_softirq_irqoff(TASKLET_SOFTIRQ);
1784  	local_irq_restore(flags);
1785  }
1786 +EXPORT_SYMBOL(___tasklet_schedule);
1787  
1788 -EXPORT_SYMBOL(__tasklet_schedule);
1789  
1790  void __tasklet_hi_schedule(struct tasklet_struct *t)
1791  {
1792 +#ifdef CONFIG_LITMUS_NVIDIA
1793 +	if(is_nvidia_func(t->func))
1794 +	{	
1795 +		u32 nvidia_device = get_tasklet_nv_device_num(t);
1796 +		//		TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
1797 +		//			  __FUNCTION__, nvidia_device,litmus_clock());
1798 +
1799 +		unsigned long flags;
1800 +		struct task_struct* device_owner;
1801 +		
1802 +		lock_nv_registry(nvidia_device, &flags);
1803 +		
1804 +		device_owner = get_nv_device_owner(nvidia_device);
1805 +
1806 +		if(device_owner==NULL) 
1807 +		{
1808 +			t->owner = NULL;
1809 +		}
1810 +		else
1811 +		{
1812 +			if( is_realtime(device_owner))
1813 +			{
1814 +				TRACE("%s: Handling NVIDIA tasklet for device %u\tat %llu\n",
1815 +					  __FUNCTION__, nvidia_device,litmus_clock());				
1816 +				TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
1817 +					  __FUNCTION__,device_owner->pid,nvidia_device);
1818 +				
1819 +				t->owner = device_owner;
1820 +				sched_trace_tasklet_release(t->owner);
1821 +				if(likely(_litmus_tasklet_hi_schedule(t,nvidia_device)))
1822 +				{
1823 +					unlock_nv_registry(nvidia_device, &flags);
1824 +					return;
1825 +				}
1826 +				else
1827 +				{
1828 +					t->owner = NULL; /* fall through to normal scheduling */
1829 +				}
1830 +			}
1831 +			else
1832 +			{
1833 +				t->owner = NULL;
1834 +			}
1835 +		}
1836 +		unlock_nv_registry(nvidia_device, &flags);
1837 +	}
1838 +#endif
1839 +
1840 +	___tasklet_hi_schedule(t);
1841 +}
1842 +EXPORT_SYMBOL(__tasklet_hi_schedule);
1843 +
1844 +void ___tasklet_hi_schedule(struct tasklet_struct* t)
1845 +{
1846  	unsigned long flags;
1847  
1848  	local_irq_save(flags);
1849 @@ -382,19 +543,72 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
1850  	raise_softirq_irqoff(HI_SOFTIRQ);
1851  	local_irq_restore(flags);
1852  }
1853 -
1854 -EXPORT_SYMBOL(__tasklet_hi_schedule);
1855 +EXPORT_SYMBOL(___tasklet_hi_schedule);
1856  
1857  void __tasklet_hi_schedule_first(struct tasklet_struct *t)
1858  {
1859  	BUG_ON(!irqs_disabled());
1860 +#ifdef CONFIG_LITMUS_NVIDIA	
1861 +	if(is_nvidia_func(t->func))
1862 +	{	
1863 +		u32 nvidia_device = get_tasklet_nv_device_num(t);
1864 +		//		TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
1865 +		//			  __FUNCTION__, nvidia_device,litmus_clock());
1866 +		unsigned long flags;
1867 +		struct task_struct* device_owner;
1868 +		
1869 +		lock_nv_registry(nvidia_device, &flags);
1870 +
1871 +		device_owner = get_nv_device_owner(nvidia_device);
1872 +
1873 +		if(device_owner==NULL)
1874 +		{
1875 +			t->owner = NULL;
1876 +		}
1877 +		else
1878 +		{
1879 +			if(is_realtime(device_owner))
1880 +			{
1881 +				TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
1882 +					  __FUNCTION__, nvidia_device,litmus_clock());
1883 +				
1884 +				TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
1885 +					  __FUNCTION__,device_owner->pid,nvidia_device);
1886 +				
1887 +				t->owner = device_owner;
1888 +				sched_trace_tasklet_release(t->owner);
1889 +				if(likely(_litmus_tasklet_hi_schedule_first(t,nvidia_device)))
1890 +				{
1891 +					unlock_nv_registry(nvidia_device, &flags);
1892 +					return;
1893 +				}
1894 +				else
1895 +				{
1896 +					t->owner = NULL; /* fall through to normal scheduling */
1897 +				}
1898 +			}
1899 +			else
1900 +			{
1901 +				t->owner = NULL;
1902 +			}
1903 +		}
1904 +		unlock_nv_registry(nvidia_device, &flags);
1905 +	}
1906 +#endif
1907 +
1908 +	___tasklet_hi_schedule_first(t);
1909 +}
1910 +EXPORT_SYMBOL(__tasklet_hi_schedule_first);
1911 +
1912 +void ___tasklet_hi_schedule_first(struct tasklet_struct* t)
1913 +{
1914 +	BUG_ON(!irqs_disabled());
1915  
1916  	t->next = __get_cpu_var(tasklet_hi_vec).head;
1917  	__get_cpu_var(tasklet_hi_vec).head = t;
1918  	__raise_softirq_irqoff(HI_SOFTIRQ);
1919  }
1920 -
1921 -EXPORT_SYMBOL(__tasklet_hi_schedule_first);
1922 +EXPORT_SYMBOL(___tasklet_hi_schedule_first);
1923  
1924  static void tasklet_action(struct softirq_action *a)
1925  {
1926 @@ -450,6 +664,7 @@ static void tasklet_hi_action(struct softirq_action *a)
1927  			if (!atomic_read(&t->count)) {
1928  				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
1929  					BUG();
1930 +
1931  				t->func(t->data);
1932  				tasklet_unlock(t);
1933  				continue;
1934 @@ -473,8 +688,13 @@ void tasklet_init(struct tasklet_struct *t,
1935  	t->next = NULL;
1936  	t->state = 0;
1937  	atomic_set(&t->count, 0);
1938 +
1939  	t->func = func;
1940  	t->data = data;
1941 +
1942 +#ifdef CONFIG_LITMUS_SOFTIRQD
1943 +	t->owner = NULL;
1944 +#endif
1945  }
1946  
1947  EXPORT_SYMBOL(tasklet_init);
1948 @@ -489,6 +709,7 @@ void tasklet_kill(struct tasklet_struct *t)
1949  			yield();
1950  		} while (test_bit(TASKLET_STATE_SCHED, &t->state));
1951  	}
1952 +
1953  	tasklet_unlock_wait(t);
1954  	clear_bit(TASKLET_STATE_SCHED, &t->state);
1955  }
1956 @@ -694,6 +915,8 @@ void __init softirq_init(void)
1957  
1958  static int run_ksoftirqd(void * __bind_cpu)
1959  {
1960 +	unsigned long flags;
1961 +	
1962  	set_current_state(TASK_INTERRUPTIBLE);
1963  
1964  	while (!kthread_should_stop()) {
1965 @@ -712,7 +935,11 @@ static int run_ksoftirqd(void * __bind_cpu)
1966  			   don't process */
1967  			if (cpu_is_offline((long)__bind_cpu))
1968  				goto wait_to_die;
1969 -			do_softirq();
1970 +			
1971 +			local_irq_save(flags);
1972 +			____do_softirq();
1973 +			local_irq_restore(flags);
1974 +			
1975  			preempt_enable_no_resched();
1976  			cond_resched();
1977  			preempt_disable();
1978 @@ -760,6 +987,7 @@ void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
1979  	for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
1980  		if (*i == t) {
1981  			*i = t->next;
1982 +
1983  			/* If this was the tail element, move the tail ptr */
1984  			if (*i == NULL)
1985  				per_cpu(tasklet_vec, cpu).tail = i;
1986 diff --git a/kernel/workqueue.c b/kernel/workqueue.c
1987 index f77afd9..2293aad 100644
1988 --- a/kernel/workqueue.c
1989 +++ b/kernel/workqueue.c
1990 @@ -47,6 +47,13 @@
1991  
1992  #include "workqueue_sched.h"
1993  
1994 +#ifdef CONFIG_LITMUS_NVIDIA
1995 +#include <litmus/litmus.h>
1996 +#include <litmus/sched_trace.h>
1997 +#include <litmus/nvidia_info.h>
1998 +#endif
1999 +
2000 +
2001  enum {
2002  	/* global_cwq flags */
2003  	GCWQ_MANAGE_WORKERS	= 1 << 0,	/* need to manage workers */
2004 @@ -1010,9 +1017,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
2005  		work_flags |= WORK_STRUCT_DELAYED;
2006  		worklist = &cwq->delayed_works;
2007  	}
2008 -
2009  	insert_work(cwq, work, worklist, work_flags);
2010 -
2011  	spin_unlock_irqrestore(&gcwq->lock, flags);
2012  }
2013  
2014 @@ -2526,10 +2531,70 @@ EXPORT_SYMBOL(cancel_delayed_work_sync);
2015   */
2016  int schedule_work(struct work_struct *work)
2017  {
2018 -	return queue_work(system_wq, work);
2019 +#if 0
2020 +#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD)
2021 +	if(is_nvidia_func(work->func))
2022 +	{
2023 +		u32 nvidiaDevice = get_work_nv_device_num(work);
2024 +		
2025 +		//1) Ask Litmus which task owns GPU <nvidiaDevice>. (API to be defined.)
2026 +		unsigned long flags;
2027 +		struct task_struct* device_owner;
2028 +		
2029 +		lock_nv_registry(nvidiaDevice, &flags);
2030 +		
2031 +		device_owner = get_nv_device_owner(nvidiaDevice);
2032 +		
2033 +		//2) If there is an owner, set work->owner to the owner's task struct.
2034 +		if(device_owner==NULL) 
2035 +		{
2036 +			work->owner = NULL;
2037 +			//TRACE("%s: the owner task of NVIDIA Device %u is NULL\n",__FUNCTION__,nvidiaDevice);
2038 +		}
2039 +		else
2040 +		{
2041 +			if( is_realtime(device_owner))
2042 +			{
2043 +				TRACE("%s: Handling NVIDIA work for device\t%u\tat\t%llu\n",
2044 +					  __FUNCTION__, nvidiaDevice,litmus_clock());
2045 +				TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
2046 +					  __FUNCTION__,
2047 +					  device_owner->pid,
2048 +					  nvidiaDevice);
2049 +				
2050 +				//3) Call litmus_schedule_work() and return (don't execute the rest
2051 +				//	of schedule_schedule()).
2052 +				work->owner = device_owner;
2053 +				sched_trace_work_release(work->owner);
2054 +				if(likely(litmus_schedule_work(work, nvidiaDevice)))
2055 +				{
2056 +					unlock_nv_registry(nvidiaDevice, &flags);
2057 +					return 1;
2058 +				}
2059 +				else
2060 +				{
2061 +					work->owner = NULL; /* fall through to normal work scheduling */
2062 +				}
2063 +			}
2064 +			else
2065 +			{
2066 +				work->owner = NULL;
2067 +			}
2068 +		}
2069 +		unlock_nv_registry(nvidiaDevice, &flags);
2070 +	}
2071 +#endif
2072 +#endif
2073 +	return(__schedule_work(work));
2074  }
2075  EXPORT_SYMBOL(schedule_work);
2076  
2077 +int __schedule_work(struct work_struct* work)
2078 +{
2079 +	return queue_work(system_wq, work);
2080 +}
2081 +EXPORT_SYMBOL(__schedule_work);
2082 +
2083  /*
2084   * schedule_work_on - put work task on a specific cpu
2085   * @cpu: cpu to put the work task on
2086 diff --git a/litmus/Kconfig b/litmus/Kconfig
2087 index ad8dc83..5109cf7 100644
2088 --- a/litmus/Kconfig
2089 +++ b/litmus/Kconfig
2090 @@ -62,6 +62,25 @@ config LITMUS_LOCKING
2091  
2092  endmenu
2093  
2094 +menu "Performance Enhancements"
2095 +
2096 +config SCHED_CPU_AFFINITY
2097 +	bool "Local Migration Affinity"
2098 +	default y
2099 +	help
2100 +	  Rescheduled tasks prefer CPUs near to their previously used CPU.  This
2101 +	  may improve performance through possible preservation of cache affinity.
2102 +
2103 +	  Warning: May make bugs ahrder to find since tasks may migrate less often.
2104 +
2105 +	  NOTES:
2106 +	  	* Pfair/PD^2 does not support this option.
2107 +		* Only x86 currently supported.
2108 +
2109 +	  Say Yes if unsure.
2110 +
2111 +endmenu
2112 +
2113  menu "Tracing"
2114  
2115  config FEATHER_TRACE
2116 @@ -182,4 +201,106 @@ config SCHED_DEBUG_TRACE_CALLER
2117  
2118  endmenu
2119  
2120 +menu "Interrupt Handling"
2121 +
2122 +config LITMUS_THREAD_ALL_SOFTIRQ
2123 +       bool "Process all softirqs in ksoftirqd threads."
2124 +       default n
2125 +       help
2126 +	     (Experimental) Thread all softirqs to ksoftirqd
2127 +		 daemon threads, similar to PREEMPT_RT.  I/O
2128 +		 throughput will will drop with this enabled, but
2129 +		 latencies due to interrupts will be reduced.
2130 +
2131 +		 WARNING: Timer responsiveness will likely be
2132 +		 decreased as timer callbacks are also threaded.
2133 +		 This is unlike PREEEMPT_RTs hardirqs.
2134 +
2135 +		If unsure, say No.
2136 +
2137 +
2138 +choice 
2139 +	prompt "Scheduling of interrupt bottom-halves in Litmus."
2140 +	default LITMUS_SOFTIRQD_NONE
2141 +	depends on LITMUS_LOCKING && !LITMUS_THREAD_ALL_SOFTIRQ
2142 +	help
2143 +		Schedule tasklets with known priorities in Litmus.
2144 +
2145 +config LITMUS_SOFTIRQD_NONE
2146 +	bool "No tasklet scheduling in Litmus."
2147 +	help
2148 +	  Don't schedule tasklets in Litmus.  Default.
2149 +
2150 +config LITMUS_SOFTIRQD
2151 +	bool "Spawn klitirqd interrupt handling threads."
2152 +	help
2153 +	  Create klitirqd interrupt handling threads.  Work must be
2154 +	  specifically dispatched to these workers.  (Softirqs for
2155 +	  Litmus tasks are not magically redirected to klitirqd.)
2156 +
2157 +	  G-EDF/RM, C-EDF/RM ONLY for now!
2158 +
2159 +
2160 +config LITMUS_PAI_SOFTIRQD
2161 +	bool "Defer tasklets to context switch points."
2162 +	help
2163 +	  Only execute scheduled tasklet bottom halves at
2164 +	  scheduling points.  Trades context switch overhead
2165 +	  at the cost of non-preemptive durations of bottom half
2166 +	  processing.
2167 +		 
2168 +	  G-EDF/RM, C-EDF/RM ONLY for now!	 
2169 +		 
2170 +endchoice	   
2171 +	   
2172 +
2173 +config NR_LITMUS_SOFTIRQD
2174 +	   int "Number of klitirqd."
2175 +	   depends on LITMUS_SOFTIRQD
2176 +	   range 1 4096
2177 +	   default "1"
2178 +	   help
2179 +	     Should be <= to the number of CPUs in your system.
2180 +
2181 +config LITMUS_NVIDIA
2182 +	  bool "Litmus handling of NVIDIA interrupts."
2183 +	  depends on LITMUS_SOFTIRQD || LITMUS_PAI_SOFTIRQD
2184 +	  default n
2185 +	  help
2186 +	    Direct tasklets from NVIDIA devices to Litmus's klitirqd.
2187 +
2188 +		If unsure, say No.
2189 +
2190 +config NV_DEVICE_NUM
2191 +	   int "Number of NVIDIA GPUs."
2192 +	   depends on LITMUS_SOFTIRQD || LITMUS_PAI_SOFTIRQD
2193 +	   range 1 4096
2194 +	   default "1"
2195 +	   help
2196 +	     Should be (<= to the number of CPUs) and
2197 +		 (<= to the number of GPUs) in your system.
2198 +
2199 +choice
2200 +	  prompt "CUDA/Driver Version Support"
2201 +	  default CUDA_4_0
2202 +	  depends on LITMUS_NVIDIA
2203 +	  help
2204 +	  	Select the version of CUDA/driver to support.
2205 +	
2206 +config CUDA_4_0
2207 +	  bool "CUDA 4.0"
2208 +	  depends on LITMUS_NVIDIA
2209 +	  help
2210 +	  	Support CUDA 4.0 RC2 (dev. driver version: x86_64-270.40)
2211 +
2212 +config CUDA_3_2
2213 +	  bool "CUDA 3.2"
2214 +	  depends on LITMUS_NVIDIA
2215 +	  help
2216 +	  	Support CUDA 3.2 (dev. driver version: x86_64-260.24)
2217 +
2218 +endchoice
2219 +
2220 +endmenu
2221 +
2222  endmenu
2223 diff --git a/litmus/Makefile b/litmus/Makefile
2224 index ad9936e..869939e 100644
2225 --- a/litmus/Makefile
2226 +++ b/litmus/Makefile
2227 @@ -19,10 +19,15 @@ obj-y     = sched_plugin.o litmus.o \
2228  	    sched_gsn_edf.o \
2229  	    sched_psn_edf.o
2230  
2231 -obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o
2232 +obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o sched_cfifo.o fifo_common.o sched_crm.o rm_common.o sched_crm_srt.o rm_srt_common.o
2233  obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o
2234 +obj-$(CONFIG_SCHED_CPU_AFFINITY) += affinity.o
2235  
2236  obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o
2237  obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o
2238  obj-$(CONFIG_SCHED_DEBUG_TRACE) += sched_trace.o
2239  obj-$(CONFIG_SCHED_OVERHEAD_TRACE) += trace.o
2240 +
2241 +obj-$(CONFIG_LITMUS_SOFTIRQD) += litmus_softirq.o
2242 +obj-$(CONFIG_LITMUS_PAI_SOFTIRQD) += litmus_pai_softirq.o
2243 +obj-$(CONFIG_LITMUS_NVIDIA) += nvidia_info.o sched_trace_external.o
2244 diff --git a/litmus/affinity.c b/litmus/affinity.c
2245 new file mode 100644
2246 index 0000000..3b430d1
2247 --- /dev/null
2248 +++ b/litmus/affinity.c
2249 @@ -0,0 +1,49 @@
2250 +#include <linux/cpu.h>
2251 +
2252 +#include <litmus/affinity.h>
2253 +
2254 +struct neighborhood neigh_info[NR_CPUS];
2255 +
2256 +/* called by _init_litmus() */
2257 +void init_topology(void)
2258 +{
2259 +	int cpu;
2260 +	int i;
2261 +	int chk;
2262 +	int depth = num_cache_leaves;
2263 +
2264 +	if(depth > NUM_CACHE_LEVELS)
2265 +		depth = NUM_CACHE_LEVELS;
2266 +
2267 +	for_each_online_cpu(cpu)
2268 +	{
2269 +		for(i = 0; i < depth; ++i)
2270 +		{
2271 +			long unsigned int firstbits;
2272 +
2273 +			chk = get_shared_cpu_map((struct cpumask *)&neigh_info[cpu].neighbors[i], cpu, i);
2274 +			if(chk) /* failed */
2275 +			{
2276 +				neigh_info[cpu].size[i] = 0;
2277 +			}
2278 +			else
2279 +			{
2280 +				/* size = num bits in mask */
2281 +				neigh_info[cpu].size[i] = cpumask_weight((struct cpumask *)&neigh_info[cpu].neighbors[i]);
2282 +			}
2283 +			firstbits = *neigh_info[cpu].neighbors[i]->bits;
2284 +			printk("CPU %d has %d neighbors at level %d. (mask = %lx)\n",
2285 +							cpu, neigh_info[cpu].size[i], i, firstbits);
2286 +		}
2287 +
2288 +		/* set data for non-existent levels */
2289 +		for(; i < NUM_CACHE_LEVELS; ++i)
2290 +		{
2291 +			neigh_info[cpu].size[i] = 0;
2292 +
2293 +			printk("CPU %d has %d neighbors at level %d. (mask = %lx)\n",
2294 +							cpu, neigh_info[cpu].size[i], i, 0lu);
2295 +		}
2296 +	}
2297 +}
2298 +
2299 diff --git a/litmus/edf_common.c b/litmus/edf_common.c
2300 index 9b44dc2..0a06d7a 100644
2301 --- a/litmus/edf_common.c
2302 +++ b/litmus/edf_common.c
2303 @@ -63,8 +63,52 @@ int edf_higher_prio(struct task_struct* first,
2304  
2305  #endif
2306  
2307 +	if (!is_realtime(second_task))
2308 +		return true;
2309 +
2310 +	if (earlier_deadline(first_task, second_task))
2311 +		return true;
2312 +
2313 +	if (get_deadline(first_task) == get_deadline(second_task))
2314 +	{
2315 +		if (shorter_period(first_task, second_task))
2316 +		{
2317 +			return true;
2318 +		}
2319 +		if (get_rt_period(first_task) == get_rt_period(second_task))
2320 +		{
2321 +#ifdef CONFIG_LITMUS_SOFTIRQD
2322 +			if (first_task->rt_param.is_proxy_thread < second_task->rt_param.is_proxy_thread)
2323 +		    {
2324 +				return true;
2325 +			}
2326 +			if (first_task->rt_param.is_proxy_thread == second_task->rt_param.is_proxy_thread)
2327 +			{
2328 +#endif      
2329 +			if (first_task->pid < second_task->pid)
2330 +			{   
2331 +				return true;
2332 +			}
2333 +			if (first_task->pid == second_task->pid)
2334 +			{
2335 +				return !second->rt_param.inh_task;
2336 +			}
2337 +#ifdef CONFIG_LITMUS_SOFTIRQD
2338 +			}
2339 +#endif
2340 +		}
2341 +	}
2342 +	
2343 +	return false;
2344  
2345 +#if 0
2346  	return !is_realtime(second_task)  ||
2347 +    
2348 +#ifdef CONFIG_LITMUS_SOFTIRQD
2349 +        /* proxy threads always lose w/o inheritance. */
2350 +        (first_task->rt_param.is_proxy_thread <
2351 +            second_task->rt_param.is_proxy_thread) ||
2352 +#endif
2353  
2354  		/* is the deadline of the first task earlier?
2355  		 * Then it has higher priority.
2356 @@ -82,6 +126,7 @@ int edf_higher_prio(struct task_struct* first,
2357  		 */
2358  		(first_task->pid == second_task->pid &&
2359  		 !second->rt_param.inh_task)));
2360 +#endif
2361  }
2362  
2363  int edf_ready_order(struct bheap_node* a, struct bheap_node* b)
2364 diff --git a/litmus/fdso.c b/litmus/fdso.c
2365 index aa7b384..2b7f9ba 100644
2366 --- a/litmus/fdso.c
2367 +++ b/litmus/fdso.c
2368 @@ -22,6 +22,7 @@ extern struct fdso_ops generic_lock_ops;
2369  
2370  static const struct fdso_ops* fdso_ops[] = {
2371  	&generic_lock_ops, /* FMLP_SEM */
2372 +	&generic_lock_ops, /* KFMLP_SEM */
2373  	&generic_lock_ops, /* SRP_SEM */
2374  };
2375  
2376 diff --git a/litmus/fifo_common.c b/litmus/fifo_common.c
2377 new file mode 100644
2378 index 0000000..c94510a
2379 --- /dev/null
2380 +++ b/litmus/fifo_common.c
2381 @@ -0,0 +1,124 @@
2382 +/*
2383 + * kernel/fifo_common.c
2384 + *
2385 + * Common functions for EDF based scheduler.
2386 + */
2387 +
2388 +#include <linux/percpu.h>
2389 +#include <linux/sched.h>
2390 +#include <linux/list.h>
2391 +
2392 +#include <litmus/litmus.h>
2393 +#include <litmus/sched_plugin.h>
2394 +#include <litmus/sched_trace.h>
2395 +
2396 +#include <litmus/fifo_common.h>
2397 +
2398 +/* fifo_higher_prio -  returns true if first has a higher EDF priority
2399 + *                    than second. Deadline ties are broken by PID.
2400 + *
2401 + * both first and second may be NULL
2402 + */
2403 +int fifo_higher_prio(struct task_struct* first,
2404 +		    struct task_struct* second)
2405 +{
2406 +	struct task_struct *first_task = first;
2407 +	struct task_struct *second_task = second;
2408 +
2409 +	/* There is no point in comparing a task to itself. */
2410 +	if (first && first == second) {
2411 +		TRACE_TASK(first,
2412 +			   "WARNING: pointless edf priority comparison.\n");
2413 +		return 0;
2414 +	}
2415 +
2416 +
2417 +	/* check for NULL tasks */
2418 +	if (!first || !second)
2419 +		return first && !second;
2420 +
2421 +#ifdef CONFIG_LITMUS_LOCKING
2422 +
2423 +	/* Check for inherited priorities. Change task
2424 +	 * used for comparison in such a case.
2425 +	 */
2426 +	if (unlikely(first->rt_param.inh_task))
2427 +		first_task = first->rt_param.inh_task;
2428 +	if (unlikely(second->rt_param.inh_task))
2429 +		second_task = second->rt_param.inh_task;
2430 +
2431 +	/* Check for priority boosting. Tie-break by start of boosting.
2432 +	 */
2433 +	if (unlikely(is_priority_boosted(first_task))) {
2434 +		/* first_task is boosted, how about second_task? */
2435 +		if (!is_priority_boosted(second_task) ||
2436 +		    lt_before(get_boost_start(first_task),
2437 +			      get_boost_start(second_task)))
2438 +			return 1;
2439 +		else
2440 +			return 0;
2441 +	} else if (unlikely(is_priority_boosted(second_task)))
2442 +		/* second_task is boosted, first is not*/
2443 +		return 0;
2444 +
2445 +#endif
2446 +
2447 +
2448 +	return !is_realtime(second_task)  ||
2449 +    
2450 +#ifdef CONFIG_LITMUS_SOFTIRQD
2451 +        /* proxy threads always lose w/o inheritance. */
2452 +        (first_task->rt_param.is_proxy_thread <
2453 +            second_task->rt_param.is_proxy_thread) ||
2454 +#endif
2455 +
2456 +		/* is the deadline of the first task earlier?
2457 +		 * Then it has higher priority.
2458 +		 */
2459 +		earlier_release(first_task, second_task) ||
2460 +
2461 +		/* Do we have a deadline tie?
2462 +		 * Then break by PID.
2463 +		 */
2464 +		(get_release(first_task) == get_release(second_task) &&
2465 +	        (first_task->pid < second_task->pid ||
2466 +
2467 +		/* If the PIDs are the same then the task with the inherited
2468 +		 * priority wins.
2469 +		 */
2470 +		(first_task->pid == second_task->pid &&
2471 +		 !second->rt_param.inh_task)));
2472 +}
2473 +
2474 +int fifo_ready_order(struct bheap_node* a, struct bheap_node* b)
2475 +{
2476 +	return fifo_higher_prio(bheap2task(a), bheap2task(b));
2477 +}
2478 +
2479 +void fifo_domain_init(rt_domain_t* rt, check_resched_needed_t resched,
2480 +		      release_jobs_t release)
2481 +{
2482 +	rt_domain_init(rt,  fifo_ready_order, resched, release);
2483 +}
2484 +
2485 +/* need_to_preempt - check whether the task t needs to be preempted
2486 + *                   call only with irqs disabled and with  ready_lock acquired
2487 + *                   THIS DOES NOT TAKE NON-PREEMPTIVE SECTIONS INTO ACCOUNT!
2488 + */
2489 +int fifo_preemption_needed(rt_domain_t* rt, struct task_struct *t)
2490 +{
2491 +	/* we need the read lock for fifo_ready_queue */
2492 +	/* no need to preempt if there is nothing pending */
2493 +	if (!__jobs_pending(rt))
2494 +		return 0;
2495 +	/* we need to reschedule if t doesn't exist */
2496 +	if (!t)
2497 +		return 1;
2498 +
2499 +	/* NOTE: We cannot check for non-preemptibility since we
2500 +	 *       don't know what address space we're currently in.
2501 +	 */
2502 +
2503 +	/* make sure to get non-rt stuff out of the way */
2504 +	return !is_realtime(t) || fifo_higher_prio(__next_ready(rt), t);
2505 +}
2506 diff --git a/litmus/litmus.c b/litmus/litmus.c
2507 index 26938ac..29363c6 100644
2508 --- a/litmus/litmus.c
2509 +++ b/litmus/litmus.c
2510 @@ -17,6 +17,14 @@
2511  #include <litmus/litmus_proc.h>
2512  #include <litmus/sched_trace.h>
2513  
2514 +#ifdef CONFIG_SCHED_CPU_AFFINITY
2515 +#include <litmus/affinity.h>
2516 +#endif
2517 +
2518 +#ifdef CONFIG_LITMUS_NVIDIA
2519 +#include <litmus/nvidia_info.h>
2520 +#endif
2521 +
2522  /* Number of RT tasks that exist in the system */
2523  atomic_t rt_task_count 		= ATOMIC_INIT(0);
2524  static DEFINE_RAW_SPINLOCK(task_transition_lock);
2525 @@ -47,6 +55,28 @@ void bheap_node_free(struct bheap_node* hn)
2526  struct release_heap* release_heap_alloc(int gfp_flags);
2527  void release_heap_free(struct release_heap* rh);
2528  
2529 +#ifdef CONFIG_LITMUS_NVIDIA
2530 +/*
2531 + * sys_register_nv_device
2532 + * @nv_device_id: The Nvidia device id that the task want to register
2533 + * @reg_action: set to '1' to register the specified device. zero otherwise.
2534 + * Syscall for register task's designated nvidia device into NV_DEVICE_REG array
2535 + * Returns EFAULT  if nv_device_id is out of range.
2536 + *	   0       if success
2537 + */
2538 +asmlinkage long sys_register_nv_device(int nv_device_id, int reg_action)
2539 +{
2540 +	/* register the device to caller (aka 'current') */
2541 +	return(reg_nv_device(nv_device_id, reg_action));
2542 +}
2543 +#else
2544 +asmlinkage long sys_register_nv_device(int nv_device_id, int reg_action)
2545 +{
2546 +	return(-EINVAL);
2547 +}
2548 +#endif
2549 +
2550 +
2551  /*
2552   * sys_set_task_rt_param
2553   * @pid: Pid of the task which scheduling parameters must be changed
2554 @@ -115,7 +145,7 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param)
2555  		tp.cls != RT_CLASS_BEST_EFFORT)
2556  	{
2557  		printk(KERN_INFO "litmus: real-time task %d rejected "
2558 -				 "because its class is invalid\n");
2559 +				 "because its class is invalid\n", pid);
2560  		goto out_unlock;
2561  	}
2562  	if (tp.budget_policy != NO_ENFORCEMENT &&
2563 @@ -131,6 +161,22 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param)
2564  
2565  	target->rt_param.task_params = tp;
2566  
2567 +#ifdef CONFIG_LITMUS_SOFTIRQD
2568 +	/* proxy thread off by default */
2569 +	target->rt_param.is_proxy_thread = 0;
2570 +    target->rt_param.cur_klitirqd = NULL;
2571 +	//init_MUTEX(&target->rt_param.klitirqd_sem);
2572 +	mutex_init(&target->rt_param.klitirqd_sem);
2573 +	//init_completion(&target->rt_param.klitirqd_sem);
2574 +	//target->rt_param.klitirqd_sem_stat = NOT_HELD;
2575 +	atomic_set(&target->rt_param.klitirqd_sem_stat, NOT_HELD);
2576 +#endif
2577 +
2578 +#ifdef CONFIG_LITMUS_NVIDIA
2579 +	atomic_set(&target->rt_param.nv_int_count, 0);
2580 +#endif
2581 +
2582 +
2583  	retval = 0;
2584        out_unlock:
2585  	read_unlock_irq(&tasklist_lock);
2586 @@ -265,6 +311,7 @@ asmlinkage long sys_query_job_no(unsigned int __user *job)
2587  	return retval;
2588  }
2589  
2590 +
2591  /* sys_null_call() is only used for determining raw system call
2592   * overheads (kernel entry, kernel exit). It has no useful side effects.
2593   * If ts is non-NULL, then the current Feather-Trace time is recorded.
2594 @@ -278,7 +325,7 @@ asmlinkage long sys_null_call(cycles_t __user *ts)
2595  		now = get_cycles();
2596  		ret = put_user(now, ts);
2597  	}
2598 -
2599 +	
2600  	return ret;
2601  }
2602  
2603 @@ -299,6 +346,20 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
2604  	 * at this point in time.
2605  	 */
2606  	WARN_ON(p->rt_param.inh_task);
2607 +   
2608 +#ifdef CONFIG_LITMUS_SOFTIRQD
2609 +	/* We probably should not have any tasklets executing for
2610 +     * us at this time.
2611 +	 */    
2612 +    WARN_ON(p->rt_param.cur_klitirqd);
2613 +	WARN_ON(atomic_read(&p->rt_param.klitirqd_sem_stat) == HELD);
2614 +
2615 +	if(p->rt_param.cur_klitirqd)
2616 +		flush_pending(p->rt_param.cur_klitirqd, p);
2617 +
2618 +	if(atomic_read(&p->rt_param.klitirqd_sem_stat) == HELD)
2619 +		up_and_set_stat(p, NOT_HELD, &p->rt_param.klitirqd_sem);
2620 +#endif
2621  
2622  	/* Cleanup everything else. */
2623  	memset(&p->rt_param, 0, sizeof(p->rt_param));
2624 @@ -399,7 +460,7 @@ static void synch_on_plugin_switch(void* info)
2625   */
2626  int switch_sched_plugin(struct sched_plugin* plugin)
2627  {
2628 -	unsigned long flags;
2629 +	//unsigned long flags;
2630  	int ret = 0;
2631  
2632  	BUG_ON(!plugin);
2633 @@ -413,8 +474,15 @@ int switch_sched_plugin(struct sched_plugin* plugin)
2634  	while (atomic_read(&cannot_use_plugin) < num_online_cpus())
2635  		cpu_relax();
2636  
2637 +#ifdef CONFIG_LITMUS_SOFTIRQD
2638 +	if(!klitirqd_is_dead())
2639 +	{
2640 +		kill_klitirqd();
2641 +	}
2642 +#endif
2643 +
2644  	/* stop task transitions */
2645 -	raw_spin_lock_irqsave(&task_transition_lock, flags);
2646 +	//raw_spin_lock_irqsave(&task_transition_lock, flags);
2647  
2648  	/* don't switch if there are active real-time tasks */
2649  	if (atomic_read(&rt_task_count) == 0) {
2650 @@ -432,7 +500,7 @@ int switch_sched_plugin(struct sched_plugin* plugin)
2651  	} else
2652  		ret = -EBUSY;
2653  out:
2654 -	raw_spin_unlock_irqrestore(&task_transition_lock, flags);
2655 +	//raw_spin_unlock_irqrestore(&task_transition_lock, flags);
2656  	atomic_set(&cannot_use_plugin, 0);
2657  	return ret;
2658  }
2659 @@ -540,6 +608,10 @@ static int __init _init_litmus(void)
2660  
2661  	init_litmus_proc();
2662  
2663 +#ifdef CONFIG_SCHED_CPU_AFFINITY
2664 +	init_topology();
2665 +#endif
2666 +
2667  	return 0;
2668  }
2669  
2670 diff --git a/litmus/litmus_pai_softirq.c b/litmus/litmus_pai_softirq.c
2671 new file mode 100644
2672 index 0000000..b31eeb8
2673 --- /dev/null
2674 +++ b/litmus/litmus_pai_softirq.c
2675 @@ -0,0 +1,64 @@
2676 +#include <linux/interrupt.h>
2677 +#include <linux/percpu.h>
2678 +#include <linux/cpu.h>
2679 +#include <linux/kthread.h>
2680 +#include <linux/ftrace.h>
2681 +#include <linux/smp.h>
2682 +#include <linux/slab.h>
2683 +#include <linux/mutex.h>
2684 +
2685 +#include <linux/sched.h>
2686 +#include <linux/cpuset.h>
2687 +
2688 +#include <litmus/litmus.h>
2689 +#include <litmus/sched_trace.h>
2690 +#include <litmus/jobs.h>
2691 +#include <litmus/sched_plugin.h>
2692 +#include <litmus/litmus_softirq.h>
2693 +
2694 +
2695 +
2696 +int __litmus_tasklet_schedule(struct tasklet_struct *t, unsigned int k_id)
2697 +{
2698 +	int ret = 0; /* assume failure */
2699 +    if(unlikely((t->owner == NULL) || !is_realtime(t->owner)))
2700 +    {
2701 +        TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
2702 +        BUG();
2703 +    }
2704 +
2705 +    ret = litmus->enqueue_pai_tasklet(t);
2706 +	
2707 +	return(ret);
2708 +}
2709 +
2710 +EXPORT_SYMBOL(__litmus_tasklet_schedule);
2711 +
2712 +
2713 +
2714 +// failure causes default Linux handling.
2715 +int __litmus_tasklet_hi_schedule(struct tasklet_struct *t, unsigned int k_id)
2716 +{
2717 +	int ret = 0; /* assume failure */
2718 +	return(ret);
2719 +}
2720 +EXPORT_SYMBOL(__litmus_tasklet_hi_schedule);
2721 +
2722 +
2723 +// failure causes default Linux handling.
2724 +int __litmus_tasklet_hi_schedule_first(struct tasklet_struct *t, unsigned int k_id)
2725 +{
2726 +	int ret = 0; /* assume failure */
2727 +	return(ret);
2728 +}
2729 +EXPORT_SYMBOL(__litmus_tasklet_hi_schedule_first);
2730 +
2731 +
2732 +// failure causes default Linux handling.
2733 +int __litmus_schedule_work(struct work_struct *w, unsigned int k_id)
2734 +{
2735 +	int ret = 0; /* assume failure */
2736 +	return(ret);
2737 +}
2738 +EXPORT_SYMBOL(__litmus_schedule_work);
2739 +
2740 diff --git a/litmus/litmus_proc.c b/litmus/litmus_proc.c
2741 index 4bf725a..3815133 100644
2742 --- a/litmus/litmus_proc.c
2743 +++ b/litmus/litmus_proc.c
2744 @@ -20,11 +20,18 @@ static struct proc_dir_entry *litmus_dir = NULL,
2745  #ifdef CONFIG_RELEASE_MASTER
2746  	*release_master_file = NULL,
2747  #endif
2748 +#ifdef CONFIG_LITMUS_SOFTIRQD
2749 +	*klitirqd_file = NULL,
2750 +#endif
2751  	*plugs_file = NULL;
2752  
2753  /* in litmus/sync.c */
2754  int count_tasks_waiting_for_release(void);
2755  
2756 +extern int proc_read_klitirqd_stats(char *page, char **start,
2757 +									off_t off, int count,
2758 +									int *eof, void *data);
2759 +
2760  static int proc_read_stats(char *page, char **start,
2761  			   off_t off, int count,
2762  			   int *eof, void *data)
2763 @@ -161,6 +168,12 @@ int __init init_litmus_proc(void)
2764  	release_master_file->write_proc  = proc_write_release_master;
2765  #endif
2766  
2767 +#ifdef CONFIG_LITMUS_SOFTIRQD
2768 +	klitirqd_file =
2769 +		create_proc_read_entry("klitirqd_stats", 0444, litmus_dir,
2770 +							   proc_read_klitirqd_stats, NULL);
2771 +#endif	
2772 +	
2773  	stat_file = create_proc_read_entry("stats", 0444, litmus_dir,
2774  					   proc_read_stats, NULL);
2775  
2776 @@ -187,6 +200,10 @@ void exit_litmus_proc(void)
2777  		remove_proc_entry("stats", litmus_dir);
2778  	if (curr_file)
2779  		remove_proc_entry("active_plugin", litmus_dir);
2780 +#ifdef CONFIG_LITMUS_SOFTIRQD
2781 +	if (klitirqd_file)
2782 +		remove_proc_entry("klitirqd_stats", litmus_dir);
2783 +#endif
2784  #ifdef CONFIG_RELEASE_MASTER
2785  	if (release_master_file)
2786  		remove_proc_entry("release_master", litmus_dir);
2787 diff --git a/litmus/litmus_softirq.c b/litmus/litmus_softirq.c
2788 new file mode 100644
2789 index 0000000..c49676c
2790 --- /dev/null
2791 +++ b/litmus/litmus_softirq.c
2792 @@ -0,0 +1,1584 @@
2793 +#include <linux/interrupt.h>
2794 +#include <linux/percpu.h>
2795 +#include <linux/cpu.h>
2796 +#include <linux/kthread.h>
2797 +#include <linux/ftrace.h>
2798 +#include <linux/smp.h>
2799 +#include <linux/slab.h>
2800 +#include <linux/mutex.h>
2801 +
2802 +#include <linux/sched.h>
2803 +#include <linux/cpuset.h>
2804 +
2805 +#include <litmus/litmus.h>
2806 +#include <litmus/sched_trace.h>
2807 +#include <litmus/jobs.h>
2808 +#include <litmus/sched_plugin.h>
2809 +#include <litmus/litmus_softirq.h>
2810 +
2811 +/* TODO: Remove unneeded mb() and other barriers. */
2812 +
2813 +
2814 +/* counts number of daemons ready to handle litmus irqs. */
2815 +static atomic_t num_ready_klitirqds = ATOMIC_INIT(0);
2816 +
2817 +enum pending_flags
2818 +{
2819 +    LIT_TASKLET_LOW = 0x1,
2820 +    LIT_TASKLET_HI  = LIT_TASKLET_LOW<<1,
2821 +	LIT_WORK = LIT_TASKLET_HI<<1
2822 +};
2823 +
2824 +/* only support tasklet processing for now. */
2825 +struct tasklet_head
2826 +{
2827 +	struct tasklet_struct *head;
2828 +	struct tasklet_struct **tail;
2829 +};
2830 +
2831 +struct klitirqd_info
2832 +{
2833 +	struct task_struct*		klitirqd;
2834 +    struct task_struct*     current_owner;
2835 +    int						terminating;
2836 +
2837 +
2838 +	raw_spinlock_t			lock;
2839 +	
2840 +	u32						pending;
2841 +	atomic_t				num_hi_pending;
2842 +	atomic_t				num_low_pending;
2843 +	atomic_t				num_work_pending;
2844 +
2845 +	/* in order of priority */
2846 +	struct tasklet_head     pending_tasklets_hi;
2847 +	struct tasklet_head		pending_tasklets;
2848 +	struct list_head		worklist;
2849 +};
2850 +
2851 +/* one list for each klitirqd */
2852 +static struct klitirqd_info klitirqds[NR_LITMUS_SOFTIRQD];
2853 +
2854 +
2855 +
2856 +
2857 +
2858 +int proc_read_klitirqd_stats(char *page, char **start,
2859 +							 off_t off, int count,
2860 +							 int *eof, void *data)
2861 +{
2862 +	int len = snprintf(page, PAGE_SIZE,
2863 +				"num ready klitirqds: %d\n\n",
2864 +				atomic_read(&num_ready_klitirqds));
2865 +	
2866 +	if(klitirqd_is_ready())
2867 +	{
2868 +		int i;
2869 +		for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i)
2870 +		{
2871 +			len +=
2872 +				snprintf(page + len - 1, PAGE_SIZE, /* -1 to strip off \0 */
2873 +						 "klitirqd_th%d: %s/%d\n"
2874 +						 "\tcurrent_owner: %s/%d\n"
2875 +						 "\tpending: %x\n"
2876 +						 "\tnum hi: %d\n"
2877 +						 "\tnum low: %d\n"
2878 +						 "\tnum work: %d\n\n",
2879 +						 i,
2880 +						 klitirqds[i].klitirqd->comm, klitirqds[i].klitirqd->pid,
2881 +						 (klitirqds[i].current_owner != NULL) ?
2882 +						 	klitirqds[i].current_owner->comm : "(null)",
2883 +						 (klitirqds[i].current_owner != NULL) ?
2884 +							klitirqds[i].current_owner->pid : 0,
2885 +						 klitirqds[i].pending,
2886 +						 atomic_read(&klitirqds[i].num_hi_pending),
2887 +						 atomic_read(&klitirqds[i].num_low_pending),
2888 +						 atomic_read(&klitirqds[i].num_work_pending));
2889 +		}
2890 +	}
2891 +
2892 +	return(len);
2893 +}
2894 +
2895 +				   
2896 +
2897 +
2898 +
2899 +#if 0
2900 +static atomic_t dump_id = ATOMIC_INIT(0);
2901 +
2902 +static void __dump_state(struct klitirqd_info* which, const char* caller)
2903 +{
2904 +	struct tasklet_struct* list;
2905 +
2906 +	int id = atomic_inc_return(&dump_id);
2907 +
2908 +	//if(in_interrupt())
2909 +	{
2910 +		if(which->current_owner)
2911 +		{
2912 +			TRACE("(id: %d  caller: %s)\n"
2913 +				"klitirqd: %s/%d\n"
2914 +				"current owner: %s/%d\n"
2915 +				"pending: %x\n",
2916 +				id, caller,
2917 +				which->klitirqd->comm, which->klitirqd->pid,
2918 +				which->current_owner->comm, which->current_owner->pid,
2919 +				which->pending);
2920 +		}
2921 +		else
2922 +		{
2923 +			TRACE("(id: %d  caller: %s)\n"
2924 +				"klitirqd: %s/%d\n"
2925 +				"current owner: %p\n"
2926 +				"pending: %x\n",
2927 +				id, caller,
2928 +				which->klitirqd->comm, which->klitirqd->pid,
2929 +				NULL,
2930 +				which->pending);
2931 +		}
2932 +
2933 +		list = which->pending_tasklets.head;
2934 +		while(list)
2935 +		{
2936 +			struct tasklet_struct *t = list;
2937 +			list = list->next; /* advance */
2938 +			if(t->owner)
2939 +				TRACE("(id: %d  caller: %s) Tasklet: %x, Owner = %s/%d\n", id, caller, t, t->owner->comm, t->owner->pid);
2940 +			else
2941 +				TRACE("(id: %d  caller: %s) Tasklet: %x, Owner = %p\n", id, caller, t, NULL);
2942 +		}
2943 +	}
2944 +}
2945 +
2946 +static void dump_state(struct klitirqd_info* which, const char* caller)
2947 +{
2948 +	unsigned long flags;
2949 +
2950 +	raw_spin_lock_irqsave(&which->lock, flags);
2951 +    __dump_state(which, caller);
2952 +    raw_spin_unlock_irqrestore(&which->lock, flags);
2953 +}
2954 +#endif
2955 +
2956 +
2957 +/* forward declarations */
2958 +static void ___litmus_tasklet_schedule(struct tasklet_struct *t,
2959 +									   struct klitirqd_info *which,
2960 +									   int wakeup);
2961 +static void ___litmus_tasklet_hi_schedule(struct tasklet_struct *t,
2962 +										  struct klitirqd_info *which,
2963 +										  int wakeup);
2964 +static void ___litmus_schedule_work(struct work_struct *w,
2965 +									struct klitirqd_info *which,
2966 +									int wakeup);
2967 +
2968 +
2969 +
2970 +inline unsigned int klitirqd_id(struct task_struct* tsk)
2971 +{
2972 +    int i;
2973 +    for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i)
2974 +    {
2975 +        if(klitirqds[i].klitirqd == tsk)
2976 +        {
2977 +            return i;
2978 +        }
2979 +    }
2980 +    
2981 +    BUG();
2982 +    
2983 +    return 0;
2984 +}
2985 +
2986 +
2987 +inline static u32 litirq_pending_hi_irqoff(struct klitirqd_info* which)
2988 +{
2989 +    return (which->pending & LIT_TASKLET_HI);
2990 +}
2991 +
2992 +inline static u32 litirq_pending_low_irqoff(struct klitirqd_info* which)
2993 +{
2994 +    return (which->pending & LIT_TASKLET_LOW);
2995 +}
2996 +
2997 +inline static u32 litirq_pending_work_irqoff(struct klitirqd_info* which)
2998 +{
2999 +	return (which->pending & LIT_WORK);
3000 +}
3001 +
3002 +inline static u32 litirq_pending_irqoff(struct klitirqd_info* which)
3003 +{
3004 +    return(which->pending);
3005 +}
3006 +
3007 +
3008 +inline static u32 litirq_pending(struct klitirqd_info* which)
3009 +{
3010 +    unsigned long flags;
3011 +    u32 pending;
3012 +    
3013 +    raw_spin_lock_irqsave(&which->lock, flags);
3014 +    pending = litirq_pending_irqoff(which);
3015 +    raw_spin_unlock_irqrestore(&which->lock, flags);
3016 +    
3017 +    return pending;
3018 +};
3019 +
3020 +inline static u32 litirq_pending_with_owner(struct klitirqd_info* which, struct task_struct* owner)
3021 +{
3022 +	unsigned long flags;
3023 +	u32 pending;
3024 +
3025 +	raw_spin_lock_irqsave(&which->lock, flags);
3026 +	pending = litirq_pending_irqoff(which);
3027 +	if(pending)
3028 +	{
3029 +		if(which->current_owner != owner)
3030 +		{
3031 +			pending = 0;  // owner switch!
3032 +		}
3033 +	}
3034 +	raw_spin_unlock_irqrestore(&which->lock, flags);
3035 +
3036 +	return pending;
3037 +}
3038 +
3039 +
3040 +inline static u32 litirq_pending_and_sem_and_owner(struct klitirqd_info* which,
3041 +				struct mutex** sem,
3042 +				struct task_struct** t)
3043 +{
3044 +	unsigned long flags;
3045 +	u32 pending;
3046 +
3047 +	/* init values */
3048 +	*sem = NULL;
3049 +	*t = NULL;
3050 +
3051 +	raw_spin_lock_irqsave(&which->lock, flags);
3052 +
3053 +	pending = litirq_pending_irqoff(which);
3054 +	if(pending)
3055 +	{
3056 +		if(which->current_owner != NULL)
3057 +		{
3058 +			*t = which->current_owner;
3059 +			*sem = &tsk_rt(which->current_owner)->klitirqd_sem;
3060 +		}
3061 +		else
3062 +		{
3063 +			BUG();
3064 +		}
3065 +	}
3066 +	raw_spin_unlock_irqrestore(&which->lock, flags);
3067 +
3068 +	if(likely(*sem))
3069 +	{
3070 +		return pending;
3071 +	}
3072 +	else
3073 +	{
3074 +		return 0;
3075 +	}
3076 +}
3077 +
3078 +/* returns true if the next piece of work to do is from a different owner.
3079 + */
3080 +static int tasklet_ownership_change(
3081 +				struct klitirqd_info* which,
3082 +				enum pending_flags taskletQ)
3083 +{
3084 +	/* this function doesn't have to look at work objects since they have
3085 +	   priority below tasklets. */
3086 +
3087 +    unsigned long flags;
3088 +    int ret = 0;
3089 +
3090 +    raw_spin_lock_irqsave(&which->lock, flags);
3091 +    
3092 +	switch(taskletQ)
3093 +	{
3094 +	case LIT_TASKLET_HI:
3095 +		if(litirq_pending_hi_irqoff(which))
3096 +		{
3097 +			ret = (which->pending_tasklets_hi.head->owner != 
3098 +						which->current_owner);
3099 +		}
3100 +		break;
3101 +	case LIT_TASKLET_LOW:
3102 +		if(litirq_pending_low_irqoff(which))
3103 +		{
3104 +			ret = (which->pending_tasklets.head->owner !=
3105 +						which->current_owner);
3106 +		}
3107 +		break;
3108 +	default:
3109 +		break;
3110 +	}
3111 +    
3112 +    raw_spin_unlock_irqrestore(&which->lock, flags);
3113 +    
3114 +    TRACE_TASK(which->klitirqd, "ownership change needed: %d\n", ret);
3115 +    
3116 +    return ret;
3117 +}
3118 +
3119 +
3120 +static void __reeval_prio(struct klitirqd_info* which)
3121 +{
3122 +    struct task_struct* next_owner = NULL;
3123