Attachment 'gpusync-rtss12.patch'

Download

   1 From a463f9a9e04385f0729f7435a0a6dff7d89b25de Mon Sep 17 00:00:00 2001
   2 From: Glenn Elliott <gelliott@cs.unc.edu>
   3 Date: Sat, 26 May 2012 17:29:58 -0400
   4 Subject: [PATCH] GPUSync patch for Litmus 2012.1.
   5 
   6 ---
   7  arch/x86/kernel/irq.c                 |    4 +
   8  arch/x86/kernel/syscall_table_32.S    |    1 +
   9  include/linux/completion.h            |    1 +
  10  include/linux/interrupt.h             |   10 +-
  11  include/linux/mutex.h                 |   10 +
  12  include/linux/semaphore.h             |    9 +
  13  include/linux/workqueue.h             |   18 +
  14  include/litmus/binheap.h              |  207 +++
  15  include/litmus/edf_common.h           |   12 +
  16  include/litmus/fdso.h                 |   14 +-
  17  include/litmus/fpmath.h               |  145 ++
  18  include/litmus/gpu_affinity.h         |   49 +
  19  include/litmus/ikglp_lock.h           |  160 ++
  20  include/litmus/kexclu_affinity.h      |   35 +
  21  include/litmus/kfmlp_lock.h           |   97 ++
  22  include/litmus/litmus.h               |    9 +-
  23  include/litmus/litmus_softirq.h       |  199 +++
  24  include/litmus/locking.h              |  142 +-
  25  include/litmus/nvidia_info.h          |   46 +
  26  include/litmus/preempt.h              |    2 +-
  27  include/litmus/rsm_lock.h             |   54 +
  28  include/litmus/rt_param.h             |  100 +-
  29  include/litmus/sched_plugin.h         |   76 +-
  30  include/litmus/sched_trace.h          |  218 ++-
  31  include/litmus/sched_trace_external.h |   78 +
  32  include/litmus/trace.h                |   34 +-
  33  include/litmus/unistd_32.h            |    5 +-
  34  include/litmus/unistd_64.h            |    9 +-
  35  kernel/lockdep.c                      |    7 +-
  36  kernel/mutex.c                        |  125 ++
  37  kernel/sched.c                        |   27 +
  38  kernel/semaphore.c                    |   13 +-
  39  kernel/softirq.c                      |  322 +++-
  40  kernel/workqueue.c                    |   71 +-
  41  litmus/Kconfig                        |  148 +-
  42  litmus/Makefile                       |   11 +-
  43  litmus/affinity.c                     |    2 +-
  44  litmus/binheap.c                      |  443 +++++
  45  litmus/edf_common.c                   |  147 +-
  46  litmus/fdso.c                         |   13 +
  47  litmus/gpu_affinity.c                 |  113 ++
  48  litmus/ikglp_lock.c                   | 2838 +++++++++++++++++++++++++++++++++
  49  litmus/jobs.c                         |   17 +-
  50  litmus/kexclu_affinity.c              |   92 ++
  51  litmus/kfmlp_lock.c                   | 1002 ++++++++++++
  52  litmus/litmus.c                       |  126 +-
  53  litmus/litmus_pai_softirq.c           |   64 +
  54  litmus/litmus_proc.c                  |   17 +
  55  litmus/litmus_softirq.c               | 1582 ++++++++++++++++++
  56  litmus/locking.c                      |  393 ++++-
  57  litmus/nvidia_info.c                  |  597 +++++++
  58  litmus/preempt.c                      |    5 +
  59  litmus/rsm_lock.c                     |  796 +++++++++
  60  litmus/sched_cedf.c                   | 1062 +++++++++++-
  61  litmus/sched_gsn_edf.c                | 1032 ++++++++++--
  62  litmus/sched_litmus.c                 |    2 +
  63  litmus/sched_plugin.c                 |  135 +-
  64  litmus/sched_task_trace.c             |  282 +++-
  65  litmus/sched_trace_external.c         |   64 +
  66  59 files changed, 13012 insertions(+), 280 deletions(-)
  67  create mode 100644 include/litmus/binheap.h
  68  create mode 100644 include/litmus/fpmath.h
  69  create mode 100644 include/litmus/gpu_affinity.h
  70  create mode 100644 include/litmus/ikglp_lock.h
  71  create mode 100644 include/litmus/kexclu_affinity.h
  72  create mode 100644 include/litmus/kfmlp_lock.h
  73  create mode 100644 include/litmus/litmus_softirq.h
  74  create mode 100644 include/litmus/nvidia_info.h
  75  create mode 100644 include/litmus/rsm_lock.h
  76  create mode 100644 include/litmus/sched_trace_external.h
  77  create mode 100644 litmus/binheap.c
  78  create mode 100644 litmus/gpu_affinity.c
  79  create mode 100644 litmus/ikglp_lock.c
  80  create mode 100644 litmus/kexclu_affinity.c
  81  create mode 100644 litmus/kfmlp_lock.c
  82  create mode 100644 litmus/litmus_pai_softirq.c
  83  create mode 100644 litmus/litmus_softirq.c
  84  create mode 100644 litmus/nvidia_info.c
  85  create mode 100644 litmus/rsm_lock.c
  86  create mode 100644 litmus/sched_trace_external.c
  87 
  88 diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
  89 index 6c0802e..680a5cb 100644
  90 --- a/arch/x86/kernel/irq.c
  91 +++ b/arch/x86/kernel/irq.c
  92 @@ -10,6 +10,10 @@
  93  #include <linux/ftrace.h>
  94  #include <linux/delay.h>
  95  
  96 +#ifdef CONFIG_LITMUS_NVIDIA
  97 +#include <litmus/sched_trace.h>
  98 +#endif
  99 +
 100  #include <asm/apic.h>
 101  #include <asm/io_apic.h>
 102  #include <asm/irq.h>
 103 diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S
 104 index d012622..0cb4373 100644
 105 --- a/arch/x86/kernel/syscall_table_32.S
 106 +++ b/arch/x86/kernel/syscall_table_32.S
 107 @@ -358,3 +358,4 @@ ENTRY(sys_call_table)
 108  	.long sys_wait_for_ts_release
 109  	.long sys_release_ts		/* +10 */
 110  	.long sys_null_call
 111 +	.long sys_register_nv_device
 112 diff --git a/include/linux/completion.h b/include/linux/completion.h
 113 index 9d72727..cff405c 100644
 114 --- a/include/linux/completion.h
 115 +++ b/include/linux/completion.h
 116 @@ -76,6 +76,7 @@ static inline void init_completion(struct completion *x)
 117  	init_waitqueue_head(&x->wait);
 118  }
 119  
 120 +extern void __wait_for_completion_locked(struct completion *);
 121  extern void wait_for_completion(struct completion *);
 122  extern int wait_for_completion_interruptible(struct completion *x);
 123  extern int wait_for_completion_killable(struct completion *x);
 124 diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
 125 index f6efed0..57a7bc8 100644
 126 --- a/include/linux/interrupt.h
 127 +++ b/include/linux/interrupt.h
 128 @@ -445,6 +445,7 @@ static inline void __raise_softirq_irqoff(unsigned int nr)
 129  
 130  extern void raise_softirq_irqoff(unsigned int nr);
 131  extern void raise_softirq(unsigned int nr);
 132 +extern void wakeup_softirqd(void);
 133  
 134  /* This is the worklist that queues up per-cpu softirq work.
 135   *
 136 @@ -500,6 +501,10 @@ struct tasklet_struct
 137  	atomic_t count;
 138  	void (*func)(unsigned long);
 139  	unsigned long data;
 140 +
 141 +#if defined(CONFIG_LITMUS_SOFTIRQD) || defined(CONFIG_LITMUS_PAI_SOFTIRQD)
 142 +	struct task_struct *owner;
 143 +#endif
 144  };
 145  
 146  #define DECLARE_TASKLET(name, func, data) \
 147 @@ -537,6 +542,7 @@ static inline void tasklet_unlock_wait(struct tasklet_struct *t)
 148  #define tasklet_unlock(t) do { } while (0)
 149  #endif
 150  
 151 +extern void ___tasklet_schedule(struct tasklet_struct *t);
 152  extern void __tasklet_schedule(struct tasklet_struct *t);
 153  
 154  static inline void tasklet_schedule(struct tasklet_struct *t)
 155 @@ -545,6 +551,7 @@ static inline void tasklet_schedule(struct tasklet_struct *t)
 156  		__tasklet_schedule(t);
 157  }
 158  
 159 +extern void ___tasklet_hi_schedule(struct tasklet_struct *t);
 160  extern void __tasklet_hi_schedule(struct tasklet_struct *t);
 161  
 162  static inline void tasklet_hi_schedule(struct tasklet_struct *t)
 163 @@ -553,6 +560,7 @@ static inline void tasklet_hi_schedule(struct tasklet_struct *t)
 164  		__tasklet_hi_schedule(t);
 165  }
 166  
 167 +extern void ___tasklet_hi_schedule_first(struct tasklet_struct *t);
 168  extern void __tasklet_hi_schedule_first(struct tasklet_struct *t);
 169  
 170  /*
 171 @@ -582,7 +590,7 @@ static inline void tasklet_disable(struct tasklet_struct *t)
 172  }
 173  
 174  static inline void tasklet_enable(struct tasklet_struct *t)
 175 -{
 176 +{	
 177  	smp_mb__before_atomic_dec();
 178  	atomic_dec(&t->count);
 179  }
 180 diff --git a/include/linux/mutex.h b/include/linux/mutex.h
 181 index a940fe4..cb47deb 100644
 182 --- a/include/linux/mutex.h
 183 +++ b/include/linux/mutex.h
 184 @@ -126,6 +126,15 @@ static inline int mutex_is_locked(struct mutex *lock)
 185  	return atomic_read(&lock->count) != 1;
 186  }
 187  
 188 +/* return non-zero to abort.  only pre-side-effects may abort */
 189 +typedef int (*side_effect_t)(unsigned long);
 190 +extern void mutex_lock_sfx(struct mutex *lock,
 191 +						   side_effect_t pre, unsigned long pre_arg,
 192 +						   side_effect_t post, unsigned long post_arg);
 193 +extern void mutex_unlock_sfx(struct mutex *lock,
 194 +							 side_effect_t pre, unsigned long pre_arg,
 195 +							 side_effect_t post, unsigned long post_arg);
 196 +
 197  /*
 198   * See kernel/mutex.c for detailed documentation of these APIs.
 199   * Also see Documentation/mutex-design.txt.
 200 @@ -153,6 +162,7 @@ extern void mutex_lock(struct mutex *lock);
 201  extern int __must_check mutex_lock_interruptible(struct mutex *lock);
 202  extern int __must_check mutex_lock_killable(struct mutex *lock);
 203  
 204 +
 205  # define mutex_lock_nested(lock, subclass) mutex_lock(lock)
 206  # define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock)
 207  # define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock)
 208 diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h
 209 index 39fa049..c83fc2b 100644
 210 --- a/include/linux/semaphore.h
 211 +++ b/include/linux/semaphore.h
 212 @@ -43,4 +43,13 @@ extern int __must_check down_trylock(struct semaphore *sem);
 213  extern int __must_check down_timeout(struct semaphore *sem, long jiffies);
 214  extern void up(struct semaphore *sem);
 215  
 216 +extern void __down(struct semaphore *sem);
 217 +extern void __up(struct semaphore *sem);
 218 +
 219 +struct semaphore_waiter {
 220 +	struct list_head list;
 221 +	struct task_struct *task;
 222 +	int up;
 223 +};
 224 +
 225  #endif /* __LINUX_SEMAPHORE_H */
 226 diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
 227 index f584aba..1ec2ec7 100644
 228 --- a/include/linux/workqueue.h
 229 +++ b/include/linux/workqueue.h
 230 @@ -83,6 +83,9 @@ struct work_struct {
 231  #ifdef CONFIG_LOCKDEP
 232  	struct lockdep_map lockdep_map;
 233  #endif
 234 +#ifdef CONFIG_LITMUS_SOFTIRQD
 235 +	struct task_struct *owner;
 236 +#endif
 237  };
 238  
 239  #define WORK_DATA_INIT()	ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU)
 240 @@ -115,11 +118,25 @@ struct execute_work {
 241  #define __WORK_INIT_LOCKDEP_MAP(n, k)
 242  #endif
 243  
 244 +#ifdef CONFIG_LITMUS_SOFTIRQD
 245 +#define __WORK_INIT_OWNER() \
 246 +	.owner = NULL,
 247 +
 248 +#define PREPARE_OWNER(_work, _owner) \
 249 +	do { \
 250 +		(_work)->owner = (_owner); \
 251 +	} while(0)
 252 +#else
 253 +#define __WORK_INIT_OWNER()
 254 +#define PREPARE_OWNER(_work, _owner)
 255 +#endif
 256 +
 257  #define __WORK_INITIALIZER(n, f) {				\
 258  	.data = WORK_DATA_STATIC_INIT(),			\
 259  	.entry	= { &(n).entry, &(n).entry },			\
 260  	.func = (f),						\
 261  	__WORK_INIT_LOCKDEP_MAP(#n, &(n))			\
 262 +	__WORK_INIT_OWNER() \
 263  	}
 264  
 265  #define __DELAYED_WORK_INITIALIZER(n, f) {			\
 266 @@ -357,6 +374,7 @@ extern int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
 267  extern void flush_workqueue(struct workqueue_struct *wq);
 268  extern void flush_scheduled_work(void);
 269  
 270 +extern int __schedule_work(struct work_struct *work);
 271  extern int schedule_work(struct work_struct *work);
 272  extern int schedule_work_on(int cpu, struct work_struct *work);
 273  extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay);
 274 diff --git a/include/litmus/binheap.h b/include/litmus/binheap.h
 275 new file mode 100644
 276 index 0000000..9e966e3
 277 --- /dev/null
 278 +++ b/include/litmus/binheap.h
 279 @@ -0,0 +1,207 @@
 280 +#ifndef LITMUS_BINARY_HEAP_H
 281 +#define LITMUS_BINARY_HEAP_H
 282 +
 283 +#include <linux/kernel.h>
 284 +
 285 +/**
 286 + * Simple binary heap with add, arbitrary delete, delete_root, and top
 287 + * operations.
 288 + *
 289 + * Style meant to conform with list.h.
 290 + *
 291 + * Motivation: Linux's prio_heap.h is of fixed size. Litmus's binomial
 292 + * heap may be overkill (and perhaps not general enough) for some applications.
 293 + *
 294 + * Note: In order to make node swaps fast, a node inserted with a data pointer
 295 + * may not always hold said data pointer. This is similar to the binomial heap
 296 + * implementation. This does make node deletion tricky since we have to
 297 + * (1) locate the node that holds the data pointer to delete, and (2) the
 298 + * node that was originally inserted with said data pointer. These have to be
 299 + * coalesced into a single node before removal (see usage of
 300 + * __binheap_safe_swap()). We have to track node references to accomplish this.
 301 + */
 302 +
 303 +struct binheap_node {
 304 +	void	*data;
 305 +	struct binheap_node *parent;
 306 +	struct binheap_node *left;
 307 +	struct binheap_node *right;
 308 +
 309 +	/* pointer to binheap_node that holds *data for which this binheap_node
 310 +	 * was originally inserted.  (*data "owns" this node)
 311 +	 */
 312 +	struct binheap_node *ref;
 313 +	struct binheap_node **ref_ptr;
 314 +};
 315 +
 316 +/**
 317 + * Signature of compator function.  Assumed 'less-than' (min-heap).
 318 + * Pass in 'greater-than' for max-heap.
 319 + *
 320 + * TODO: Consider macro-based implementation that allows comparator to be
 321 + * inlined (similar to Linux red/black tree) for greater efficiency.
 322 + */
 323 +typedef int (*binheap_order_t)(struct binheap_node *a,
 324 +							   struct binheap_node *b);
 325 +
 326 +
 327 +struct binheap_handle {
 328 +	struct binheap_node *root;
 329 +
 330 +	/* pointer to node to take next inserted child */
 331 +	struct binheap_node *next;
 332 +
 333 +	/* pointer to last node in complete binary tree */
 334 +	struct binheap_node *last;
 335 +
 336 +	/* comparator function pointer */
 337 +	binheap_order_t compare;
 338 +};
 339 +
 340 +
 341 +#define BINHEAP_POISON	((void*)(0xdeadbeef))
 342 +
 343 +
 344 +/**
 345 + * binheap_entry - get the struct for this heap node.
 346 + *  Only valid when called upon heap nodes other than the root handle.
 347 + * @ptr:	the heap node.
 348 + * @type:	the type of struct pointed to by binheap_node::data.
 349 + * @member:	unused.
 350 + */
 351 +#define binheap_entry(ptr, type, member) \
 352 +((type *)((ptr)->data))
 353 +
 354 +/**
 355 + * binheap_node_container - get the struct that contains this node.
 356 + *  Only valid when called upon heap nodes other than the root handle.
 357 + * @ptr:	the heap node.
 358 + * @type:	the type of struct the node is embedded in.
 359 + * @member:	the name of the binheap_struct within the (type) struct.
 360 + */
 361 +#define binheap_node_container(ptr, type, member) \
 362 +container_of((ptr), type, member)
 363 +
 364 +/**
 365 + * binheap_top_entry - get the struct for the node at the top of the heap.
 366 + *  Only valid when called upon the heap handle node.
 367 + * @ptr:    the special heap-handle node.
 368 + * @type:   the type of the struct the head is embedded in.
 369 + * @member:	the name of the binheap_struct within the (type) struct.
 370 + */
 371 +#define binheap_top_entry(ptr, type, member) \
 372 +binheap_entry((ptr)->root, type, member)
 373 +
 374 +/**
 375 + * binheap_delete_root - remove the root element from the heap.
 376 + * @handle:	 handle to the heap.
 377 + * @type:    the type of the struct the head is embedded in.
 378 + * @member:	 the name of the binheap_struct within the (type) struct.
 379 + */
 380 +#define binheap_delete_root(handle, type, member) \
 381 +__binheap_delete_root((handle), &((type *)((handle)->root->data))->member)
 382 +
 383 +/**
 384 + * binheap_delete - remove an arbitrary element from the heap.
 385 + * @to_delete:  pointer to node to be removed.
 386 + * @handle:	 handle to the heap.
 387 + */
 388 +#define binheap_delete(to_delete, handle) \
 389 +__binheap_delete((to_delete), (handle))
 390 +
 391 +/**
 392 + * binheap_add - insert an element to the heap
 393 + * new_node: node to add.
 394 + * @handle:	 handle to the heap.
 395 + * @type:    the type of the struct the head is embedded in.
 396 + * @member:	 the name of the binheap_struct within the (type) struct.
 397 + */
 398 +#define binheap_add(new_node, handle, type, member) \
 399 +__binheap_add((new_node), (handle), container_of((new_node), type, member))
 400 +
 401 +/**
 402 + * binheap_decrease - re-eval the position of a node (based upon its
 403 + * original data pointer).
 404 + * @handle: handle to the heap.
 405 + * @orig_node: node that was associated with the data pointer
 406 + *             (whose value has changed) when said pointer was
 407 + *             added to the heap.
 408 + */
 409 +#define binheap_decrease(orig_node, handle) \
 410 +__binheap_decrease((orig_node), (handle))
 411 +
 412 +#define BINHEAP_NODE_INIT() { NULL, BINHEAP_POISON, NULL, NULL , NULL, NULL}
 413 +
 414 +#define BINHEAP_NODE(name) \
 415 +	struct binheap_node name = BINHEAP_NODE_INIT()
 416 +
 417 +
 418 +static inline void INIT_BINHEAP_NODE(struct binheap_node *n)
 419 +{
 420 +	n->data = NULL;
 421 +	n->parent = BINHEAP_POISON;
 422 +	n->left = NULL;
 423 +	n->right = NULL;
 424 +	n->ref = NULL;
 425 +	n->ref_ptr = NULL;
 426 +}
 427 +
 428 +static inline void INIT_BINHEAP_HANDLE(
 429 +	struct binheap_handle *handle,
 430 +	binheap_order_t compare)
 431 +{
 432 +	handle->root = NULL;
 433 +	handle->next = NULL;
 434 +	handle->last = NULL;
 435 +	handle->compare = compare;
 436 +}
 437 +
 438 +/* Returns true (1) if binheap is empty. */
 439 +static inline int binheap_empty(struct binheap_handle *handle)
 440 +{
 441 +	return(handle->root == NULL);
 442 +}
 443 +
 444 +/* Returns true (1) if binheap node is in a heap. */
 445 +static inline int binheap_is_in_heap(struct binheap_node *node)
 446 +{
 447 +	return (node->parent != BINHEAP_POISON);
 448 +}
 449 +
 450 +
 451 +int binheap_is_in_this_heap(struct binheap_node *node, struct binheap_handle* heap);
 452 +
 453 +
 454 +
 455 +void __binheap_add(struct binheap_node *new_node,
 456 +	struct binheap_handle *handle,
 457 +	void *data);
 458 +
 459 +
 460 +/**
 461 + * Removes the root node from the heap. The node is removed after coalescing
 462 + * the binheap_node with its original data pointer at the root of the tree.
 463 + *
 464 + * The 'last' node in the tree is then swapped up to the root and bubbled
 465 + * down.
 466 + */
 467 +void __binheap_delete_root(struct binheap_handle *handle,
 468 +	struct binheap_node *container);
 469 +
 470 +/**
 471 + * Delete an arbitrary node.  Bubble node to delete up to the root,
 472 + * and then delete to root.
 473 + */
 474 +void __binheap_delete(
 475 +	struct binheap_node *node_to_delete,
 476 +	struct binheap_handle *handle);
 477 +
 478 +/**
 479 + * Bubble up a node whose pointer has decreased in value.
 480 + */
 481 +void __binheap_decrease(struct binheap_node *orig_node,
 482 +						struct binheap_handle *handle);
 483 +
 484 +
 485 +#endif
 486 +
 487 diff --git a/include/litmus/edf_common.h b/include/litmus/edf_common.h
 488 index bbaf22e..63dff7e 100644
 489 --- a/include/litmus/edf_common.h
 490 +++ b/include/litmus/edf_common.h
 491 @@ -20,6 +20,18 @@ int edf_higher_prio(struct task_struct* first,
 492  
 493  int edf_ready_order(struct bheap_node* a, struct bheap_node* b);
 494  
 495 +#ifdef CONFIG_LITMUS_NESTED_LOCKING
 496 +/* binheap_nodes must be embedded within 'struct litmus_lock' */
 497 +int edf_max_heap_order(struct binheap_node *a, struct binheap_node *b);
 498 +int edf_min_heap_order(struct binheap_node *a, struct binheap_node *b);
 499 +int edf_max_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b);
 500 +int edf_min_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b);
 501 +
 502 +int __edf_higher_prio(struct task_struct* first, comparison_mode_t first_mode,
 503 +					  struct task_struct* second, comparison_mode_t second_mode);
 504 +
 505 +#endif
 506 +
 507  int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t);
 508  
 509  #endif
 510 diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h
 511 index caf2a1e..1f5d3bd 100644
 512 --- a/include/litmus/fdso.h
 513 +++ b/include/litmus/fdso.h
 514 @@ -20,7 +20,16 @@ typedef enum  {
 515  	FMLP_SEM	= 0,
 516  	SRP_SEM		= 1,
 517  
 518 -	MAX_OBJ_TYPE	= 1
 519 +	RSM_MUTEX	= 2,
 520 +	IKGLP_SEM	= 3,
 521 +	KFMLP_SEM	= 4,
 522 +
 523 +	IKGLP_SIMPLE_GPU_AFF_OBS = 5,
 524 +	IKGLP_GPU_AFF_OBS = 6,
 525 +	KFMLP_SIMPLE_GPU_AFF_OBS = 7,
 526 +	KFMLP_GPU_AFF_OBS = 8,
 527 +
 528 +	MAX_OBJ_TYPE	= 8
 529  } obj_type_t;
 530  
 531  struct inode_obj_id {
 532 @@ -64,8 +73,11 @@ static inline void* od_lookup(int od, obj_type_t type)
 533  }
 534  
 535  #define lookup_fmlp_sem(od)((struct pi_semaphore*)  od_lookup(od, FMLP_SEM))
 536 +#define lookup_kfmlp_sem(od)((struct pi_semaphore*)  od_lookup(od, KFMLP_SEM))
 537  #define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM))
 538  #define lookup_ics(od)     ((struct ics*)           od_lookup(od, ICS_ID))
 539  
 540 +#define lookup_rsm_mutex(od)((struct litmus_lock*)  od_lookup(od, FMLP_SEM))
 541 +
 542  
 543  #endif
 544 diff --git a/include/litmus/fpmath.h b/include/litmus/fpmath.h
 545 new file mode 100644
 546 index 0000000..04d4bca
 547 --- /dev/null
 548 +++ b/include/litmus/fpmath.h
 549 @@ -0,0 +1,145 @@
 550 +#ifndef __FP_MATH_H__
 551 +#define __FP_MATH_H__
 552 +
 553 +#ifndef __KERNEL__
 554 +#include <stdint.h>
 555 +#define abs(x) (((x) < 0) ? -(x) : x)
 556 +#endif
 557 +
 558 +// Use 64-bit because we want to track things at the nanosecond scale.
 559 +// This can lead to very large numbers.
 560 +typedef int64_t fpbuf_t;
 561 +typedef struct
 562 +{
 563 +	fpbuf_t val;
 564 +} fp_t;
 565 +
 566 +#define FP_SHIFT 10
 567 +#define ROUND_BIT (FP_SHIFT - 1)
 568 +
 569 +#define _fp(x) ((fp_t) {x})
 570 +
 571 +#ifdef __KERNEL__
 572 +static const fp_t LITMUS_FP_ZERO = {.val = 0};
 573 +static const fp_t LITMUS_FP_ONE = {.val = (1 << FP_SHIFT)};
 574 +#endif
 575 +
 576 +static inline fp_t FP(fpbuf_t x)
 577 +{
 578 +	return _fp(((fpbuf_t) x) << FP_SHIFT);
 579 +}
 580 +
 581 +/* divide two integers to obtain a fixed point value  */
 582 +static inline fp_t _frac(fpbuf_t a, fpbuf_t b)
 583 +{
 584 +	return _fp(FP(a).val / (b));
 585 +}
 586 +
 587 +static inline fpbuf_t _point(fp_t x)
 588 +{
 589 +	return (x.val % (1 << FP_SHIFT));
 590 +
 591 +}
 592 +
 593 +#define fp2str(x) x.val
 594 +/*(x.val >> FP_SHIFT), (x.val % (1 << FP_SHIFT)) */
 595 +#define _FP_  "%ld/1024"
 596 +
 597 +static inline fpbuf_t _floor(fp_t x)
 598 +{
 599 +	return x.val >> FP_SHIFT;
 600 +}
 601 +
 602 +/* FIXME: negative rounding */
 603 +static inline fpbuf_t _round(fp_t x)
 604 +{
 605 +	return _floor(x) + ((x.val >> ROUND_BIT) & 1);
 606 +}
 607 +
 608 +/* multiply two fixed point values */
 609 +static inline fp_t _mul(fp_t a, fp_t b)
 610 +{
 611 +	return _fp((a.val * b.val) >> FP_SHIFT);
 612 +}
 613 +
 614 +static inline fp_t _div(fp_t a, fp_t b)
 615 +{
 616 +#if !defined(__KERNEL__) && !defined(unlikely)
 617 +#define unlikely(x) (x)
 618 +#define DO_UNDEF_UNLIKELY
 619 +#endif
 620 +	/* try not to overflow */
 621 +	if (unlikely(  a.val > (2l << ((sizeof(fpbuf_t)*8) - FP_SHIFT)) ))
 622 +		return _fp((a.val / b.val) << FP_SHIFT);
 623 +	else
 624 +		return _fp((a.val << FP_SHIFT) / b.val);
 625 +#ifdef DO_UNDEF_UNLIKELY
 626 +#undef unlikely
 627 +#undef DO_UNDEF_UNLIKELY
 628 +#endif
 629 +}
 630 +
 631 +static inline fp_t _add(fp_t a, fp_t b)
 632 +{
 633 +	return _fp(a.val + b.val);
 634 +}
 635 +
 636 +static inline fp_t _sub(fp_t a, fp_t b)
 637 +{
 638 +	return _fp(a.val - b.val);
 639 +}
 640 +
 641 +static inline fp_t _neg(fp_t x)
 642 +{
 643 +	return _fp(-x.val);
 644 +}
 645 +
 646 +static inline fp_t _abs(fp_t x)
 647 +{
 648 +	return _fp(abs(x.val));
 649 +}
 650 +
 651 +/* works the same as casting float/double to integer */
 652 +static inline fpbuf_t _fp_to_integer(fp_t x)
 653 +{
 654 +	return _floor(_abs(x)) * ((x.val > 0) ? 1 : -1);
 655 +}
 656 +
 657 +static inline fp_t _integer_to_fp(fpbuf_t x)
 658 +{
 659 +	return _frac(x,1);
 660 +}
 661 +
 662 +static inline int _leq(fp_t a, fp_t b)
 663 +{
 664 +	return a.val <= b.val;
 665 +}
 666 +
 667 +static inline int _geq(fp_t a, fp_t b)
 668 +{
 669 +	return a.val >= b.val;
 670 +}
 671 +
 672 +static inline int _lt(fp_t a, fp_t b)
 673 +{
 674 +	return a.val < b.val;
 675 +}
 676 +
 677 +static inline int _gt(fp_t a, fp_t b)
 678 +{
 679 +	return a.val > b.val;
 680 +}
 681 +
 682 +static inline int _eq(fp_t a, fp_t b)
 683 +{
 684 +	return a.val == b.val;
 685 +}
 686 +
 687 +static inline fp_t _max(fp_t a, fp_t b)
 688 +{
 689 +	if (a.val < b.val)
 690 +		return b;
 691 +	else
 692 +		return a;
 693 +}
 694 +#endif
 695 diff --git a/include/litmus/gpu_affinity.h b/include/litmus/gpu_affinity.h
 696 new file mode 100644
 697 index 0000000..6b3fb8b
 698 --- /dev/null
 699 +++ b/include/litmus/gpu_affinity.h
 700 @@ -0,0 +1,49 @@
 701 +#ifndef LITMUS_GPU_AFFINITY_H
 702 +#define LITMUS_GPU_AFFINITY_H
 703 +
 704 +#include <litmus/rt_param.h>
 705 +#include <litmus/sched_plugin.h>
 706 +#include <litmus/litmus.h>
 707 +
 708 +void update_gpu_estimate(struct task_struct* t, lt_t observed);
 709 +gpu_migration_dist_t gpu_migration_distance(int a, int b);
 710 +
 711 +static inline void reset_gpu_tracker(struct task_struct* t)
 712 +{
 713 +	t->rt_param.accum_gpu_time = 0;
 714 +}
 715 +
 716 +static inline void start_gpu_tracker(struct task_struct* t)
 717 +{
 718 +	t->rt_param.gpu_time_stamp = litmus_clock();
 719 +}
 720 +
 721 +static inline void stop_gpu_tracker(struct task_struct* t)
 722 +{
 723 +	lt_t now = litmus_clock();
 724 +	t->rt_param.accum_gpu_time += (now - t->rt_param.gpu_time_stamp);
 725 +}
 726 +
 727 +static inline lt_t get_gpu_time(struct task_struct* t)
 728 +{
 729 +	return t->rt_param.accum_gpu_time;
 730 +}
 731 +
 732 +static inline lt_t get_gpu_estimate(struct task_struct* t, gpu_migration_dist_t dist)
 733 +{
 734 +	int i;
 735 +	fpbuf_t temp = _fp_to_integer(t->rt_param.gpu_migration_est[dist].est);
 736 +	lt_t val = (temp >= 0) ? temp : 0;  // never allow negative estimates...
 737 +
 738 +	WARN_ON(temp < 0);
 739 +
 740 +	// lower-bound a distant migration to be at least equal to the level
 741 +	// below it.
 742 +	for(i = dist-1; (val == 0) && (i >= MIG_LOCAL); --i) {
 743 +		val = _fp_to_integer(t->rt_param.gpu_migration_est[i].est);
 744 +	}
 745 +
 746 +	return ((val > 0) ? val : dist+1);
 747 +}
 748 +
 749 +#endif
 750 diff --git a/include/litmus/ikglp_lock.h b/include/litmus/ikglp_lock.h
 751 new file mode 100644
 752 index 0000000..af6f151
 753 --- /dev/null
 754 +++ b/include/litmus/ikglp_lock.h
 755 @@ -0,0 +1,160 @@
 756 +#ifndef LITMUS_IKGLP_H
 757 +#define LITMUS_IKGLP_H
 758 +
 759 +#include <litmus/litmus.h>
 760 +#include <litmus/binheap.h>
 761 +#include <litmus/locking.h>
 762 +
 763 +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
 764 +#include <litmus/kexclu_affinity.h>
 765 +
 766 +struct ikglp_affinity;
 767 +#endif
 768 +
 769 +typedef struct ikglp_heap_node
 770 +{
 771 +	struct task_struct *task;
 772 +	struct binheap_node node;
 773 +} ikglp_heap_node_t;
 774 +
 775 +struct fifo_queue;
 776 +struct ikglp_wait_state;
 777 +
 778 +typedef struct ikglp_donee_heap_node
 779 +{
 780 +	struct task_struct *task;
 781 +	struct fifo_queue *fq;
 782 +	struct ikglp_wait_state *donor_info;  // cross-linked with ikglp_wait_state_t of donor
 783 +
 784 +	struct binheap_node node;
 785 +} ikglp_donee_heap_node_t;
 786 +
 787 +// Maintains the state of a request as it goes through the IKGLP
 788 +typedef struct ikglp_wait_state {
 789 +	struct task_struct *task;  // pointer back to the requesting task
 790 +
 791 +	// Data for while waiting in FIFO Queue
 792 +	wait_queue_t fq_node;
 793 +	ikglp_heap_node_t global_heap_node;
 794 +	ikglp_donee_heap_node_t donee_heap_node;
 795 +
 796 +	// Data for while waiting in PQ
 797 +	ikglp_heap_node_t pq_node;
 798 +
 799 +	// Data for while waiting as a donor
 800 +	ikglp_donee_heap_node_t *donee_info;  // cross-linked with donee's ikglp_donee_heap_node_t
 801 +	struct nested_info prio_donation;
 802 +	struct binheap_node node;
 803 +} ikglp_wait_state_t;
 804 +
 805 +/* struct for semaphore with priority inheritance */
 806 +struct fifo_queue
 807 +{
 808 +	wait_queue_head_t wait;
 809 +	struct task_struct* owner;
 810 +
 811 +	// used for bookkeepping
 812 +	ikglp_heap_node_t global_heap_node;
 813 +	ikglp_donee_heap_node_t donee_heap_node;
 814 +
 815 +	struct task_struct* hp_waiter;
 816 +	int count; /* number of waiters + holder */
 817 +
 818 +	struct nested_info nest;
 819 +};
 820 +
 821 +struct ikglp_semaphore
 822 +{
 823 +	struct litmus_lock litmus_lock;
 824 +
 825 +	raw_spinlock_t	lock;
 826 +	raw_spinlock_t	real_lock;
 827 +
 828 +	int nr_replicas; // AKA k
 829 +	int m;
 830 +
 831 +	int max_fifo_len; // max len of a fifo queue
 832 +	int nr_in_fifos;
 833 +
 834 +	struct binheap_handle top_m;  // min heap, base prio
 835 +	int top_m_size;  // number of nodes in top_m
 836 +
 837 +	struct binheap_handle not_top_m; // max heap, base prio
 838 +
 839 +	struct binheap_handle donees;	// min-heap, base prio
 840 +	struct fifo_queue *shortest_fifo_queue; // pointer to shortest fifo queue
 841 +
 842 +	/* data structures for holding requests */
 843 +	struct fifo_queue *fifo_queues; // array nr_replicas in length
 844 +	struct binheap_handle priority_queue;	// max-heap, base prio
 845 +	struct binheap_handle donors;	// max-heap, base prio
 846 +
 847 +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
 848 +	struct ikglp_affinity *aff_obs;
 849 +#endif
 850 +};
 851 +
 852 +static inline struct ikglp_semaphore* ikglp_from_lock(struct litmus_lock* lock)
 853 +{
 854 +	return container_of(lock, struct ikglp_semaphore, litmus_lock);
 855 +}
 856 +
 857 +int ikglp_lock(struct litmus_lock* l);
 858 +int ikglp_unlock(struct litmus_lock* l);
 859 +int ikglp_close(struct litmus_lock* l);
 860 +void ikglp_free(struct litmus_lock* l);
 861 +struct litmus_lock* ikglp_new(int m, struct litmus_lock_ops*, void* __user arg);
 862 +
 863 +
 864 +
 865 +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
 866 +
 867 +struct ikglp_queue_info
 868 +{
 869 +	struct fifo_queue* q;
 870 +	lt_t estimated_len;
 871 +	int *nr_cur_users;
 872 +};
 873 +
 874 +struct ikglp_affinity_ops
 875 +{
 876 +	struct fifo_queue* (*advise_enqueue)(struct ikglp_affinity* aff, struct task_struct* t);	// select FIFO
 877 +	ikglp_wait_state_t* (*advise_steal)(struct ikglp_affinity* aff, struct fifo_queue* dst);	// select steal from FIFO
 878 +	ikglp_donee_heap_node_t* (*advise_donee_selection)(struct ikglp_affinity* aff, struct task_struct* t);	// select a donee
 879 +	ikglp_wait_state_t* (*advise_donor_to_fq)(struct ikglp_affinity* aff, struct fifo_queue* dst);	// select a donor to move to PQ
 880 +
 881 +	void (*notify_enqueue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t);	// fifo enqueue
 882 +	void (*notify_dequeue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t);	// fifo dequeue
 883 +	void (*notify_acquired)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t);	// replica acquired
 884 +	void (*notify_freed)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t);		// replica freed
 885 +	int (*replica_to_resource)(struct ikglp_affinity* aff, struct fifo_queue* fq);		// convert a replica # to a GPU (includes offsets and simult user folding)
 886 +};
 887 +
 888 +struct ikglp_affinity
 889 +{
 890 +	struct affinity_observer obs;
 891 +	struct ikglp_affinity_ops *ops;
 892 +	struct ikglp_queue_info *q_info;
 893 +	int *nr_cur_users_on_rsrc;
 894 +	int offset;
 895 +	int nr_simult;
 896 +	int nr_rsrc;
 897 +	int relax_max_fifo_len;
 898 +};
 899 +
 900 +static inline struct ikglp_affinity* ikglp_aff_obs_from_aff_obs(struct affinity_observer* aff_obs)
 901 +{
 902 +	return container_of(aff_obs, struct ikglp_affinity, obs);
 903 +}
 904 +
 905 +int ikglp_aff_obs_close(struct affinity_observer*);
 906 +void ikglp_aff_obs_free(struct affinity_observer*);
 907 +struct affinity_observer* ikglp_gpu_aff_obs_new(struct affinity_observer_ops*,
 908 +												void* __user arg);
 909 +struct affinity_observer* ikglp_simple_gpu_aff_obs_new(struct affinity_observer_ops*,
 910 +												void* __user arg);
 911 +#endif
 912 +
 913 +
 914 +
 915 +#endif
 916 diff --git a/include/litmus/kexclu_affinity.h b/include/litmus/kexclu_affinity.h
 917 new file mode 100644
 918 index 0000000..f6355de
 919 --- /dev/null
 920 +++ b/include/litmus/kexclu_affinity.h
 921 @@ -0,0 +1,35 @@
 922 +#ifndef LITMUS_AFF_OBS_H
 923 +#define LITMUS_AFF_OBS_H
 924 +
 925 +#include <litmus/locking.h>
 926 +
 927 +struct affinity_observer_ops;
 928 +
 929 +struct affinity_observer
 930 +{
 931 +	struct affinity_observer_ops* ops;
 932 +	int type;
 933 +	int ident;
 934 +
 935 +	struct litmus_lock* lock;  // the lock under observation
 936 +};
 937 +
 938 +typedef int (*aff_obs_open_t)(struct affinity_observer* aff_obs,
 939 +							  void* __user arg);
 940 +typedef int (*aff_obs_close_t)(struct affinity_observer* aff_obs);
 941 +typedef void (*aff_obs_free_t)(struct affinity_observer* aff_obs);
 942 +
 943 +struct affinity_observer_ops
 944 +{
 945 +	aff_obs_open_t open;
 946 +	aff_obs_close_t close;
 947 +	aff_obs_free_t deallocate;
 948 +};
 949 +
 950 +struct litmus_lock* get_lock_from_od(int od);
 951 +
 952 +void affinity_observer_new(struct affinity_observer* aff,
 953 +						   struct affinity_observer_ops* ops,
 954 +						   struct affinity_observer_args* args);
 955 +
 956 +#endif
 957 diff --git a/include/litmus/kfmlp_lock.h b/include/litmus/kfmlp_lock.h
 958 new file mode 100644
 959 index 0000000..5f0aae6
 960 --- /dev/null
 961 +++ b/include/litmus/kfmlp_lock.h
 962 @@ -0,0 +1,97 @@
 963 +#ifndef LITMUS_KFMLP_H
 964 +#define LITMUS_KFMLP_H
 965 +
 966 +#include <litmus/litmus.h>
 967 +#include <litmus/locking.h>
 968 +
 969 +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
 970 +#include <litmus/kexclu_affinity.h>
 971 +
 972 +struct kfmlp_affinity;
 973 +#endif
 974 +
 975 +/* struct for semaphore with priority inheritance */
 976 +struct kfmlp_queue
 977 +{
 978 +	wait_queue_head_t wait;
 979 +	struct task_struct* owner;
 980 +	struct task_struct* hp_waiter;
 981 +	int count; /* number of waiters + holder */
 982 +};
 983 +
 984 +struct kfmlp_semaphore
 985 +{
 986 +	struct litmus_lock litmus_lock;
 987 +
 988 +	spinlock_t	lock;
 989 +
 990 +	int num_resources; /* aka k */
 991 +
 992 +	struct kfmlp_queue *queues; /* array */
 993 +	struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */
 994 +
 995 +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
 996 +	struct kfmlp_affinity *aff_obs;
 997 +#endif
 998 +};
 999 +
1000 +static inline struct kfmlp_semaphore* kfmlp_from_lock(struct litmus_lock* lock)
1001 +{
1002 +	return container_of(lock, struct kfmlp_semaphore, litmus_lock);
1003 +}
1004 +
1005 +int kfmlp_lock(struct litmus_lock* l);
1006 +int kfmlp_unlock(struct litmus_lock* l);
1007 +int kfmlp_close(struct litmus_lock* l);
1008 +void kfmlp_free(struct litmus_lock* l);
1009 +struct litmus_lock* kfmlp_new(struct litmus_lock_ops*, void* __user arg);
1010 +
1011 +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
1012 +
1013 +struct kfmlp_queue_info
1014 +{
1015 +	struct kfmlp_queue* q;
1016 +	lt_t estimated_len;
1017 +	int *nr_cur_users;
1018 +};
1019 +
1020 +struct kfmlp_affinity_ops
1021 +{
1022 +	struct kfmlp_queue* (*advise_enqueue)(struct kfmlp_affinity* aff, struct task_struct* t);
1023 +	struct task_struct* (*advise_steal)(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from);
1024 +	void (*notify_enqueue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t);
1025 +	void (*notify_dequeue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t);
1026 +	void (*notify_acquired)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t);
1027 +	void (*notify_freed)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t);
1028 +	int (*replica_to_resource)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq);
1029 +};
1030 +
1031 +struct kfmlp_affinity
1032 +{
1033 +	struct affinity_observer obs;
1034 +	struct kfmlp_affinity_ops *ops;
1035 +	struct kfmlp_queue_info *q_info;
1036 +	int *nr_cur_users_on_rsrc;
1037 +	int offset;
1038 +	int nr_simult;
1039 +	int nr_rsrc;
1040 +};
1041 +
1042 +static inline struct kfmlp_affinity* kfmlp_aff_obs_from_aff_obs(struct affinity_observer* aff_obs)
1043 +{
1044 +	return container_of(aff_obs, struct kfmlp_affinity, obs);
1045 +}
1046 +
1047 +int kfmlp_aff_obs_close(struct affinity_observer*);
1048 +void kfmlp_aff_obs_free(struct affinity_observer*);
1049 +struct affinity_observer* kfmlp_gpu_aff_obs_new(struct affinity_observer_ops*,
1050 +											void* __user arg);
1051 +struct affinity_observer* kfmlp_simple_gpu_aff_obs_new(struct affinity_observer_ops*,
1052 +												void* __user arg);
1053 +
1054 +
1055 +#endif
1056 +
1057 +#endif
1058 +
1059 +
1060 diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
1061 index 0b071fd..71df378 100644
1062 --- a/include/litmus/litmus.h
1063 +++ b/include/litmus/litmus.h
1064 @@ -26,6 +26,7 @@ static inline int in_list(struct list_head* list)
1065  		);
1066  }
1067  
1068 +
1069  struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq);
1070  
1071  #define NO_CPU			0xffffffff
1072 @@ -53,12 +54,16 @@ void litmus_exit_task(struct task_struct *tsk);
1073  #define get_rt_phase(t)		(tsk_rt(t)->task_params.phase)
1074  #define get_partition(t) 	(tsk_rt(t)->task_params.cpu)
1075  #define get_deadline(t)		(tsk_rt(t)->job_params.deadline)
1076 +#define get_period(t)		(tsk_rt(t)->task_params.period)
1077  #define get_release(t)		(tsk_rt(t)->job_params.release)
1078  #define get_class(t)		(tsk_rt(t)->task_params.cls)
1079  
1080  #define is_priority_boosted(t)	(tsk_rt(t)->priority_boosted)
1081  #define get_boost_start(t)	(tsk_rt(t)->boost_start_time)
1082  
1083 +#define effective_priority(t) ((!(tsk_rt(t)->inh_task)) ? t : tsk_rt(t)->inh_task)
1084 +#define base_priority(t) (t)
1085 +
1086  inline static int budget_exhausted(struct task_struct* t)
1087  {
1088  	return get_exec_time(t) >= get_exec_cost(t);
1089 @@ -114,10 +119,12 @@ static inline lt_t litmus_clock(void)
1090  #define earlier_deadline(a, b) (lt_before(\
1091  	(a)->rt_param.job_params.deadline,\
1092  	(b)->rt_param.job_params.deadline))
1093 +#define shorter_period(a, b) (lt_before(\
1094 +	(a)->rt_param.task_params.period,\
1095 +	(b)->rt_param.task_params.period))
1096  #define earlier_release(a, b)  (lt_before(\
1097  	(a)->rt_param.job_params.release,\
1098  	(b)->rt_param.job_params.release))
1099 -
1100  void preempt_if_preemptable(struct task_struct* t, int on_cpu);
1101  
1102  #ifdef CONFIG_LITMUS_LOCKING
1103 diff --git a/include/litmus/litmus_softirq.h b/include/litmus/litmus_softirq.h
1104 new file mode 100644
1105 index 0000000..1eb5ea1
1106 --- /dev/null
1107 +++ b/include/litmus/litmus_softirq.h
1108 @@ -0,0 +1,199 @@
1109 +#ifndef __LITMUS_SOFTIRQ_H
1110 +#define __LITMUS_SOFTIRQ_H
1111 +
1112 +#include <linux/interrupt.h>
1113 +#include <linux/workqueue.h>
1114 +
1115 +/*
1116 +   Threaded tasklet handling for Litmus.  Tasklets
1117 +   are scheduled with the priority of the tasklet's
1118 +   owner---that is, the RT task on behalf the tasklet
1119 +   runs.
1120 +
1121 +   Tasklets are current scheduled in FIFO order with
1122 +   NO priority inheritance for "blocked" tasklets.
1123 +
1124 +   klitirqd assumes the priority of the owner of the
1125 +   tasklet when the tasklet is next to execute.
1126 +
1127 +   Currently, hi-tasklets are scheduled before
1128 +   low-tasklets, regardless of priority of low-tasklets.
1129 +   And likewise, low-tasklets are scheduled before work
1130 +   queue objects.  This priority inversion probably needs
1131 +   to be fixed, though it is not an issue if our work with
1132 +   GPUs as GPUs are owned (and associated klitirqds) for
1133 +   exclusive time periods, thus no inversions can
1134 +   occur.
1135 + */
1136 +
1137 +
1138 +
1139 +#define NR_LITMUS_SOFTIRQD CONFIG_NR_LITMUS_SOFTIRQD
1140 +
1141 +/* Spawns NR_LITMUS_SOFTIRQD klitirqd daemons.
1142 +   Actual launch of threads is deffered to kworker's
1143 +   workqueue, so daemons will likely not be immediately
1144 +   running when this function returns, though the required
1145 +   data will be initialized.
1146 +
1147 +   @affinity_set: an array expressing the processor affinity
1148 +    for each of the NR_LITMUS_SOFTIRQD daemons.  May be set
1149 +    to NULL for global scheduling.
1150 +
1151 +	- Examples -
1152 +	8-CPU system with two CPU clusters:
1153 +		affinity[] = {0, 0, 0, 0, 3, 3, 3, 3}
1154 +		NOTE: Daemons not actually bound to specified CPU, but rather
1155 +		cluster in which the CPU resides.
1156 +
1157 +	8-CPU system, partitioned:
1158 +		affinity[] = {0, 1, 2, 3, 4, 5, 6, 7}
1159 +
1160 +	FIXME: change array to a CPU topology or array of cpumasks
1161 +
1162 + */
1163 +void spawn_klitirqd(int* affinity);
1164 +
1165 +
1166 +/* Raises a flag to tell klitirqds to terminate.
1167 +   Termination is async, so some threads may be running
1168 +   after function return. */
1169 +void kill_klitirqd(void);
1170 +
1171 +
1172 +/* Returns 1 if all NR_LITMUS_SOFTIRQD klitirqs are ready
1173 +   to handle tasklets. 0, otherwise.*/
1174 +int klitirqd_is_ready(void);
1175 +
1176 +/* Returns 1 if no NR_LITMUS_SOFTIRQD klitirqs are ready
1177 +   to handle tasklets. 0, otherwise.*/
1178 +int klitirqd_is_dead(void);
1179 +
1180 +/* Flushes all pending work out to the OS for regular
1181 + * tasklet/work processing of the specified 'owner'
1182 + *
1183 + * PRECOND: klitirqd_thread must have a clear entry
1184 + * in the GPU registry, otherwise this call will become
1185 + * a no-op as work will loop back to the klitirqd_thread.
1186 + *
1187 + * Pass NULL for owner to flush ALL pending items.
1188 + */
1189 +void flush_pending(struct task_struct* klitirqd_thread,
1190 +				   struct task_struct* owner);
1191 +
1192 +struct task_struct* get_klitirqd(unsigned int k_id);
1193 +
1194 +
1195 +extern int __litmus_tasklet_schedule(
1196 +        struct tasklet_struct *t,
1197 +        unsigned int k_id);
1198 +
1199 +/* schedule a tasklet on klitirqd #k_id */
1200 +static inline int litmus_tasklet_schedule(
1201 +    struct tasklet_struct *t,
1202 +    unsigned int k_id)
1203 +{
1204 +	int ret = 0;
1205 +	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
1206 +		ret = __litmus_tasklet_schedule(t, k_id);
1207 +	return(ret);
1208 +}
1209 +
1210 +/* for use by __tasklet_schedule() */
1211 +static inline int _litmus_tasklet_schedule(
1212 +    struct tasklet_struct *t,
1213 +    unsigned int k_id)
1214 +{
1215 +    return(__litmus_tasklet_schedule(t, k_id));
1216 +}
1217 +
1218 +
1219 +
1220 +
1221 +extern int __litmus_tasklet_hi_schedule(struct tasklet_struct *t,
1222 +                                         unsigned int k_id);
1223 +
1224 +/* schedule a hi tasklet on klitirqd #k_id */
1225 +static inline int litmus_tasklet_hi_schedule(struct tasklet_struct *t,
1226 +                                              unsigned int k_id)
1227 +{
1228 +	int ret = 0;
1229 +	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
1230 +		ret = __litmus_tasklet_hi_schedule(t, k_id);
1231 +	return(ret);
1232 +}
1233 +
1234 +/* for use by __tasklet_hi_schedule() */
1235 +static inline int _litmus_tasklet_hi_schedule(struct tasklet_struct *t,
1236 +                                               unsigned int k_id)
1237 +{
1238 +    return(__litmus_tasklet_hi_schedule(t, k_id));
1239 +}
1240 +
1241 +
1242 +
1243 +
1244 +
1245 +extern int __litmus_tasklet_hi_schedule_first(
1246 +    struct tasklet_struct *t,
1247 +    unsigned int k_id);
1248 +
1249 +/* schedule a hi tasklet on klitirqd #k_id on next go-around */
1250 +/* PRECONDITION: Interrupts must be disabled. */
1251 +static inline int litmus_tasklet_hi_schedule_first(
1252 +    struct tasklet_struct *t,
1253 +    unsigned int k_id)
1254 +{
1255 +	int ret = 0;
1256 +	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
1257 +		ret = __litmus_tasklet_hi_schedule_first(t, k_id);
1258 +	return(ret);
1259 +}
1260 +
1261 +/* for use by __tasklet_hi_schedule_first() */
1262 +static inline int _litmus_tasklet_hi_schedule_first(
1263 +    struct tasklet_struct *t,
1264 +    unsigned int k_id)
1265 +{
1266 +    return(__litmus_tasklet_hi_schedule_first(t, k_id));
1267 +}
1268 +
1269 +
1270 +
1271 +//////////////
1272 +
1273 +extern int __litmus_schedule_work(
1274 +	struct work_struct* w,
1275 +	unsigned int k_id);
1276 +
1277 +static inline int litmus_schedule_work(
1278 +	struct work_struct* w,
1279 +	unsigned int k_id)
1280 +{
1281 +	return(__litmus_schedule_work(w, k_id));
1282 +}
1283 +
1284 +
1285 +
1286 +///////////// mutex operations for client threads.
1287 +
1288 +void down_and_set_stat(struct task_struct* t,
1289 +					 enum klitirqd_sem_status to_set,
1290 +					 struct mutex* sem);
1291 +
1292 +void __down_and_reset_and_set_stat(struct task_struct* t,
1293 +				enum klitirqd_sem_status to_reset,
1294 +				enum klitirqd_sem_status to_set,
1295 +				struct mutex* sem);
1296 +
1297 +void up_and_set_stat(struct task_struct* t,
1298 +					enum klitirqd_sem_status to_set,
1299 +					struct mutex* sem);
1300 +
1301 +
1302 +
1303 +void release_klitirqd_lock(struct task_struct* t);
1304 +
1305 +int reacquire_klitirqd_lock(struct task_struct* t);
1306 +
1307 +#endif
1308 diff --git a/include/litmus/locking.h b/include/litmus/locking.h
1309 index 4d7b870..36647fe 100644
1310 --- a/include/litmus/locking.h
1311 +++ b/include/litmus/locking.h
1312 @@ -1,28 +1,160 @@
1313  #ifndef LITMUS_LOCKING_H
1314  #define LITMUS_LOCKING_H
1315  
1316 +#include <linux/list.h>
1317 +
1318  struct litmus_lock_ops;
1319  
1320 +#ifdef CONFIG_LITMUS_NESTED_LOCKING
1321 +struct nested_info
1322 +{
1323 +	struct litmus_lock *lock;
1324 +	struct task_struct *hp_waiter_eff_prio;
1325 +	struct task_struct **hp_waiter_ptr;
1326 +    struct binheap_node hp_binheap_node;
1327 +};
1328 +
1329 +static inline struct task_struct* top_priority(struct binheap_handle* handle) {
1330 +	if(!binheap_empty(handle)) {
1331 +		return (struct task_struct*)(binheap_top_entry(handle, struct nested_info, hp_binheap_node)->hp_waiter_eff_prio);
1332 +	}
1333 +	return NULL;
1334 +}
1335 +
1336 +void print_hp_waiters(struct binheap_node* n, int depth);
1337 +#endif
1338 +
1339 +
1340  /* Generic base struct for LITMUS^RT userspace semaphores.
1341   * This structure should be embedded in protocol-specific semaphores.
1342   */
1343  struct litmus_lock {
1344  	struct litmus_lock_ops *ops;
1345  	int type;
1346 +
1347 +	int ident;
1348 +
1349 +#ifdef CONFIG_LITMUS_NESTED_LOCKING
1350 +	struct nested_info nest;
1351 +//#ifdef CONFIG_DEBUG_SPINLOCK
1352 +	char cheat_lockdep[2];
1353 +	struct lock_class_key key;
1354 +//#endif
1355 +#endif
1356  };
1357  
1358 +#ifdef CONFIG_LITMUS_DGL_SUPPORT
1359 +
1360 +#define MAX_DGL_SIZE CONFIG_LITMUS_MAX_DGL_SIZE
1361 +
1362 +typedef struct dgl_wait_state {
1363 +	struct task_struct *task;	/* task waiting on DGL */
1364 +	struct litmus_lock *locks[MAX_DGL_SIZE];	/* requested locks in DGL */
1365 +	int size;			/* size of the DGL */
1366 +	int nr_remaining;	/* nr locks remainging before DGL is complete */
1367 +	int last_primary;	/* index lock in locks[] that has active priority */
1368 +	wait_queue_t wq_nodes[MAX_DGL_SIZE];
1369 +} dgl_wait_state_t;
1370 +
1371 +void wake_or_wait_on_next_lock(dgl_wait_state_t *dgl_wait);
1372 +void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lock*/);
1373 +
1374 +void init_dgl_waitqueue_entry(wait_queue_t *wq_node, dgl_wait_state_t* dgl_wait);
1375 +int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key);
1376 +void __waitqueue_dgl_remove_first(wait_queue_head_t *wq, dgl_wait_state_t** dgl_wait, struct task_struct **task);
1377 +#endif
1378 +
1379 +typedef int (*lock_op_t)(struct litmus_lock *l);
1380 +typedef lock_op_t lock_close_t;
1381 +typedef lock_op_t lock_lock_t;
1382 +typedef lock_op_t lock_unlock_t;
1383 +
1384 +typedef int (*lock_open_t)(struct litmus_lock *l, void* __user arg);
1385 +typedef void (*lock_free_t)(struct litmus_lock *l);
1386 +
1387  struct litmus_lock_ops {
1388  	/* Current task tries to obtain / drop a reference to a lock.
1389  	 * Optional methods, allowed by default. */
1390 -	int (*open)(struct litmus_lock*, void* __user);
1391 -	int (*close)(struct litmus_lock*);
1392 +	lock_open_t open;
1393 +	lock_close_t close;
1394  
1395  	/* Current tries to lock/unlock this lock (mandatory methods). */
1396 -	int (*lock)(struct litmus_lock*);
1397 -	int (*unlock)(struct litmus_lock*);
1398 +	lock_lock_t lock;
1399 +	lock_unlock_t unlock;
1400  
1401  	/* The lock is no longer being referenced (mandatory method). */
1402 -	void (*deallocate)(struct litmus_lock*);
1403 +	lock_free_t deallocate;
1404 +
1405 +#ifdef CONFIG_LITMUS_NESTED_LOCKING
1406 +	void (*propagate_increase_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags);
1407 +	void (*propagate_decrease_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags);
1408 +#endif
1409 +
1410 +#ifdef CONFIG_LITMUS_DGL_SUPPORT
1411 +	raw_spinlock_t* (*get_dgl_spin_lock)(struct litmus_lock *l);
1412 +	int (*dgl_lock)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node);
1413 +	int (*is_owner)(struct litmus_lock *l, struct task_struct *t);
1414 +	void (*enable_priority)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait);
1415 +#endif
1416  };
1417  
1418 +
1419 +/*
1420 + Nested inheritance can be achieved with fine-grain locking when there is
1421 + no need for DGL support, presuming locks are acquired in a partial order
1422 + (no cycles!).  However, DGLs allow locks to be acquired in any order.  This
1423 + makes nested inheritance very difficult (we don't yet know a solution) to
1424 + realize with fine-grain locks, so we use a big lock instead.
1425 +
1426 + Code contains both fine-grain and coarse-grain methods together, side-by-side.
1427 + Each lock operation *IS NOT* surrounded by ifdef/endif to help make code more
1428 + readable.  However, this leads to the odd situation where both code paths
1429 + appear together in code as if they were both active together.
1430 +
1431 + THIS IS NOT REALLY THE CASE!  ONLY ONE CODE PATH IS ACTUALLY ACTIVE!
1432 +
1433 + Example:
1434 +	lock_global_irqsave(coarseLock, flags);
1435 +	lock_fine_irqsave(fineLock, flags);
1436 +
1437 + Reality (coarse):
1438 +	lock_global_irqsave(coarseLock, flags);
1439 +	//lock_fine_irqsave(fineLock, flags);
1440 +
1441 + Reality (fine):
1442 +	//lock_global_irqsave(coarseLock, flags);
1443 +	lock_fine_irqsave(fineLock, flags);
1444 +
1445 + Be careful when you read code involving nested inheritance.
1446 + */
1447 +#if defined(CONFIG_LITMUS_DGL_SUPPORT)
1448 +/* DGL requires a big lock to implement nested inheritance */
1449 +#define lock_global_irqsave(lock, flags)		raw_spin_lock_irqsave((lock), (flags))
1450 +#define lock_global(lock)						raw_spin_lock((lock))
1451 +#define unlock_global_irqrestore(lock, flags)	raw_spin_unlock_irqrestore((lock), (flags))
1452 +#define unlock_global(lock)						raw_spin_unlock((lock))
1453 +
1454 +/* fine-grain locking are no-ops with DGL support */
1455 +#define lock_fine_irqsave(lock, flags)
1456 +#define lock_fine(lock)
1457 +#define unlock_fine_irqrestore(lock, flags)
1458 +#define unlock_fine(lock)
1459 +
1460 +#elif defined(CONFIG_LITMUS_NESTED_LOCKING)
1461 +/* Use fine-grain locking when DGLs are disabled. */
1462 +/* global locking are no-ops without DGL support */
1463 +#define lock_global_irqsave(lock, flags)
1464 +#define lock_global(lock)
1465 +#define unlock_global_irqrestore(lock, flags)
1466 +#define unlock_global(lock)
1467 +
1468 +#define lock_fine_irqsave(lock, flags)			raw_spin_lock_irqsave((lock), (flags))
1469 +#define lock_fine(lock)							raw_spin_lock((lock))
1470 +#define unlock_fine_irqrestore(lock, flags)		raw_spin_unlock_irqrestore((lock), (flags))
1471 +#define unlock_fine(lock)						raw_spin_unlock((lock))
1472 +
1473  #endif
1474 +
1475 +
1476 +#endif
1477 +
1478 diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h
1479 new file mode 100644
1480 index 0000000..97c9577
1481 --- /dev/null
1482 +++ b/include/litmus/nvidia_info.h
1483 @@ -0,0 +1,46 @@
1484 +#ifndef __LITMUS_NVIDIA_H
1485 +#define __LITMUS_NVIDIA_H
1486 +
1487 +#include <linux/interrupt.h>
1488 +
1489 +
1490 +#include <litmus/litmus_softirq.h>
1491 +
1492 +
1493 +//#define NV_DEVICE_NUM NR_LITMUS_SOFTIRQD
1494 +#define NV_DEVICE_NUM CONFIG_NV_DEVICE_NUM
1495 +#define NV_MAX_SIMULT_USERS CONFIG_NV_MAX_SIMULT_USERS
1496 +
1497 +int init_nvidia_info(void);
1498 +void shutdown_nvidia_info(void);
1499 +
1500 +int is_nvidia_func(void* func_addr);
1501 +
1502 +void dump_nvidia_info(const struct tasklet_struct *t);
1503 +
1504 +
1505 +// Returns the Nvidia device # associated with provided tasklet and work_struct.
1506 +u32 get_tasklet_nv_device_num(const struct tasklet_struct *t);
1507 +u32 get_work_nv_device_num(const struct work_struct *t);
1508 +
1509 +
1510 +int init_nv_device_reg(void);
1511 +//int get_nv_device_id(struct task_struct* owner);
1512 +
1513 +
1514 +int reg_nv_device(int reg_device_id, int register_device, struct task_struct *t);
1515 +
1516 +struct task_struct* get_nv_max_device_owner(u32 target_device_id);
1517 +//int is_nv_device_owner(u32 target_device_id);
1518 +
1519 +void lock_nv_registry(u32 reg_device_id, unsigned long* flags);
1520 +void unlock_nv_registry(u32 reg_device_id, unsigned long* flags);
1521 +
1522 +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1523 +void pai_check_priority_increase(struct task_struct *t, int reg_device_id);
1524 +void pai_check_priority_decrease(struct task_struct *t, int reg_device_id);
1525 +#endif
1526 +
1527 +//void increment_nv_int_count(u32 device);
1528 +
1529 +#endif
1530 diff --git a/include/litmus/preempt.h b/include/litmus/preempt.h
1531 index 380b886..8f3a9ca 100644
1532 --- a/include/litmus/preempt.h
1533 +++ b/include/litmus/preempt.h
1534 @@ -26,12 +26,12 @@ const char* sched_state_name(int s);
1535  				    (x), #x, __FUNCTION__);		\
1536  	} while (0);
1537  
1538 +//#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) /* ignore */
1539  #define TRACE_SCHED_STATE_CHANGE(x, y, cpu)				\
1540  	TRACE_STATE("[P%d] 0x%x (%s) -> 0x%x (%s)\n",			\
1541  		    cpu,  (x), sched_state_name(x),			\
1542  		    (y), sched_state_name(y))
1543  
1544 -
1545  typedef enum scheduling_state {
1546  	TASK_SCHEDULED    = (1 << 0),  /* The currently scheduled task is the one that
1547  					* should be scheduled, and the processor does not
1548 diff --git a/include/litmus/rsm_lock.h b/include/litmus/rsm_lock.h
1549 new file mode 100644
1550 index 0000000..a151896
1551 --- /dev/null
1552 +++ b/include/litmus/rsm_lock.h
1553 @@ -0,0 +1,54 @@
1554 +#ifndef LITMUS_RSM_H
1555 +#define LITMUS_RSM_H
1556 +
1557 +#include <litmus/litmus.h>
1558 +#include <litmus/binheap.h>
1559 +#include <litmus/locking.h>
1560 +
1561 +/* struct for semaphore with priority inheritance */
1562 +struct rsm_mutex {
1563 +	struct litmus_lock litmus_lock;
1564 +
1565 +	/* current resource holder */
1566 +	struct task_struct *owner;
1567 +
1568 +	/* highest-priority waiter */
1569 +	struct task_struct *hp_waiter;
1570 +
1571 +	/* FIFO queue of waiting tasks -- for now.  time stamp in the future. */
1572 +	wait_queue_head_t	wait;
1573 +
1574 +	/* we do some nesting within spinlocks, so we can't use the normal
1575 +	 sleeplocks found in wait_queue_head_t. */
1576 +	raw_spinlock_t		lock;
1577 +};
1578 +
1579 +static inline struct rsm_mutex* rsm_mutex_from_lock(struct litmus_lock* lock)
1580 +{
1581 +	return container_of(lock, struct rsm_mutex, litmus_lock);
1582 +}
1583 +
1584 +#ifdef CONFIG_LITMUS_DGL_SUPPORT
1585 +int rsm_mutex_is_owner(struct litmus_lock *l, struct task_struct *t);
1586 +int rsm_mutex_dgl_lock(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node);
1587 +void rsm_mutex_enable_priority(struct litmus_lock *l, dgl_wait_state_t* dgl_wait);
1588 +#endif
1589 +
1590 +void rsm_mutex_propagate_increase_inheritance(struct litmus_lock* l,
1591 +											  struct task_struct* t,
1592 +											  raw_spinlock_t* to_unlock,
1593 +											  unsigned long irqflags);
1594 +
1595 +void rsm_mutex_propagate_decrease_inheritance(struct litmus_lock* l,
1596 +											  struct task_struct* t,
1597 +											  raw_spinlock_t* to_unlock,
1598 +											  unsigned long irqflags);
1599 +
1600 +int rsm_mutex_lock(struct litmus_lock* l);
1601 +int rsm_mutex_unlock(struct litmus_lock* l);
1602 +int rsm_mutex_close(struct litmus_lock* l);
1603 +void rsm_mutex_free(struct litmus_lock* l);
1604 +struct litmus_lock* rsm_mutex_new(struct litmus_lock_ops*);
1605 +
1606 +
1607 +#endif
1608 \ No newline at end of file
1609 diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
1610 index d6d7991..0198884 100644
1611 --- a/include/litmus/rt_param.h
1612 +++ b/include/litmus/rt_param.h
1613 @@ -5,6 +5,8 @@
1614  #ifndef _LINUX_RT_PARAM_H_
1615  #define _LINUX_RT_PARAM_H_
1616  
1617 +#include <litmus/fpmath.h>
1618 +
1619  /* Litmus time type. */
1620  typedef unsigned long long lt_t;
1621  
1622 @@ -24,6 +26,7 @@ static inline int lt_after_eq(lt_t a, lt_t b)
1623  typedef enum {
1624  	RT_CLASS_HARD,
1625  	RT_CLASS_SOFT,
1626 +	RT_CLASS_SOFT_W_SLIP,
1627  	RT_CLASS_BEST_EFFORT
1628  } task_class_t;
1629  
1630 @@ -52,6 +55,19 @@ union np_flag {
1631  	} np;
1632  };
1633  
1634 +struct affinity_observer_args
1635 +{
1636 +	int lock_od;
1637 +};
1638 +
1639 +struct gpu_affinity_observer_args
1640 +{
1641 +	struct affinity_observer_args obs;
1642 +	int replica_to_gpu_offset;
1643 +	int nr_simult_users;
1644 +	int relaxed_rules;
1645 +};
1646 +
1647  /* The definition of the data that is shared between the kernel and real-time
1648   * tasks via a shared page (see litmus/ctrldev.c).
1649   *
1650 @@ -75,6 +91,9 @@ struct control_page {
1651  /* don't export internal data structures to user space (liblitmus) */
1652  #ifdef __KERNEL__
1653  
1654 +#include <litmus/binheap.h>
1655 +#include <linux/semaphore.h>
1656 +
1657  struct _rt_domain;
1658  struct bheap_node;
1659  struct release_heap;
1660 @@ -100,6 +119,31 @@ struct rt_job {
1661  
1662  struct pfair_param;
1663  
1664 +enum klitirqd_sem_status
1665 +{
1666 +	NEED_TO_REACQUIRE,
1667 +	REACQUIRING,
1668 +	NOT_HELD,
1669 +	HELD
1670 +};
1671 +
1672 +typedef enum gpu_migration_dist
1673 +{
1674 +	// TODO: Make this variable against NR_NVIDIA_GPUS
1675 +	MIG_LOCAL = 0,
1676 +	MIG_NEAR = 1,
1677 +	MIG_MED = 2,
1678 +	MIG_FAR = 3,	// 8 GPUs in a binary tree hierarchy
1679 +	MIG_NONE = 4,
1680 +
1681 +	MIG_LAST = MIG_NONE
1682 +} gpu_migration_dist_t;
1683 +
1684 +typedef struct feedback_est{
1685 +	fp_t est;
1686 +	fp_t accum_err;
1687 +} feedback_est_t;
1688 +
1689  /*	RT task parameters for scheduling extensions
1690   *	These parameters are inherited during clone and therefore must
1691   *	be explicitly set up before the task set is launched.
1692 @@ -114,6 +158,52 @@ struct rt_param {
1693  	/* is the task present? (true if it can be scheduled) */
1694  	unsigned int		present:1;
1695  
1696 +#ifdef CONFIG_LITMUS_SOFTIRQD
1697 +    /* proxy threads have minimum priority by default */
1698 +    unsigned int        is_proxy_thread:1;
1699 +
1700 +	/* pointer to klitirqd currently working on this
1701 +	   task_struct's behalf.  only set by the task pointed
1702 +	   to by klitirqd.
1703 +
1704 +	   ptr only valid if is_proxy_thread == 0
1705 +	 */
1706 +	struct task_struct* cur_klitirqd;
1707 +
1708 +	/* Used to implement mutual execution exclusion between
1709 +	 * job and klitirqd execution.  Job must always hold
1710 +	 * it's klitirqd_sem to execute.  klitirqd instance
1711 +	 * must hold the semaphore before executing on behalf
1712 +	 * of a job.
1713 +	 */
1714 +	struct mutex				klitirqd_sem;
1715 +
1716 +	/* status of held klitirqd_sem, even if the held klitirqd_sem is from
1717 +	   another task (only proxy threads do this though).
1718 +	 */
1719 +	atomic_t					klitirqd_sem_stat;
1720 +#endif
1721 +
1722 +#ifdef CONFIG_LITMUS_NVIDIA
1723 +	/* number of top-half interrupts handled on behalf of current job */
1724 +	atomic_t					nv_int_count;
1725 +	long unsigned int			held_gpus;  // bitmap of held GPUs.
1726 +
1727 +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
1728 +	fp_t	gpu_fb_param_a[MIG_LAST+1];
1729 +	fp_t	gpu_fb_param_b[MIG_LAST+1];
1730 +
1731 +	gpu_migration_dist_t	gpu_migration;
1732 +	int				last_gpu;
1733 +	feedback_est_t	gpu_migration_est[MIG_LAST+1]; // local, near, med, far
1734 +
1735 +	lt_t accum_gpu_time;
1736 +	lt_t gpu_time_stamp;
1737 +
1738 +	unsigned int suspend_gpu_tracker_on_block:1;
1739 +#endif
1740 +#endif
1741 +
1742  #ifdef CONFIG_LITMUS_LOCKING
1743  	/* Is the task being priority-boosted by a locking protocol? */
1744  	unsigned int		priority_boosted:1;
1745 @@ -133,7 +223,15 @@ struct rt_param {
1746  	 * could point to self if PI does not result in
1747  	 * an increased task priority.
1748  	 */
1749 -	 struct task_struct*	inh_task;
1750 +	struct task_struct*	inh_task;
1751 +
1752 +#ifdef CONFIG_LITMUS_NESTED_LOCKING
1753 +	raw_spinlock_t			hp_blocked_tasks_lock;
1754 +	struct binheap_handle	hp_blocked_tasks;
1755 +
1756 +	/* pointer to lock upon which is currently blocked */
1757 +	struct litmus_lock* blocked_lock;
1758 +#endif
1759  
1760  #ifdef CONFIG_NP_SECTION
1761  	/* For the FMLP under PSN-EDF, it is required to make the task
1762 diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h
1763 index 6e7cabd..24a6858 100644
1764 --- a/include/litmus/sched_plugin.h
1765 +++ b/include/litmus/sched_plugin.h
1766 @@ -11,6 +11,12 @@
1767  #include <litmus/locking.h>
1768  #endif
1769  
1770 +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
1771 +#include <litmus/kexclu_affinity.h>
1772 +#endif
1773 +
1774 +#include <linux/interrupt.h>
1775 +
1776  /************************ setup/tear down ********************/
1777  
1778  typedef long (*activate_plugin_t) (void);
1779 @@ -29,7 +35,6 @@ typedef struct task_struct* (*schedule_t)(struct task_struct * prev);
1780   */
1781  typedef void (*finish_switch_t)(struct task_struct *prev);
1782  
1783 -
1784  /********************* task state changes ********************/
1785  
1786  /* Called to setup a new real-time task.
1787 @@ -58,6 +63,47 @@ typedef void (*task_exit_t)    (struct task_struct *);
1788  typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type,
1789  				 void* __user config);
1790  
1791 +struct affinity_observer;
1792 +typedef long (*allocate_affinity_observer_t) (
1793 +								struct affinity_observer **aff_obs, int type,
1794 +								void* __user config);
1795 +
1796 +typedef void (*increase_prio_t)(struct task_struct* t, struct task_struct* prio_inh);
1797 +typedef void (*decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh);
1798 +typedef void (*nested_increase_prio_t)(struct task_struct* t, struct task_struct* prio_inh,
1799 +									  raw_spinlock_t *to_unlock, unsigned long irqflags);
1800 +typedef void (*nested_decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh,
1801 +									  raw_spinlock_t *to_unlock, unsigned long irqflags);
1802 +
1803 +typedef void (*increase_prio_klitirq_t)(struct task_struct* klitirqd,
1804 +                                        struct task_struct* old_owner,
1805 +                                        struct task_struct* new_owner);
1806 +typedef void (*decrease_prio_klitirqd_t)(struct task_struct* klitirqd,
1807 +                                         struct task_struct* old_owner);
1808 +
1809 +
1810 +typedef int (*enqueue_pai_tasklet_t)(struct tasklet_struct* tasklet);
1811 +typedef void (*change_prio_pai_tasklet_t)(struct task_struct *old_prio,
1812 +										  struct task_struct *new_prio);
1813 +typedef void (*run_tasklets_t)(struct task_struct* next);
1814 +
1815 +typedef raw_spinlock_t* (*get_dgl_spinlock_t) (struct task_struct *t);
1816 +
1817 +
1818 +typedef int (*higher_prio_t)(struct task_struct* a, struct task_struct* b);
1819 +
1820 +#ifdef CONFIG_LITMUS_NESTED_LOCKING
1821 +
1822 +typedef enum
1823 +{
1824 +	BASE,
1825 +	EFFECTIVE
1826 +} comparison_mode_t;
1827 +
1828 +typedef int (*__higher_prio_t)(struct task_struct* a, comparison_mode_t a_mod,
1829 +							  struct task_struct* b, comparison_mode_t b_mod);
1830 +#endif
1831 +
1832  
1833  /********************* sys call backends  ********************/
1834  /* This function causes the caller to sleep until the next release */
1835 @@ -88,14 +134,40 @@ struct sched_plugin {
1836  	/*	task state changes 	*/
1837  	admit_task_t		admit_task;
1838  
1839 -        task_new_t 		task_new;
1840 +    task_new_t			task_new;
1841  	task_wake_up_t		task_wake_up;
1842  	task_block_t		task_block;
1843  	task_exit_t 		task_exit;
1844  
1845 +	higher_prio_t		compare;
1846 +
1847  #ifdef CONFIG_LITMUS_LOCKING
1848  	/*	locking protocols	*/
1849  	allocate_lock_t		allocate_lock;
1850 +	increase_prio_t		increase_prio;
1851 +	decrease_prio_t		decrease_prio;
1852 +#endif
1853 +#ifdef CONFIG_LITMUS_NESTED_LOCKING
1854 +	nested_increase_prio_t nested_increase_prio;
1855 +	nested_decrease_prio_t nested_decrease_prio;
1856 +	__higher_prio_t		__compare;
1857 +#endif
1858 +#ifdef CONFIG_LITMUS_DGL_SUPPORT
1859 +	get_dgl_spinlock_t	get_dgl_spinlock;
1860 +#endif
1861 +
1862 +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
1863 +	allocate_affinity_observer_t allocate_aff_obs;
1864 +#endif
1865 +
1866 +#ifdef CONFIG_LITMUS_SOFTIRQD
1867 +    increase_prio_klitirq_t		increase_prio_klitirqd;
1868 +    decrease_prio_klitirqd_t	decrease_prio_klitirqd;
1869 +#endif
1870 +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1871 +	enqueue_pai_tasklet_t		enqueue_pai_tasklet;
1872 +	change_prio_pai_tasklet_t	change_prio_pai_tasklet;
1873 +	run_tasklets_t				run_tasklets;
1874  #endif
1875  } __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
1876  
1877 diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h
1878 index 7ca34cb..b1b71f6 100644
1879 --- a/include/litmus/sched_trace.h
1880 +++ b/include/litmus/sched_trace.h
1881 @@ -10,13 +10,14 @@ struct st_trace_header {
1882  	u8	type;		/* Of what type is this record?  */
1883  	u8	cpu;		/* On which CPU was it recorded? */
1884  	u16	pid;		/* PID of the task.              */
1885 -	u32	job;		/* The job sequence number.      */
1886 -};
1887 +	u32 job:24;		/* The job sequence number.      */
1888 +	u8  extra;
1889 +} __attribute__((packed));
1890  
1891  #define ST_NAME_LEN 16
1892  struct st_name_data {
1893  	char	cmd[ST_NAME_LEN];/* The name of the executable of this process. */
1894 -};
1895 +} __attribute__((packed));
1896  
1897  struct st_param_data {		/* regular params */
1898  	u32	wcet;
1899 @@ -25,30 +26,29 @@ struct st_param_data {		/* regular params */
1900  	u8	partition;
1901  	u8	class;
1902  	u8	__unused[2];
1903 -};
1904 +} __attribute__((packed));
1905  
1906  struct st_release_data {	/* A job is was/is going to be released. */
1907  	u64	release;	/* What's the release time?              */
1908  	u64	deadline;	/* By when must it finish?		 */
1909 -};
1910 +} __attribute__((packed));
1911  
1912  struct st_assigned_data {	/* A job was asigned to a CPU. 		 */
1913  	u64	when;
1914  	u8	target;		/* Where should it execute?	         */
1915  	u8	__unused[7];
1916 -};
1917 +} __attribute__((packed));
1918  
1919  struct st_switch_to_data {	/* A process was switched to on a given CPU.   */
1920  	u64	when;		/* When did this occur?                        */
1921  	u32	exec_time;	/* Time the current job has executed.          */
1922  	u8	__unused[4];
1923 -
1924 -};
1925 +} __attribute__((packed));
1926  
1927  struct st_switch_away_data {	/* A process was switched away from on a given CPU. */
1928  	u64	when;
1929  	u64	exec_time;
1930 -};
1931 +} __attribute__((packed));
1932  
1933  struct st_completion_data {	/* A job completed. */
1934  	u64	when;
1935 @@ -56,35 +56,108 @@ struct st_completion_data {	/* A job completed. */
1936  				 * next task automatically; set to 0 otherwise.
1937  				 */
1938  	u8	__uflags:7;
1939 -	u8	__unused[7];
1940 -};
1941 +	u16 nv_int_count;
1942 +	u8	__unused[5];
1943 +} __attribute__((packed));
1944  
1945  struct st_block_data {		/* A task blocks. */
1946  	u64	when;
1947  	u64	__unused;
1948 -};
1949 +} __attribute__((packed));
1950  
1951  struct st_resume_data {		/* A task resumes. */
1952  	u64	when;
1953  	u64	__unused;
1954 -};
1955 +} __attribute__((packed));
1956  
1957  struct st_action_data {
1958  	u64	when;
1959  	u8	action;
1960  	u8	__unused[7];
1961 -};
1962 +} __attribute__((packed));
1963  
1964  struct st_sys_release_data {
1965  	u64	when;
1966  	u64	release;
1967 -};
1968 +} __attribute__((packed));
1969 +
1970 +
1971 +struct st_tasklet_release_data {
1972 +	u64 when;
1973 +	u64 __unused;
1974 +} __attribute__((packed));
1975 +
1976 +struct st_tasklet_begin_data {
1977 +	u64 when;
1978 +	u16 exe_pid;
1979 +	u8  __unused[6];
1980 +} __attribute__((packed));
1981 +
1982 +struct st_tasklet_end_data {
1983 +	u64 when;
1984 +	u16 exe_pid;
1985 +	u8	flushed;
1986 +	u8	__unused[5];
1987 +} __attribute__((packed));
1988 +
1989 +
1990 +struct st_work_release_data {
1991 +	u64 when;
1992 +	u64 __unused;
1993 +} __attribute__((packed));
1994 +
1995 +struct st_work_begin_data {
1996 +	u64 when;
1997 +	u16 exe_pid;
1998 +	u8	__unused[6];
1999 +} __attribute__((packed));
2000 +
2001 +struct st_work_end_data {
2002 +	u64 when;
2003 +	u16 exe_pid;
2004 +	u8	flushed;
2005 +	u8	__unused[5];
2006 +} __attribute__((packed));
2007 +
2008 +struct st_effective_priority_change_data {
2009 +	u64 when;
2010 +	u16 inh_pid;
2011 +	u8	__unused[6];
2012 +} __attribute__((packed));
2013 +
2014 +struct st_nv_interrupt_begin_data {
2015 +	u64 when;
2016 +	u32 device;
2017 +	u32 serialNumber;
2018 +} __attribute__((packed));
2019 +
2020 +struct st_nv_interrupt_end_data {
2021 +	u64 when;
2022 +	u32 device;
2023 +	u32 serialNumber;
2024 +} __attribute__((packed));
2025 +
2026 +struct st_prediction_err_data {
2027 +	u64 distance;
2028 +	u64 rel_err;
2029 +} __attribute__((packed));
2030 +
2031 +struct st_migration_data {
2032 +	u64 observed;
2033 +	u64 estimated;
2034 +} __attribute__((packed));
2035 +
2036 +struct migration_info {
2037 +	u64 observed;
2038 +	u64 estimated;
2039 +	u8 distance;
2040 +} __attribute__((packed));
2041  
2042  #define DATA(x) struct st_ ## x ## _data x;
2043  
2044  typedef enum {
2045 -        ST_NAME = 1,		/* Start at one, so that we can spot
2046 -				 * uninitialized records. */
2047 +    ST_NAME = 1, /* Start at one, so that we can spot
2048 +				  * uninitialized records. */
2049  	ST_PARAM,
2050  	ST_RELEASE,
2051  	ST_ASSIGNED,
2052 @@ -94,7 +167,19 @@ typedef enum {
2053  	ST_BLOCK,
2054  	ST_RESUME,
2055  	ST_ACTION,
2056 -	ST_SYS_RELEASE
2057 +	ST_SYS_RELEASE,
2058 +	ST_TASKLET_RELEASE,
2059 +	ST_TASKLET_BEGIN,
2060 +	ST_TASKLET_END,
2061 +	ST_WORK_RELEASE,
2062 +	ST_WORK_BEGIN,
2063 +	ST_WORK_END,
2064 +	ST_EFF_PRIO_CHANGE,
2065 +	ST_NV_INTERRUPT_BEGIN,
2066 +	ST_NV_INTERRUPT_END,
2067 +
2068 +	ST_PREDICTION_ERR,
2069 +	ST_MIGRATION,
2070  } st_event_record_type_t;
2071  
2072  struct st_event_record {
2073 @@ -113,8 +198,20 @@ struct st_event_record {
2074  		DATA(resume);
2075  		DATA(action);
2076  		DATA(sys_release);
2077 +		DATA(tasklet_release);
2078 +		DATA(tasklet_begin);
2079 +		DATA(tasklet_end);
2080 +		DATA(work_release);
2081 +		DATA(work_begin);
2082 +		DATA(work_end);
2083 +		DATA(effective_priority_change);
2084 +		DATA(nv_interrupt_begin);
2085 +		DATA(nv_interrupt_end);
2086 +
2087 +		DATA(prediction_err);
2088 +		DATA(migration);
2089  	} data;
2090 -};
2091 +} __attribute__((packed));
2092  
2093  #undef DATA
2094  
2095 @@ -129,6 +226,8 @@ struct st_event_record {
2096  	ft_event1(id, callback, task)
2097  #define SCHED_TRACE2(id, callback, task, xtra) \
2098  	ft_event2(id, callback, task, xtra)
2099 +#define SCHED_TRACE3(id, callback, task, xtra1, xtra2) \
2100 +	ft_event3(id, callback, task, xtra1, xtra2)
2101  
2102  /* provide prototypes; needed on sparc64 */
2103  #ifndef NO_TASK_TRACE_DECLS
2104 @@ -155,12 +254,58 @@ feather_callback void do_sched_trace_action(unsigned long id,
2105  feather_callback void do_sched_trace_sys_release(unsigned long id,
2106  						 lt_t* start);
2107  
2108 +
2109 +feather_callback void do_sched_trace_tasklet_release(unsigned long id,
2110 +												   struct task_struct* owner);
2111 +feather_callback void do_sched_trace_tasklet_begin(unsigned long id,
2112 +												  struct task_struct* owner);
2113 +feather_callback void do_sched_trace_tasklet_end(unsigned long id,
2114 +												 struct task_struct* owner,
2115 +												 unsigned long flushed);
2116 +
2117 +feather_callback void do_sched_trace_work_release(unsigned long id,
2118 +													 struct task_struct* owner);
2119 +feather_callback void do_sched_trace_work_begin(unsigned long id,
2120 +												struct task_struct* owner,
2121 +												struct task_struct* exe);
2122 +feather_callback void do_sched_trace_work_end(unsigned long id,
2123 +											  struct task_struct* owner,
2124 +											  struct task_struct* exe,
2125 +											  unsigned long flushed);
2126 +
2127 +feather_callback void do_sched_trace_eff_prio_change(unsigned long id,
2128 +											  struct task_struct* task,
2129 +											  struct task_struct* inh);
2130 +
2131 +feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id,
2132 +												u32 device);
2133 +feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id,
2134 +												unsigned long unused);
2135 +
2136 +feather_callback void do_sched_trace_prediction_err(unsigned long id,
2137 +													  struct task_struct* task,
2138 +													  gpu_migration_dist_t* distance,
2139 +													  fp_t* rel_err);
2140 +
2141 +
2142 +
2143 +
2144 +
2145 +feather_callback void do_sched_trace_migration(unsigned long id,
2146 +											  struct task_struct* task,
2147 +											  struct migration_info* mig_info);
2148 +
2149 +
2150 +/* returns true if we're tracing an interrupt on current CPU */
2151 +/* int is_interrupt_tracing_active(void); */
2152 +
2153  #endif
2154  
2155  #else
2156  
2157  #define SCHED_TRACE(id, callback, task)        /* no tracing */
2158  #define SCHED_TRACE2(id, callback, task, xtra) /* no tracing */
2159 +#define SCHED_TRACE3(id, callback, task, xtra1, xtra2)
2160  
2161  #endif
2162  
2163 @@ -193,6 +338,41 @@ feather_callback void do_sched_trace_sys_release(unsigned long id,
2164  	SCHED_TRACE(SCHED_TRACE_BASE_ID + 10, do_sched_trace_sys_release, when)
2165  
2166  
2167 +#define sched_trace_tasklet_release(t) \
2168 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 11, do_sched_trace_tasklet_release, t)
2169 +
2170 +#define sched_trace_tasklet_begin(t) \
2171 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, do_sched_trace_tasklet_begin, t)
2172 +
2173 +#define sched_trace_tasklet_end(t, flushed) \
2174 +	SCHED_TRACE2(SCHED_TRACE_BASE_ID + 13, do_sched_trace_tasklet_end, t, flushed)
2175 +
2176 +
2177 +#define sched_trace_work_release(t) \
2178 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 14, do_sched_trace_work_release, t)
2179 +
2180 +#define sched_trace_work_begin(t, e) \
2181 +	SCHED_TRACE2(SCHED_TRACE_BASE_ID + 15, do_sched_trace_work_begin, t, e)
2182 +
2183 +#define sched_trace_work_end(t, e, flushed) \
2184 +	SCHED_TRACE3(SCHED_TRACE_BASE_ID + 16, do_sched_trace_work_end, t, e, flushed)
2185 +
2186 +
2187 +#define sched_trace_eff_prio_change(t, inh) \
2188 +	SCHED_TRACE2(SCHED_TRACE_BASE_ID + 17, do_sched_trace_eff_prio_change, t, inh)
2189 +
2190 +
2191 +#define sched_trace_nv_interrupt_begin(d) \
2192 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 18, do_sched_trace_nv_interrupt_begin, d)
2193 +#define sched_trace_nv_interrupt_end(d) \
2194 +	SCHED_TRACE(SCHED_TRACE_BASE_ID + 19, do_sched_trace_nv_interrupt_end, d)
2195 +
2196 +#define sched_trace_prediction_err(t, dist, rel_err) \
2197 +	SCHED_TRACE3(SCHED_TRACE_BASE_ID + 20, do_sched_trace_prediction_err, t, dist, rel_err)
2198 +
2199 +#define sched_trace_migration(t, mig_info) \
2200 +	SCHED_TRACE2(SCHED_TRACE_BASE_ID + 21, do_sched_trace_migration, t, mig_info)
2201 +
2202  #define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */
2203  
2204  #endif /* __KERNEL__ */
2205 diff --git a/include/litmus/sched_trace_external.h b/include/litmus/sched_trace_external.h
2206 new file mode 100644
2207 index 0000000..e70e45e
2208 --- /dev/null
2209 +++ b/include/litmus/sched_trace_external.h
2210 @@ -0,0 +1,78 @@
2211 +/*
2212 + * sched_trace.h -- record scheduler events to a byte stream for offline analysis.
2213 + */
2214 +#ifndef _LINUX_SCHED_TRACE_EXTERNAL_H_
2215 +#define _LINUX_SCHED_TRACE_EXTERNAL_H_
2216 +
2217 +
2218 +#ifdef CONFIG_SCHED_TASK_TRACE
2219 +extern void __sched_trace_tasklet_begin_external(struct task_struct* t);
2220 +static inline void sched_trace_tasklet_begin_external(struct task_struct* t)
2221 +{
2222 +	__sched_trace_tasklet_begin_external(t);
2223 +}
2224 +
2225 +extern void __sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed);
2226 +static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed)
2227 +{
2228 +	__sched_trace_tasklet_end_external(t, flushed);
2229 +}
2230 +
2231 +extern void __sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e);
2232 +static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e)
2233 +{
2234 +	__sched_trace_work_begin_external(t, e);
2235 +}
2236 +
2237 +extern void __sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f);
2238 +static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f)
2239 +{
2240 +	__sched_trace_work_end_external(t, e, f);
2241 +}
2242 +
2243 +#ifdef CONFIG_LITMUS_NVIDIA
2244 +extern void __sched_trace_nv_interrupt_begin_external(u32 device);
2245 +static inline void sched_trace_nv_interrupt_begin_external(u32 device)
2246 +{
2247 +	__sched_trace_nv_interrupt_begin_external(device);
2248 +}
2249 +
2250 +extern void __sched_trace_nv_interrupt_end_external(u32 device);
2251 +static inline void sched_trace_nv_interrupt_end_external(u32 device)
2252 +{
2253 +	__sched_trace_nv_interrupt_end_external(device);
2254 +}
2255 +#endif
2256 +
2257 +#else
2258 +
2259 +// no tracing.
2260 +static inline void sched_trace_tasklet_begin_external(struct task_struct* t){}
2261 +static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed){}
2262 +static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e){}
2263 +static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f){}
2264 +
2265 +#ifdef CONFIG_LITMUS_NVIDIA
2266 +static inline void sched_trace_nv_interrupt_begin_external(u32 device){}
2267 +static inline void sched_trace_nv_interrupt_end_external(u32 device){}
2268 +#endif
2269 +
2270 +#endif
2271 +
2272 +
2273 +#ifdef CONFIG_LITMUS_NVIDIA
2274 +
2275 +#define EX_TS(evt) \
2276 +extern void __##evt(void); \
2277 +static inline void EX_##evt(void) { __##evt(); }
2278 +
2279 +EX_TS(TS_NV_TOPISR_START)
2280 +EX_TS(TS_NV_TOPISR_END)
2281 +EX_TS(TS_NV_BOTISR_START)
2282 +EX_TS(TS_NV_BOTISR_END)
2283 +EX_TS(TS_NV_RELEASE_BOTISR_START)
2284 +EX_TS(TS_NV_RELEASE_BOTISR_END)
2285 +
2286 +#endif
2287 +
2288 +#endif
2289 diff --git a/include/litmus/trace.h b/include/litmus/trace.h
2290 index e809376..e078aee 100644
2291 --- a/include/litmus/trace.h
2292 +++ b/include/litmus/trace.h
2293 @@ -103,14 +103,46 @@ feather_callback void save_task_latency(unsigned long event, unsigned long when_
2294  #define TS_LOCK_START			TIMESTAMP(170)
2295  #define TS_LOCK_SUSPEND			TIMESTAMP(171)
2296  #define TS_LOCK_RESUME			TIMESTAMP(172)
2297 -#define TS_LOCK_END			TIMESTAMP(173)
2298 +#define TS_LOCK_END				TIMESTAMP(173)
2299 +
2300 +#ifdef CONFIG_LITMUS_DGL_SUPPORT
2301 +#define TS_DGL_LOCK_START			TIMESTAMP(175)
2302 +#define TS_DGL_LOCK_SUSPEND			TIMESTAMP(176)
2303 +#define TS_DGL_LOCK_RESUME			TIMESTAMP(177)
2304 +#define TS_DGL_LOCK_END				TIMESTAMP(178)
2305 +#endif
2306  
2307  #define TS_UNLOCK_START			TIMESTAMP(180)
2308  #define TS_UNLOCK_END			TIMESTAMP(181)
2309  
2310 +#ifdef CONFIG_LITMUS_DGL_SUPPORT
2311 +#define TS_DGL_UNLOCK_START			TIMESTAMP(185)
2312 +#define TS_DGL_UNLOCK_END			TIMESTAMP(186)
2313 +#endif
2314 +
2315  #define TS_SEND_RESCHED_START(c)	CTIMESTAMP(190, c)
2316  #define TS_SEND_RESCHED_END		DTIMESTAMP(191, TSK_UNKNOWN)
2317  
2318  #define TS_RELEASE_LATENCY(when)	LTIMESTAMP(208, &(when))
2319  
2320 +
2321 +#ifdef CONFIG_LITMUS_NVIDIA
2322 +
2323 +#define TS_NV_TOPISR_START		TIMESTAMP(200)
2324 +#define TS_NV_TOPISR_END		TIMESTAMP(201)
2325 +
2326 +#define TS_NV_BOTISR_START		TIMESTAMP(202)
2327 +#define TS_NV_BOTISR_END		TIMESTAMP(203)
2328 +
2329 +#define TS_NV_RELEASE_BOTISR_START	TIMESTAMP(204)
2330 +#define TS_NV_RELEASE_BOTISR_END	TIMESTAMP(205)
2331 +
2332 +#endif
2333 +
2334 +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
2335 +#define TS_NV_SCHED_BOTISR_START	TIMESTAMP(206)
2336 +#define TS_NV_SCHED_BOTISR_END		TIMESTAMP(207)
2337 +#endif
2338 +
2339 +
2340  #endif /* !_SYS_TRACE_H_ */
2341 diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h
2342 index 94264c2..4fa514c 100644
2343 --- a/include/litmus/unistd_32.h
2344 +++ b/include/litmus/unistd_32.h
2345 @@ -17,5 +17,8 @@
2346  #define __NR_wait_for_ts_release __LSC(9)
2347  #define __NR_release_ts		__LSC(10)
2348  #define __NR_null_call		__LSC(11)
2349 +#define __NR_litmus_dgl_lock	__LSC(12)
2350 +#define __NR_litmus_dgl_unlock	__LSC(13)
2351 +#define __NR_register_nv_device			__LSC(14)
2352  
2353 -#define NR_litmus_syscalls 12
2354 +#define NR_litmus_syscalls 15
2355 diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h
2356 index d5ced0d..f80dc45 100644
2357 --- a/include/litmus/unistd_64.h
2358 +++ b/include/litmus/unistd_64.h
2359 @@ -29,5 +29,12 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release)
2360  __SYSCALL(__NR_release_ts, sys_release_ts)
2361  #define __NR_null_call				__LSC(11)
2362  __SYSCALL(__NR_null_call, sys_null_call)
2363 +#define __NR_litmus_dgl_lock		__LSC(12)
2364 +__SYSCALL(__NR_litmus_dgl_lock, sys_litmus_dgl_lock)
2365 +#define __NR_litmus_dgl_unlock		__LSC(13)
2366 +__SYSCALL(__NR_litmus_dgl_unlock, sys_litmus_dgl_unlock)
2367 +#define __NR_register_nv_device			__LSC(14)
2368 +__SYSCALL(__NR_register_nv_device, sys_register_nv_device)
2369  
2370 -#define NR_litmus_syscalls 12
2371 +
2372 +#define NR_litmus_syscalls 15
2373 diff --git a/kernel/lockdep.c b/kernel/lockdep.c
2374 index 298c927..2bdcdc3 100644
2375 --- a/kernel/lockdep.c
2376 +++ b/kernel/lockdep.c
2377 @@ -542,7 +542,7 @@ static void print_lock(struct held_lock *hlock)
2378  	print_ip_sym(hlock->acquire_ip);
2379  }
2380  
2381 -static void lockdep_print_held_locks(struct task_struct *curr)
2382 +void lockdep_print_held_locks(struct task_struct *curr)
2383  {
2384  	int i, depth = curr->lockdep_depth;
2385  
2386 @@ -558,6 +558,7 @@ static void lockdep_print_held_locks(struct task_struct *curr)
2387  		print_lock(curr->held_locks + i);
2388  	}
2389  }
2390 +EXPORT_SYMBOL(lockdep_print_held_locks);
2391  
2392  static void print_kernel_version(void)
2393  {
2394 @@ -583,6 +584,10 @@ static int static_obj(void *obj)
2395  		      end   = (unsigned long) &_end,
2396  		      addr  = (unsigned long) obj;
2397  
2398 +	// GLENN
2399 +	return 1;
2400 +
2401 +
2402  	/*
2403  	 * static variable?
2404  	 */
2405 diff --git a/kernel/mutex.c b/kernel/mutex.c
2406 index d607ed5..2f363b9 100644
2407 --- a/kernel/mutex.c
2408 +++ b/kernel/mutex.c
2409 @@ -498,3 +498,128 @@ int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
2410  	return 1;
2411  }
2412  EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
2413 +
2414 +
2415 +
2416 +
2417 +void mutex_lock_sfx(struct mutex *lock,
2418 +				   side_effect_t pre, unsigned long pre_arg,
2419 +				   side_effect_t post, unsigned long post_arg)
2420 +{
2421 +	long state = TASK_UNINTERRUPTIBLE;
2422 +
2423 +	struct task_struct *task = current;
2424 +	struct mutex_waiter waiter;
2425 +	unsigned long flags;
2426 +	
2427 +	preempt_disable();
2428 +	mutex_acquire(&lock->dep_map, subclass, 0, ip);
2429 +
2430 +	spin_lock_mutex(&lock->wait_lock, flags);
2431 +	
2432 +	if(pre)
2433 +	{
2434 +		if(unlikely(pre(pre_arg)))
2435 +		{
2436 +			// this will fuck with lockdep's CONFIG_PROVE_LOCKING...
2437 +			spin_unlock_mutex(&lock->wait_lock, flags);
2438 +			preempt_enable();
2439 +			return;
2440 +		}
2441 +	}
2442 +
2443 +	debug_mutex_lock_common(lock, &waiter);
2444 +	debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
2445 +	
2446 +	/* add waiting tasks to the end of the waitqueue (FIFO): */
2447 +	list_add_tail(&waiter.list, &lock->wait_list);
2448 +	waiter.task = task;
2449 +	
2450 +	if (atomic_xchg(&lock->count, -1) == 1)
2451 +		goto done;
2452 +	
2453 +	lock_contended(&lock->dep_map, ip);
2454 +	
2455 +	for (;;) {
2456 +		/*
2457 +		 * Lets try to take the lock again - this is needed even if
2458 +		 * we get here for the first time (shortly after failing to
2459 +		 * acquire the lock), to make sure that we get a wakeup once
2460 +		 * it's unlocked. Later on, if we sleep, this is the
2461 +		 * operation that gives us the lock. We xchg it to -1, so
2462 +		 * that when we release the lock, we properly wake up the
2463 +		 * other waiters:
2464 +		 */
2465 +		if (atomic_xchg(&lock->count, -1) == 1)
2466 +			break;
2467 +		
2468 +		__set_task_state(task, state);
2469 +		
2470 +		/* didnt get the lock, go to sleep: */
2471 +		spin_unlock_mutex(&lock->wait_lock, flags);
2472 +		preempt_enable_no_resched();
2473 +		schedule();
2474 +		preempt_disable();
2475 +		spin_lock_mutex(&lock->wait_lock, flags);
2476 +	}
2477 +	
2478 +done:
2479 +	lock_acquired(&lock->dep_map, ip);
2480 +	/* got the lock - rejoice! */
2481 +	mutex_remove_waiter(lock, &waiter, current_thread_info());
2482 +	mutex_set_owner(lock);
2483 +	
2484 +	/* set it to 0 if there are no waiters left: */
2485 +	if (likely(list_empty(&lock->wait_list)))
2486 +		atomic_set(&lock->count, 0);
2487 +	
2488 +	if(post)
2489 +		post(post_arg);	
2490 +	
2491 +	spin_unlock_mutex(&lock->wait_lock, flags);
2492 +	
2493 +	debug_mutex_free_waiter(&waiter);
2494 +	preempt_enable();
2495 +}
2496 +EXPORT_SYMBOL(mutex_lock_sfx);
2497 +
2498 +void mutex_unlock_sfx(struct mutex *lock,
2499 +					side_effect_t pre, unsigned long pre_arg,
2500 +					side_effect_t post, unsigned long post_arg)
2501 +{
2502 +	unsigned long flags;
2503 +	
2504 +	spin_lock_mutex(&lock->wait_lock, flags);
2505 +	
2506 +	if(pre)
2507 +		pre(pre_arg);
2508 +	
2509 +	//mutex_release(&lock->dep_map, nested, _RET_IP_);
2510 +	mutex_release(&lock->dep_map, 1, _RET_IP_);
2511 +	debug_mutex_unlock(lock);
2512 +	
2513 +	/*
2514 +	 * some architectures leave the lock unlocked in the fastpath failure
2515 +	 * case, others need to leave it locked. In the later case we have to
2516 +	 * unlock it here
2517 +	 */
2518 +	if (__mutex_slowpath_needs_to_unlock())
2519 +		atomic_set(&lock->count, 1);
2520 +	
2521 +	if (!list_empty(&lock->wait_list)) {
2522 +		/* get the first entry from the wait-list: */
2523 +		struct mutex_waiter *waiter =
2524 +		list_entry(lock->wait_list.next,
2525 +				   struct mutex_waiter, list);
2526 +		
2527 +		debug_mutex_wake_waiter(lock, waiter);
2528 +		
2529 +		wake_up_process(waiter->task);
2530 +	}
2531 +	
2532 +	if(post)
2533 +		post(post_arg);
2534 +	
2535 +	spin_unlock_mutex(&lock->wait_lock, flags);	
2536 +}
2537 +EXPORT_SYMBOL(mutex_unlock_sfx);
2538 diff --git a/kernel/sched.c b/kernel/sched.c
2539 index baaca61..f3d9a69 100644
2540 --- a/kernel/sched.c
2541 +++ b/kernel/sched.c
2542 @@ -83,6 +83,10 @@
2543  #include <litmus/sched_trace.h>
2544  #include <litmus/trace.h>
2545  
2546 +#ifdef CONFIG_LITMUS_SOFTIRQD
2547 +#include <litmus/litmus_softirq.h>
2548 +#endif
2549 +
2550  static void litmus_tick(struct rq*, struct task_struct*);
2551  
2552  #define CREATE_TRACE_POINTS
2553 @@ -4305,6 +4309,7 @@ pick_next_task(struct rq *rq)
2554  	BUG(); /* the idle class will always have a runnable task */
2555  }
2556  
2557 +
2558  /*
2559   * schedule() is the main scheduler function.
2560   */
2561 @@ -4323,6 +4328,10 @@ need_resched:
2562  	rcu_note_context_switch(cpu);
2563  	prev = rq->curr;
2564  
2565 +#ifdef CONFIG_LITMUS_SOFTIRQD
2566 +	release_klitirqd_lock(prev);
2567 +#endif
2568 +
2569  	/* LITMUS^RT: quickly re-evaluate the scheduling decision
2570  	 * if the previous one is no longer valid after CTX.
2571  	 */
2572 @@ -4411,13 +4420,24 @@ litmus_need_resched_nonpreemptible:
2573  		goto litmus_need_resched_nonpreemptible;
2574  
2575  	preempt_enable_no_resched();
2576 +
2577  	if (need_resched())
2578  		goto need_resched;
2579  
2580 +#ifdef LITMUS_SOFTIRQD
2581 +	reacquire_klitirqd_lock(prev);
2582 +#endif
2583 +
2584 +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
2585 +	litmus->run_tasklets(prev);
2586 +#endif	
2587 +	
2588  	srp_ceiling_block();
2589  }
2590  EXPORT_SYMBOL(schedule);
2591  
2592 +
2593 +
2594  #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
2595  
2596  static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
2597 @@ -4561,6 +4581,7 @@ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
2598  	}
2599  }
2600  
2601 +
2602  /**
2603   * __wake_up - wake up threads blocked on a waitqueue.
2604   * @q: the waitqueue
2605 @@ -4747,6 +4768,12 @@ void __sched wait_for_completion(struct completion *x)
2606  }
2607  EXPORT_SYMBOL(wait_for_completion);
2608  
2609 +void __sched __wait_for_completion_locked(struct completion *x)
2610 +{
2611 +	do_wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
2612 +}
2613 +EXPORT_SYMBOL(__wait_for_completion_locked);
2614 +
2615  /**
2616   * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
2617   * @x:  holds the state of this particular completion
2618 diff --git a/kernel/semaphore.c b/kernel/semaphore.c
2619 index 94a62c0..c947a04 100644
2620 --- a/kernel/semaphore.c
2621 +++ b/kernel/semaphore.c
2622 @@ -33,11 +33,11 @@
2623  #include <linux/spinlock.h>
2624  #include <linux/ftrace.h>
2625  
2626 -static noinline void __down(struct semaphore *sem);
2627 +noinline void __down(struct semaphore *sem);
2628  static noinline int __down_interruptible(struct semaphore *sem);
2629  static noinline int __down_killable(struct semaphore *sem);
2630  static noinline int __down_timeout(struct semaphore *sem, long jiffies);
2631 -static noinline void __up(struct semaphore *sem);
2632 +noinline void __up(struct semaphore *sem);
2633  
2634  /**
2635   * down - acquire the semaphore
2636 @@ -190,11 +190,13 @@ EXPORT_SYMBOL(up);
2637  
2638  /* Functions for the contended case */
2639  
2640 +/*
2641  struct semaphore_waiter {
2642  	struct list_head list;
2643  	struct task_struct *task;
2644  	int up;
2645  };
2646 + */
2647  
2648  /*
2649   * Because this function is inlined, the 'state' parameter will be
2650 @@ -233,10 +235,12 @@ static inline int __sched __down_common(struct semaphore *sem, long state,
2651  	return -EINTR;
2652  }
2653  
2654 -static noinline void __sched __down(struct semaphore *sem)
2655 +noinline void __sched __down(struct semaphore *sem)
2656  {
2657  	__down_common(sem, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
2658  }
2659 +EXPORT_SYMBOL(__down);
2660 +
2661  
2662  static noinline int __sched __down_interruptible(struct semaphore *sem)
2663  {
2664 @@ -253,7 +257,7 @@ static noinline int __sched __down_timeout(struct semaphore *sem, long jiffies)
2665  	return __down_common(sem, TASK_UNINTERRUPTIBLE, jiffies);
2666  }
2667  
2668 -static noinline void __sched __up(struct semaphore *sem)
2669 +noinline void __sched __up(struct semaphore *sem)
2670  {
2671  	struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list,
2672  						struct semaphore_waiter, list);
2673 @@ -261,3 +265,4 @@ static noinline void __sched __up(struct semaphore *sem)
2674  	waiter->up = 1;
2675  	wake_up_process(waiter->task);
2676  }
2677 +EXPORT_SYMBOL(__up);
2678 \ No newline at end of file
2679 diff --git a/kernel/softirq.c b/kernel/softirq.c
2680 index fca82c3..5ce2716 100644
2681 --- a/kernel/softirq.c
2682 +++ b/kernel/softirq.c
2683 @@ -29,6 +29,15 @@
2684  #include <trace/events/irq.h>
2685  
2686  #include <asm/irq.h>
2687 +
2688 +#include <litmus/litmus.h>
2689 +#include <litmus/sched_trace.h>
2690 +
2691 +#ifdef CONFIG_LITMUS_NVIDIA
2692 +#include <litmus/nvidia_info.h>
2693 +#include <litmus/trace.h>
2694 +#endif
2695 +
2696  /*
2697     - No shared variables, all the data are CPU local.
2698     - If a softirq needs serialization, let it serialize itself
2699 @@ -67,7 +76,7 @@ char *softirq_to_name[NR_SOFTIRQS] = {
2700   * to the pending events, so lets the scheduler to balance
2701   * the softirq load for us.
2702   */
2703 -static void wakeup_softirqd(void)
2704 +void wakeup_softirqd(void)
2705  {
2706  	/* Interrupts are disabled: no need to stop preemption */
2707  	struct task_struct *tsk = __this_cpu_read(ksoftirqd);
2708 @@ -193,6 +202,7 @@ void local_bh_enable_ip(unsigned long ip)
2709  }
2710  EXPORT_SYMBOL(local_bh_enable_ip);
2711  
2712 +
2713  /*
2714   * We restart softirq processing MAX_SOFTIRQ_RESTART times,
2715   * and we fall back to softirqd after that.
2716 @@ -206,65 +216,65 @@ EXPORT_SYMBOL(local_bh_enable_ip);
2717  
2718  asmlinkage void __do_softirq(void)
2719  {
2720 -	struct softirq_action *h;
2721 -	__u32 pending;
2722 -	int max_restart = MAX_SOFTIRQ_RESTART;
2723 -	int cpu;
2724 +    struct softirq_action *h; 
2725 +    __u32 pending;
2726 +    int max_restart = MAX_SOFTIRQ_RESTART;
2727 +    int cpu;
2728  
2729 -	pending = local_softirq_pending();
2730 -	account_system_vtime(current);
2731 +    pending = local_softirq_pending();
2732 +    account_system_vtime(current);
2733  
2734 -	__local_bh_disable((unsigned long)__builtin_return_address(0),
2735 -				SOFTIRQ_OFFSET);
2736 -	lockdep_softirq_enter();
2737 +    __local_bh_disable((unsigned long)__builtin_return_address(0),
2738 +                SOFTIRQ_OFFSET);
2739 +    lockdep_softirq_enter();
2740  
2741 -	cpu = smp_processor_id();
2742 +    cpu = smp_processor_id();
2743  restart:
2744 -	/* Reset the pending bitmask before enabling irqs */
2745 -	set_softirq_pending(0);
2746 +    /* Reset the pending bitmask before enabling irqs */
2747 +    set_softirq_pending(0);
2748  
2749 -	local_irq_enable();
2750 +    local_irq_enable();
2751  
2752 -	h = softirq_vec;
2753 -
2754 -	do {
2755 -		if (pending & 1) {
2756 -			unsigned int vec_nr = h - softirq_vec;
2757 -			int prev_count = preempt_count();
2758 -
2759 -			kstat_incr_softirqs_this_cpu(vec_nr);
2760 -
2761 -			trace_softirq_entry(vec_nr);
2762 -			h->action(h);
2763 -			trace_softirq_exit(vec_nr);
2764 -			if (unlikely(prev_count != preempt_count())) {
2765 -				printk(KERN_ERR "huh, entered softirq %u %s %p"
2766 -				       "with preempt_count %08x,"
2767 -				       " exited with %08x?\n", vec_nr,
2768 -				       softirq_to_name[vec_nr], h->action,
2769 -				       prev_count, preempt_count());
2770 -				preempt_count() = prev_count;
2771 -			}
2772 +    h = softirq_vec;
2773  
2774 -			rcu_bh_qs(cpu);
2775 -		}
2776 -		h++;
2777 -		pending >>= 1;
2778 -	} while (pending);
2779 +    do {
2780 +        if (pending & 1) {
2781 +            unsigned int vec_nr = h - softirq_vec;
2782 +            int prev_count = preempt_count();
2783  
2784 -	local_irq_disable();
2785 +            kstat_incr_softirqs_this_cpu(vec_nr);
2786  
2787 -	pending = local_softirq_pending();
2788 -	if (pending && --max_restart)
2789 -		goto restart;
2790 +            trace_softirq_entry(vec_nr);
2791 +            h->action(h);
2792 +            trace_softirq_exit(vec_nr);
2793 +            if (unlikely(prev_count != preempt_count())) {
2794 +                printk(KERN_ERR "huh, entered softirq %u %s %p"
2795 +                       "with preempt_count %08x,"
2796 +                       " exited with %08x?\n", vec_nr,
2797 +                       softirq_to_name[vec_nr], h->action,
2798 +                       prev_count, preempt_count());
2799 +                preempt_count() = prev_count;
2800 +            }   
2801  
2802 -	if (pending)
2803 -		wakeup_softirqd();
2804 +            rcu_bh_qs(cpu);
2805 +        }   
2806 +        h++;
2807 +        pending >>= 1;
2808 +    } while (pending);
2809  
2810 -	lockdep_softirq_exit();
2811 +    local_irq_disable();
2812  
2813 -	account_system_vtime(current);
2814 -	__local_bh_enable(SOFTIRQ_OFFSET);
2815 +    pending = local_softirq_pending();
2816 +    if (pending && --max_restart)
2817 +        goto restart;
2818 +
2819 +    if (pending)
2820 +        wakeup_softirqd();
2821 +
2822 +    lockdep_softirq_exit();
2823 +
2824 +    account_system_vtime(current);
2825 +    __local_bh_enable(SOFTIRQ_OFFSET);
2826  }
2827  
2828  #ifndef __ARCH_HAS_DO_SOFTIRQ
2829 @@ -402,8 +412,99 @@ struct tasklet_head
2830  static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
2831  static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
2832  
2833 +#ifdef CONFIG_LITMUS_NVIDIA
2834 +static int __do_nv_now(struct tasklet_struct* tasklet)
2835 +{
2836 +	int success = 1;
2837 +
2838 +	if(tasklet_trylock(tasklet)) {
2839 +		if (!atomic_read(&tasklet->count)) {
2840 +			if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) {    
2841 +				BUG();
2842 +			}
2843 +			tasklet->func(tasklet->data);
2844 +			tasklet_unlock(tasklet);
2845 +		}
2846 +		else {
2847 +			success = 0;
2848 +		}
2849 +
2850 +		tasklet_unlock(tasklet);
2851 +	}
2852 +	else {
2853 +		success = 0;
2854 +	}
2855 +
2856 +	return success;
2857 +}
2858 +#endif
2859 +
2860 +
2861  void __tasklet_schedule(struct tasklet_struct *t)
2862  {
2863 +#ifdef CONFIG_LITMUS_NVIDIA
2864 +	if(is_nvidia_func(t->func))
2865 +	{
2866 +#if 0
2867 +		// do nvidia tasklets right away and return
2868 +		if(__do_nv_now(t))
2869 +			return;
2870 +#else
2871 +		u32 nvidia_device = get_tasklet_nv_device_num(t);	
2872 +		//		TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
2873 +		//			  __FUNCTION__, nvidia_device,litmus_clock());
2874 +
2875 +		unsigned long flags;
2876 +		struct task_struct* device_owner;
2877 +
2878 +		lock_nv_registry(nvidia_device, &flags);
2879 +
2880 +		device_owner = get_nv_max_device_owner(nvidia_device);
2881 +
2882 +		if(device_owner==NULL)
2883 +		{
2884 +			t->owner = NULL;
2885 +		}
2886 +		else
2887 +		{
2888 +			if(is_realtime(device_owner))
2889 +			{
2890 +				TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
2891 +					  __FUNCTION__, nvidia_device,litmus_clock());				
2892 +				TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
2893 +					  __FUNCTION__,device_owner->pid,nvidia_device);
2894 +
2895 +				t->owner = device_owner;
2896 +				sched_trace_tasklet_release(t->owner);
2897 +
2898 +				if(likely(_litmus_tasklet_schedule(t,nvidia_device)))
2899 +				{
2900 +					unlock_nv_registry(nvidia_device, &flags);
2901 +					return;
2902 +				}
2903 +				else
2904 +				{
2905 +					t->owner = NULL; /* fall through to normal scheduling */
2906 +				}
2907 +			}
2908 +			else
2909 +			{
2910 +				t->owner = NULL;
2911 +			}
2912 +		}
2913 +		unlock_nv_registry(nvidia_device, &flags);
2914 +#endif
2915 +	}
2916 +
2917 +#endif
2918 +
2919 +	___tasklet_schedule(t);
2920 +}
2921 +EXPORT_SYMBOL(__tasklet_schedule);
2922 +
2923 +
2924 +void ___tasklet_schedule(struct tasklet_struct *t)
2925 +{
2926  	unsigned long flags;
2927  
2928  	local_irq_save(flags);
2929 @@ -413,11 +514,65 @@ void __tasklet_schedule(struct tasklet_struct *t)
2930  	raise_softirq_irqoff(TASKLET_SOFTIRQ);
2931  	local_irq_restore(flags);
2932  }
2933 +EXPORT_SYMBOL(___tasklet_schedule);
2934  
2935 -EXPORT_SYMBOL(__tasklet_schedule);
2936  
2937  void __tasklet_hi_schedule(struct tasklet_struct *t)
2938  {
2939 +#ifdef CONFIG_LITMUS_NVIDIA
2940 +	if(is_nvidia_func(t->func))
2941 +	{	
2942 +		u32 nvidia_device = get_tasklet_nv_device_num(t);
2943 +		//		TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
2944 +		//			  __FUNCTION__, nvidia_device,litmus_clock());
2945 +
2946 +		unsigned long flags;
2947 +		struct task_struct* device_owner;
2948 +		
2949 +		lock_nv_registry(nvidia_device, &flags);
2950 +		
2951 +		device_owner = get_nv_max_device_owner(nvidia_device);
2952 +
2953 +		if(device_owner==NULL) 
2954 +		{
2955 +			t->owner = NULL;
2956 +		}
2957 +		else
2958 +		{
2959 +			if( is_realtime(device_owner))
2960 +			{
2961 +				TRACE("%s: Handling NVIDIA tasklet for device %u\tat %llu\n",
2962 +					  __FUNCTION__, nvidia_device,litmus_clock());				
2963 +				TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
2964 +					  __FUNCTION__,device_owner->pid,nvidia_device);
2965 +				
2966 +				t->owner = device_owner;
2967 +				sched_trace_tasklet_release(t->owner);
2968 +				if(likely(_litmus_tasklet_hi_schedule(t,nvidia_device)))
2969 +				{
2970 +					unlock_nv_registry(nvidia_device, &flags);
2971 +					return;
2972 +				}
2973 +				else
2974 +				{
2975 +					t->owner = NULL; /* fall through to normal scheduling */
2976 +				}
2977 +			}
2978 +			else
2979 +			{
2980 +				t->owner = NULL;
2981 +			}
2982 +		}
2983 +		unlock_nv_registry(nvidia_device, &flags);
2984 +	}
2985 +#endif
2986 +
2987 +	___tasklet_hi_schedule(t);
2988 +}
2989 +EXPORT_SYMBOL(__tasklet_hi_schedule);
2990 +
2991 +void ___tasklet_hi_schedule(struct tasklet_struct* t)
2992 +{
2993  	unsigned long flags;
2994  
2995  	local_irq_save(flags);
2996 @@ -427,19 +582,72 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
2997  	raise_softirq_irqoff(HI_SOFTIRQ);
2998  	local_irq_restore(flags);
2999  }
3000 -
3001 -EXPORT_SYMBOL(__tasklet_hi_schedule);
3002 +EXPORT_SYMBOL(___tasklet_hi_schedule);
3003  
3004  void __tasklet_hi_schedule_first(struct tasklet_struct *t)
3005  {
3006  	BUG_ON(!irqs_disabled());
3007 +#ifdef CONFIG_LITMUS_NVIDIA	
3008 +	if(is_nvidia_func(t->func))
3009 +	{	
3010 +		u32 nvidia_device = get_tasklet_nv_device_num(t);
3011 +		//		TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
3012 +		//			  __FUNCTION__, nvidia_device,litmus_clock());
3013 +		unsigned long flags;
3014 +		struct task_struct* device_owner;
3015 +		
3016 +		lock_nv_registry(nvidia_device, &flags);
3017 +
3018 +		device_owner = get_nv_max_device_owner(nvidia_device);
3019 +
3020 +		if(device_owner==NULL)
3021 +		{
3022 +			t->owner = NULL;
3023 +		}
3024 +		else
3025 +		{
3026 +			if(is_realtime(device_owner))
3027 +			{
3028 +				TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
3029 +					  __FUNCTION__, nvidia_device,litmus_clock());
3030 +				
3031 +				TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
3032 +					  __FUNCTION__,device_owner->pid,nvidia_device);
3033 +				
3034 +				t->owner = device_owner;
3035 +				sched_trace_tasklet_release(t->owner);
3036 +				if(likely(_litmus_tasklet_hi_schedule_first(t,nvidia_device)))
3037 +				{
3038 +					unlock_nv_registry(nvidia_device, &flags);
3039 +					return;
3040 +				}
3041 +				else
3042 +				{
3043 +					t->owner = NULL; /* fall through to normal scheduling */
3044 +				}
3045 +			}
3046 +			else
3047 +			{
3048 +				t->owner = NULL;
3049 +			}
3050 +		}
3051 +		unlock_nv_registry(nvidia_device, &flags);
3052 +	}
3053 +#endif
3054 +
3055 +	___tasklet_hi_schedule_first(t);
3056 +}
3057 +EXPORT_SYMBOL(__tasklet_hi_schedule_first);
3058 +
3059 +void ___tasklet_hi_schedule_first(struct tasklet_struct* t)
3060 +{
3061 +	BUG_ON(!irqs_disabled());
3062  
3063  	t->next = __this_cpu_read(tasklet_hi_vec.head);
3064  	__this_cpu_write(tasklet_hi_vec.head, t);
3065  	__raise_softirq_irqoff(HI_SOFTIRQ);
3066  }
3067 -
3068 -EXPORT_SYMBOL(__tasklet_hi_schedule_first);
3069 +EXPORT_SYMBOL(___tasklet_hi_schedule_first);
3070  
3071  static void tasklet_action(struct softirq_action *a)
3072  {
3073 @@ -495,6 +703,7 @@ static void tasklet_hi_action(struct softirq_action *a)
3074  			if (!atomic_read(&t->count)) {
3075  				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
3076  					BUG();
3077 +
3078  				t->func(t->data);
3079  				tasklet_unlock(t);
3080  				continue;
3081 @@ -518,8 +727,13 @@ void tasklet_init(struct tasklet_struct *t,
3082  	t->next = NULL;
3083  	t->state = 0;
3084  	atomic_set(&t->count, 0);
3085 +
3086  	t->func = func;
3087  	t->data = data;
3088 +
3089 +#ifdef CONFIG_LITMUS_SOFTIRQD
3090 +	t->owner = NULL;
3091 +#endif
3092  }
3093  
3094  EXPORT_SYMBOL(tasklet_init);
3095 @@ -534,6 +748,7 @@ void tasklet_kill(struct tasklet_struct *t)
3096  			yield();
3097  		} while (test_bit(TASKLET_STATE_SCHED, &t->state));
3098  	}
3099 +
3100  	tasklet_unlock_wait(t);
3101  	clear_bit(TASKLET_STATE_SCHED, &t->state);