attachment:tracing-and-dflp-rtas13.patch of Publications

   1 diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
   2 index ed4c4f5..7539d84 100644
   3 --- a/arch/x86/kernel/smp.c
   4 +++ b/arch/x86/kernel/smp.c
   5 @@ -25,7 +25,6 @@
   6  
   7  #include <litmus/preempt.h>
   8  #include <litmus/debug_trace.h>
   9 -#include <litmus/trace.h>
  10  
  11  #include <asm/mtrr.h>
  12  #include <asm/tlbflush.h>
  13 @@ -122,7 +121,6 @@ static void native_smp_send_reschedule(int cpu)
  14  		WARN_ON(1);
  15  		return;
  16  	}
  17 -	TS_SEND_RESCHED_START(cpu);
  18  	apic->send_IPI_mask(cpumask_of(cpu), RESCHEDULE_VECTOR);
  19  }
  20  
  21 @@ -214,18 +212,16 @@ static void native_stop_other_cpus(int wait)
  22  void smp_reschedule_interrupt(struct pt_regs *regs)
  23  {
  24  	ack_APIC_irq();
  25 -	/* LITMUS^RT: this IPI might need to trigger the sched state machine. */
  26 -	sched_state_ipi();
  27  	inc_irq_stat(irq_resched_count);
  28 -	/*
  29 -	 * LITMUS^RT: starting from 3.0 schedule_ipi() actually does something.
  30 -	 * This may increase IPI latencies compared with previous versions.
  31 -	 */
  32  	scheduler_ipi();
  33 -	TS_SEND_RESCHED_END;
  34  	/*
  35  	 * KVM uses this interrupt to force a cpu out of guest mode
  36  	 */
  37 +
  38 +	/* LITMUS^RT: this IPI might need to trigger the sched state machine.
  39 +	 * Starting from 3.0 schedule_ipi() actually does something.  This may
  40 +	 * increase IPI latencies compared with previous versions. */
  41 +	sched_state_ipi();
  42  }
  43  
  44  void smp_call_function_interrupt(struct pt_regs *regs)
  45 @@ -251,8 +247,10 @@ extern void hrtimer_pull(void);
  46  void smp_pull_timers_interrupt(struct pt_regs *regs)
  47  {
  48  	ack_APIC_irq();
  49 +	irq_enter();
  50  	TRACE("pull timer interrupt\n");
  51  	hrtimer_pull();
  52 +	irq_exit();
  53  }
  54  
  55  struct smp_ops smp_ops = {
  56 diff --git a/include/linux/completion.h b/include/linux/completion.h
  57 index 9d72727..51494e6 100644
  58 --- a/include/linux/completion.h
  59 +++ b/include/linux/completion.h
  60 @@ -90,7 +90,6 @@ extern bool completion_done(struct completion *x);
  61  
  62  extern void complete(struct completion *);
  63  extern void complete_all(struct completion *);
  64 -extern void complete_n(struct completion *, int n);
  65  
  66  /**
  67   * INIT_COMPLETION - reinitialize a completion structure
  68 diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h
  69 index f2115b8..fd9b30d 100644
  70 --- a/include/litmus/fdso.h
  71 +++ b/include/litmus/fdso.h
  72 @@ -23,10 +23,11 @@ typedef enum  {
  73  	MPCP_SEM	= 2,
  74  	MPCP_VS_SEM	= 3,
  75  	DPCP_SEM	= 4,
  76 -
  77  	PCP_SEM         = 5,
  78  
  79 -	MAX_OBJ_TYPE	= 5
  80 +	DFLP_SEM	= 6,
  81 +
  82 +	MAX_OBJ_TYPE	= 6
  83  } obj_type_t;
  84  
  85  struct inode_obj_id {
  86 diff --git a/include/litmus/fp_common.h b/include/litmus/fp_common.h
  87 index dd1f7bf..19356c0 100644
  88 --- a/include/litmus/fp_common.h
  89 +++ b/include/litmus/fp_common.h
  90 @@ -57,7 +57,7 @@ static inline unsigned int fpq_find(struct fp_prio_queue* q)
  91  
  92  static inline void fp_prio_add(struct fp_prio_queue* q, struct task_struct* t, unsigned int index)
  93  {
  94 -
  95 +	BUG_ON(index >= LITMUS_MAX_PRIORITY);
  96  	BUG_ON(bheap_node_in_heap(tsk_rt(t)->heap_node));
  97  
  98  	fpq_set(q, index);
  99 diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
 100 index 338245a..9dfcac4 100644
 101 --- a/include/litmus/litmus.h
 102 +++ b/include/litmus/litmus.h
 103 @@ -259,4 +259,39 @@ static inline quanta_t time2quanta(lt_t time, enum round round)
 104  /* By how much is cpu staggered behind CPU 0? */
 105  u64 cpu_stagger_offset(int cpu);
 106  
 107 +static inline struct control_page* get_control_page(struct task_struct *t)
 108 +{
 109 +	return tsk_rt(t)->ctrl_page;
 110 +}
 111 +
 112 +static inline int has_control_page(struct task_struct* t)
 113 +{
 114 +	return tsk_rt(t)->ctrl_page != NULL;
 115 +}
 116 +
 117 +
 118 +#ifdef CONFIG_SCHED_OVERHEAD_TRACE
 119 +
 120 +#define TS_SYSCALL_IN_START						\
 121 +	if (has_control_page(current)) {				\
 122 +		__TS_SYSCALL_IN_START(&get_control_page(current)->ts_syscall_start); \
 123 +	}
 124 +
 125 +#define TS_SYSCALL_IN_END						\
 126 +	if (has_control_page(current)) {				\
 127 +		uint64_t irqs;						\
 128 +		local_irq_disable();					\
 129 +		irqs = get_control_page(current)->irq_count -		\
 130 +			get_control_page(current)->irq_syscall_start;	\
 131 +		__TS_SYSCALL_IN_END(&irqs);				\
 132 +		local_irq_enable();					\
 133 +	}
 134 +
 135 +#else
 136 +
 137 +#define TS_SYSCALL_IN_START
 138 +#define TS_SYSCALL_IN_END
 139 +
 140 +#endif
 141 +
 142  #endif
 143 diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
 144 index 89ac0dd..b77dde9 100644
 145 --- a/include/litmus/rt_param.h
 146 +++ b/include/litmus/rt_param.h
 147 @@ -65,12 +65,12 @@ struct rt_task {
 148  };
 149  
 150  union np_flag {
 151 -	uint32_t raw;
 152 +	uint64_t raw;
 153  	struct {
 154  		/* Is the task currently in a non-preemptive section? */
 155 -		uint32_t flag:31;
 156 +		uint64_t flag:31;
 157  		/* Should the task call into the scheduler? */
 158 -		uint32_t preempt:1;
 159 +		uint64_t preempt:1;
 160  	} np;
 161  };
 162  
 163 @@ -89,11 +89,29 @@ union np_flag {
 164   * determining preemption/migration overheads).
 165   */
 166  struct control_page {
 167 +	/* This flag is used by userspace to communicate non-preempive
 168 +	 * sections. */
 169  	volatile union np_flag sched;
 170  
 171 +	volatile uint64_t irq_count; /* Incremented by the kernel each time an IRQ is
 172 +				      * handled. */
 173 +
 174 +	/* Locking overhead tracing: userspace records here the time stamp
 175 +	 * and IRQ counter prior to starting the system call. */
 176 +	uint64_t ts_syscall_start;  /* Feather-Trace cycles */
 177 +	uint64_t irq_syscall_start; /* Snapshot of irq_count when the syscall
 178 +				     * started. */
 179 +
 180  	/* to be extended */
 181  };
 182  
 183 +/* Expected offsets within the control page. */
 184 +
 185 +#define LITMUS_CP_OFFSET_SCHED		0
 186 +#define LITMUS_CP_OFFSET_IRQ_COUNT	8
 187 +#define LITMUS_CP_OFFSET_TS_SC_START	16
 188 +#define LITMUS_CP_OFFSET_IRQ_SC_START	24
 189 +
 190  /* don't export internal data structures to user space (liblitmus) */
 191  #ifdef __KERNEL__
 192  
 193 diff --git a/include/litmus/trace.h b/include/litmus/trace.h
 194 index e809376..2cd1903 100644
 195 --- a/include/litmus/trace.h
 196 +++ b/include/litmus/trace.h
 197 @@ -16,7 +16,8 @@ enum task_type_marker {
 198  };
 199  
 200  struct timestamp {
 201 -	uint64_t		timestamp;
 202 +	uint64_t		timestamp:48;
 203 +	uint64_t		pid:16;
 204  	uint32_t		seq_no;
 205  	uint8_t			cpu;
 206  	uint8_t			event;
 207 @@ -31,11 +32,16 @@ feather_callback void save_timestamp_def(unsigned long event, unsigned long type
 208  feather_callback void save_timestamp_task(unsigned long event, unsigned long t_ptr);
 209  feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu);
 210  feather_callback void save_task_latency(unsigned long event, unsigned long when_ptr);
 211 +feather_callback void save_timestamp_time(unsigned long event, unsigned long time_ptr);
 212 +feather_callback void save_timestamp_irq(unsigned long event, unsigned long irq_count_ptr);
 213 +feather_callback void save_timestamp_hide_irq(unsigned long event);
 214  
 215  #define TIMESTAMP(id) ft_event0(id, save_timestamp)
 216  
 217  #define DTIMESTAMP(id, def)  ft_event1(id, save_timestamp_def, (unsigned long) def)
 218  
 219 +#define TIMESTAMP_CUR(id) DTIMESTAMP(id, is_realtime(current) ? TSK_RT : TSK_BE)
 220 +
 221  #define TTIMESTAMP(id, task) \
 222  	ft_event1(id, save_timestamp_task, (unsigned long) task)
 223  
 224 @@ -45,18 +51,35 @@ feather_callback void save_task_latency(unsigned long event, unsigned long when_
 225  #define LTIMESTAMP(id, task) \
 226  	ft_event1(id, save_task_latency, (unsigned long) task)
 227  
 228 +#define TIMESTAMP_TIME(id, time_ptr) \
 229 +	ft_event1(id, save_timestamp_time, (unsigned long) time_ptr)
 230 +
 231 +#define TIMESTAMP_IRQ(id, irq_count_ptr) \
 232 +	ft_event1(id, save_timestamp_irq, (unsigned long) irq_count_ptr)
 233 +
 234 +#define TIMESTAMP_IN_IRQ(id) \
 235 +	ft_event0(id, save_timestamp_hide_irq)
 236 +
 237  #else /* !CONFIG_SCHED_OVERHEAD_TRACE */
 238  
 239  #define TIMESTAMP(id)        /* no tracing */
 240  
 241  #define DTIMESTAMP(id, def)  /* no tracing */
 242  
 243 +#define TIMESTAMP_CUR(id)    /* no tracing */
 244 +
 245  #define TTIMESTAMP(id, task) /* no tracing */
 246  
 247  #define CTIMESTAMP(id, cpu)  /* no tracing */
 248  
 249  #define LTIMESTAMP(id, when_ptr) /* no tracing */
 250  
 251 +#define TIMESTAMP_TIME(id, time_ptr) /* no tracing */
 252 +
 253 +#define TIMESTAMP_IRQ(id, irq_count_ptr) /* no tracing */
 254 +
 255 +#define TIMESTAMP_IN_IRQ(id) /* no tracing */
 256 +
 257  #endif
 258  
 259  
 260 @@ -68,7 +91,20 @@ feather_callback void save_task_latency(unsigned long event, unsigned long when_
 261   * always the next number after the start time event id.
 262   */
 263  
 264 +#define __TS_SYSCALL_IN_START(p)	TIMESTAMP_TIME(10, p)
 265 +#define __TS_SYSCALL_IN_END(p)		TIMESTAMP_IRQ(11, p)
 266 +
 267 +#define TS_SYSCALL_OUT_START		TIMESTAMP_CUR(20)
 268 +#define TS_SYSCALL_OUT_END		TIMESTAMP_CUR(21)
 269 +
 270 +#define TS_LOCK_START			TIMESTAMP_CUR(30)
 271 +#define TS_LOCK_END			TIMESTAMP_CUR(31)
 272  
 273 +#define TS_LOCK_SUSPEND			TIMESTAMP_CUR(38)
 274 +#define TS_LOCK_RESUME			TIMESTAMP_CUR(39)
 275 +
 276 +#define TS_UNLOCK_START			TIMESTAMP_CUR(40)
 277 +#define TS_UNLOCK_END			TIMESTAMP_CUR(41)
 278  
 279  #define TS_SCHED_START			DTIMESTAMP(100, TSK_UNKNOWN) /* we only
 280  								      * care
 281 @@ -100,16 +136,8 @@ feather_callback void save_task_latency(unsigned long event, unsigned long when_
 282  #define TS_EXIT_NP_START		TIMESTAMP(150)
 283  #define TS_EXIT_NP_END			TIMESTAMP(151)
 284  
 285 -#define TS_LOCK_START			TIMESTAMP(170)
 286 -#define TS_LOCK_SUSPEND			TIMESTAMP(171)
 287 -#define TS_LOCK_RESUME			TIMESTAMP(172)
 288 -#define TS_LOCK_END			TIMESTAMP(173)
 289 -
 290 -#define TS_UNLOCK_START			TIMESTAMP(180)
 291 -#define TS_UNLOCK_END			TIMESTAMP(181)
 292 -
 293  #define TS_SEND_RESCHED_START(c)	CTIMESTAMP(190, c)
 294 -#define TS_SEND_RESCHED_END		DTIMESTAMP(191, TSK_UNKNOWN)
 295 +#define TS_SEND_RESCHED_END		TIMESTAMP_IN_IRQ(191)
 296  
 297  #define TS_RELEASE_LATENCY(when)	LTIMESTAMP(208, &(when))
 298  
 299 diff --git a/include/litmus/trace_irq.h b/include/litmus/trace_irq.h
 300 index f18b127..0d0c042 100644
 301 --- a/include/litmus/trace_irq.h
 302 +++ b/include/litmus/trace_irq.h
 303 @@ -3,14 +3,7 @@
 304  
 305  #ifdef CONFIG_SCHED_OVERHEAD_TRACE
 306  
 307 -extern DEFINE_PER_CPU(atomic_t, irq_fired_count);
 308 -
 309 -static inline void ft_irq_fired(void)
 310 -{
 311 -	/* Only called with preemptions disabled.  */
 312 -	atomic_inc(&__get_cpu_var(irq_fired_count));
 313 -}
 314 -
 315 +void ft_irq_fired(void);
 316  
 317  #else
 318  
 319 diff --git a/kernel/sched.c b/kernel/sched.c
 320 index 2229d0d..71463c8 100644
 321 --- a/kernel/sched.c
 322 +++ b/kernel/sched.c
 323 @@ -2597,8 +2597,12 @@ void scheduler_ipi(void)
 324  	struct rq *rq = this_rq();
 325  	struct task_struct *list = xchg(&rq->wake_list, NULL);
 326  
 327 -	if (!list)
 328 +	if (!list) {
 329 +		/* If we don't call irq_enter(), we need to trigger the IRQ
 330 +		 * tracing manually. */
 331 +		ft_irq_fired();
 332  		return;
 333 +	}
 334  
 335  	/*
 336  	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
 337 @@ -4403,14 +4407,20 @@ litmus_need_resched_nonpreemptible:
 338  		raw_spin_unlock_irq(&rq->lock);
 339  	}
 340  
 341 +	TS_SCHED2_START(prev);
 342  	sched_trace_task_switch_to(current);
 343  
 344  	post_schedule(rq);
 345  
 346 -	if (sched_state_validate_switch())
 347 +	if (sched_state_validate_switch()) {
 348 +		TS_SCHED2_END(prev);
 349  		goto litmus_need_resched_nonpreemptible;
 350 +	}
 351  
 352  	preempt_enable_no_resched();
 353 +
 354 +	TS_SCHED2_END(prev);
 355 +
 356  	if (need_resched())
 357  		goto need_resched;
 358  
 359 @@ -4684,17 +4694,6 @@ void complete_all(struct completion *x)
 360  }
 361  EXPORT_SYMBOL(complete_all);
 362  
 363 -void complete_n(struct completion *x, int n)
 364 -{
 365 -	unsigned long flags;
 366 -
 367 -	spin_lock_irqsave(&x->wait.lock, flags);
 368 -	x->done += n;
 369 -	__wake_up_common(&x->wait, TASK_NORMAL, n, 0, NULL);
 370 -	spin_unlock_irqrestore(&x->wait.lock, flags);
 371 -}
 372 -EXPORT_SYMBOL(complete_n);
 373 -
 374  static inline long __sched
 375  do_wait_for_common(struct completion *x, long timeout, int state)
 376  {
 377 diff --git a/kernel/softirq.c b/kernel/softirq.c
 378 index fca82c3..2f2df08 100644
 379 --- a/kernel/softirq.c
 380 +++ b/kernel/softirq.c
 381 @@ -211,6 +211,9 @@ asmlinkage void __do_softirq(void)
 382  	int max_restart = MAX_SOFTIRQ_RESTART;
 383  	int cpu;
 384  
 385 +	/* Mark Feather-Trace samples as "disturbed". */
 386 +	ft_irq_fired();
 387 +
 388  	pending = local_softirq_pending();
 389  	account_system_vtime(current);
 390  
 391 diff --git a/litmus/ctrldev.c b/litmus/ctrldev.c
 392 index 9969ab1..41919b2 100644
 393 --- a/litmus/ctrldev.c
 394 +++ b/litmus/ctrldev.c
 395 @@ -133,6 +133,17 @@ static int __init init_litmus_ctrl_dev(void)
 396  
 397  	BUILD_BUG_ON(sizeof(struct control_page) > PAGE_SIZE);
 398  
 399 +	BUILD_BUG_ON(sizeof(union np_flag) != sizeof(uint64_t));
 400 +
 401 +	BUILD_BUG_ON(offsetof(struct control_page, sched.raw)
 402 +		     != LITMUS_CP_OFFSET_SCHED);
 403 +	BUILD_BUG_ON(offsetof(struct control_page, irq_count)
 404 +		     != LITMUS_CP_OFFSET_IRQ_COUNT);
 405 +	BUILD_BUG_ON(offsetof(struct control_page, ts_syscall_start)
 406 +		     != LITMUS_CP_OFFSET_TS_SC_START);
 407 +	BUILD_BUG_ON(offsetof(struct control_page, irq_syscall_start)
 408 +		     != LITMUS_CP_OFFSET_IRQ_SC_START);
 409 +
 410  	printk("Initializing LITMUS^RT control device.\n");
 411  	err = misc_register(&litmus_ctrl_dev);
 412  	if (err)
 413 diff --git a/litmus/fdso.c b/litmus/fdso.c
 414 index cd85b9c..57813c0 100644
 415 --- a/litmus/fdso.c
 416 +++ b/litmus/fdso.c
 417 @@ -27,6 +27,7 @@ static const struct fdso_ops* fdso_ops[] = {
 418  	&generic_lock_ops, /* MPCP_VS_SEM */
 419  	&generic_lock_ops, /* DPCP_SEM */
 420  	&generic_lock_ops, /* PCP_SEM */
 421 +	&generic_lock_ops, /* DFLP_SEM */
 422  };
 423  
 424  static int fdso_create(void** obj_ref, obj_type_t type, void* __user config)
 425 @@ -166,14 +167,33 @@ static int put_od_entry(struct od_table_entry* od)
 426  	return 0;
 427  }
 428  
 429 +static long close_od_entry(struct od_table_entry *od)
 430 +{
 431 +	long ret;
 432 +
 433 +	/* Give the class a chance to reject the close. */
 434 +	ret = fdso_close(od);
 435 +	if (ret == 0)
 436 +		ret = put_od_entry(od);
 437 +
 438 +	return ret;
 439 +}
 440 +
 441 +#include <litmus/litmus.h>
 442 +
 443  void exit_od_table(struct task_struct* t)
 444  {
 445  	int i;
 446  
 447 +	if (is_realtime(t))
 448 +		printk("[%s] %s/%d/rt=%d (current: %s/%d/rt=%d)\n",
 449 +		       __FUNCTION__,
 450 +		       t->comm, t->pid, is_realtime(t),
 451 +		       current->comm, current->pid, is_realtime(current));
 452  	if (t->od_table) {
 453  		for (i = 0; i < MAX_OBJECT_DESCRIPTORS; i++)
 454  			if (t->od_table[i].used)
 455 -				put_od_entry(t->od_table + i);
 456 +				close_od_entry(t->od_table + i);
 457  		kfree(t->od_table);
 458  		t->od_table = NULL;
 459  	}
 460 @@ -287,11 +307,7 @@ asmlinkage long sys_od_close(int od)
 461  		return ret;
 462  
 463  
 464 -	/* give the class a chance to reject the close
 465 -	 */
 466 -	ret = fdso_close(t->od_table + od);
 467 -	if (ret == 0)
 468 -		ret = put_od_entry(t->od_table + od);
 469 +	ret = close_od_entry(t->od_table + od);
 470  
 471  	return ret;
 472  }
 473 diff --git a/litmus/fp_common.c b/litmus/fp_common.c
 474 index 31fc2db..964a472 100644
 475 --- a/litmus/fp_common.c
 476 +++ b/litmus/fp_common.c
 477 @@ -15,7 +15,7 @@
 478  #include <litmus/fp_common.h>
 479  
 480  /* fp_higher_prio -  returns true if first has a higher static priority
 481 - *                   than second. Deadline ties are broken by PID.
 482 + *                   than second. Ties are broken by PID.
 483   *
 484   * both first and second may be NULL
 485   */
 486 @@ -37,6 +37,9 @@ int fp_higher_prio(struct task_struct* first,
 487  	if (!first || !second)
 488  		return first && !second;
 489  
 490 +	if (!is_realtime(second_task))
 491 +		return 1;
 492 +
 493  #ifdef CONFIG_LITMUS_LOCKING
 494  
 495  	/* Check for inherited priorities. Change task
 496 @@ -51,33 +54,30 @@ int fp_higher_prio(struct task_struct* first,
 497  	 */
 498  	if (unlikely(is_priority_boosted(first_task))) {
 499  		/* first_task is boosted, how about second_task? */
 500 -		if (!is_priority_boosted(second_task) ||
 501 -		    lt_before(get_boost_start(first_task),
 502 -			      get_boost_start(second_task)))
 503 -			return 1;
 504 +		if (is_priority_boosted(second_task))
 505 +			/* break by priority point */
 506 +			return lt_before(get_boost_start(first_task),
 507 +					 get_boost_start(second_task));
 508  		else
 509 -			return 0;
 510 +			/* priority boosting wins. */
 511 +			return 1;
 512  	} else if (unlikely(is_priority_boosted(second_task)))
 513  		/* second_task is boosted, first is not*/
 514  		return 0;
 515  
 516  #endif
 517  
 518 +	/* Comparisons to itself are not expected; priority inheritance
 519 +	 * should also not cause this to happen. */
 520 +	BUG_ON(first_task == second_task);
 521  
 522 -	return !is_realtime(second_task)  ||
 523 -
 524 -		get_priority(first_task) < get_priority(second_task) ||
 525 -
 526 -		/* Break by PID.
 527 -		 */
 528 -		(get_priority(first_task) == get_priority(second_task) &&
 529 -	        (first_task->pid < second_task->pid ||
 530 -
 531 -		/* If the PIDs are the same then the task with the inherited
 532 -		 * priority wins.
 533 -		 */
 534 -		(first_task->pid == second_task->pid &&
 535 -		 !second->rt_param.inh_task)));
 536 +	if (get_priority(first_task) < get_priority(second_task))
 537 +		return 1;
 538 +	else if (get_priority(first_task) == get_priority(second_task))
 539 +		/* Break by PID. */
 540 +		return first_task->pid < second_task->pid;
 541 +	else
 542 +		return 0;
 543  }
 544  
 545  int fp_ready_order(struct bheap_node* a, struct bheap_node* b)
 546 diff --git a/litmus/ftdev.c b/litmus/ftdev.c
 547 index 06fcf4c..99bc39f 100644
 548 --- a/litmus/ftdev.c
 549 +++ b/litmus/ftdev.c
 550 @@ -230,13 +230,20 @@ static ssize_t ftdev_read(struct file *filp,
 551  			 * here with copied data because that data would get
 552  			 * lost if the task is interrupted (e.g., killed).
 553  			 */
 554 +			mutex_unlock(&ftdm->lock);
 555  			set_current_state(TASK_INTERRUPTIBLE);
 556 +
 557  			schedule_timeout(50);
 558 +
 559  			if (signal_pending(current)) {
 560  				if (err == 0)
 561  					/* nothing read yet, signal problem */
 562  					err = -ERESTARTSYS;
 563 -				break;
 564 +				goto out;
 565 +			}
 566 +			if (mutex_lock_interruptible(&ftdm->lock)) {
 567 +				err = -ERESTARTSYS;
 568 +				goto out;
 569  			}
 570  		} else if (copied < 0) {
 571  			/* page fault */
 572 diff --git a/litmus/litmus.c b/litmus/litmus.c
 573 index 8138432..a8203c2 100644
 574 --- a/litmus/litmus.c
 575 +++ b/litmus/litmus.c
 576 @@ -434,11 +434,11 @@ int switch_sched_plugin(struct sched_plugin* plugin)
 577  			goto out;
 578  		ret = plugin->activate_plugin();
 579  		if (0 != ret) {
 580 -			printk(KERN_INFO "Can't activate %s (%d).\n",
 581 +			printk("Can't activate %s (%d).\n",
 582  			       plugin->plugin_name, ret);
 583  			plugin = &linux_sched_plugin;
 584  		}
 585 -		printk(KERN_INFO "Switching to LITMUS^RT plugin %s.\n", plugin->plugin_name);
 586 +		printk("Switching to LITMUS^RT plugin %s.\n", plugin->plugin_name);
 587  		litmus = plugin;
 588  	} else
 589  		ret = -EBUSY;
 590 @@ -499,8 +499,17 @@ void exit_litmus(struct task_struct *dead_tsk)
 591  	}
 592  
 593  	/* main cleanup only for RT tasks */
 594 -	if (is_realtime(dead_tsk))
 595 +	if (is_realtime(dead_tsk)) {
 596 +		preempt_disable();
 597 +		printk("Tearing down real-time task %s/%d...\n",
 598 +		       dead_tsk->comm, dead_tsk->pid);
 599  		litmus_exit_task(dead_tsk);
 600 +		printk("Done with LITMUS^RT cleanup for %s/%d. "
 601 +		       "(%d remaning RT tasks)\n",
 602 +		       dead_tsk->comm, dead_tsk->pid,
 603 +		       atomic_read(&rt_task_count));
 604 +		preempt_enable();
 605 +	}
 606  }
 607  
 608  
 609 @@ -536,8 +545,6 @@ static int __init _init_litmus(void)
 610  	 */
 611  	printk("Starting LITMUS^RT kernel\n");
 612  
 613 -	BUILD_BUG_ON(sizeof(union np_flag) != sizeof(uint32_t));
 614 -
 615  	register_sched_plugin(&linux_sched_plugin);
 616  
 617  	bheap_node_cache    = KMEM_CACHE(bheap_node, SLAB_PANIC);
 618 diff --git a/litmus/locking.c b/litmus/locking.c
 619 index ca5a073..84a1d83 100644
 620 --- a/litmus/locking.c
 621 +++ b/litmus/locking.c
 622 @@ -1,3 +1,5 @@
 623 +#include <linux/sched.h>
 624 +#include <litmus/litmus.h>
 625  #include <litmus/fdso.h>
 626  
 627  #ifdef CONFIG_LITMUS_LOCKING
 628 @@ -70,6 +72,10 @@ asmlinkage long sys_litmus_lock(int lock_od)
 629  	struct od_table_entry* entry;
 630  	struct litmus_lock* l;
 631  
 632 +	TS_SYSCALL_IN_START;
 633 +
 634 +	TS_SYSCALL_IN_END;
 635 +
 636  	TS_LOCK_START;
 637  
 638  	entry = get_entry_for_od(lock_od);
 639 @@ -83,6 +89,8 @@ asmlinkage long sys_litmus_lock(int lock_od)
 640  	 * this into account when computing overheads. */
 641  	TS_LOCK_END;
 642  
 643 +	TS_SYSCALL_OUT_START;
 644 +
 645  	return err;
 646  }
 647  
 648 @@ -92,6 +100,10 @@ asmlinkage long sys_litmus_unlock(int lock_od)
 649  	struct od_table_entry* entry;
 650  	struct litmus_lock* l;
 651  
 652 +	TS_SYSCALL_IN_START;
 653 +
 654 +	TS_SYSCALL_IN_END;
 655 +
 656  	TS_UNLOCK_START;
 657  
 658  	entry = get_entry_for_od(lock_od);
 659 @@ -105,6 +117,8 @@ asmlinkage long sys_litmus_unlock(int lock_od)
 660  	 * account when computing overheads. */
 661  	TS_UNLOCK_END;
 662  
 663 +	TS_SYSCALL_OUT_START;
 664 +
 665  	return err;
 666  }
 667  
 668 @@ -154,6 +168,7 @@ out:
 669  	return passed;
 670  }
 671  
 672 +
 673  #else
 674  
 675  struct fdso_ops generic_lock_ops = {};
 676 diff --git a/litmus/preempt.c b/litmus/preempt.c
 677 index 5704d0b..6be2f26 100644
 678 --- a/litmus/preempt.c
 679 +++ b/litmus/preempt.c
 680 @@ -2,6 +2,7 @@
 681  
 682  #include <litmus/litmus.h>
 683  #include <litmus/preempt.h>
 684 +#include <litmus/trace.h>
 685  
 686  /* The rescheduling state of each processor.
 687   */
 688 @@ -47,6 +48,7 @@ void sched_state_ipi(void)
 689  		set_tsk_need_resched(current);
 690  		TRACE_STATE("IPI -> set_tsk_need_resched(%s/%d)\n",
 691  			    current->comm, current->pid);
 692 +		TS_SEND_RESCHED_END;
 693  	} else {
 694  		/* ignore */
 695  		TRACE_STATE("ignoring IPI in state %x (%s)\n",
 696 @@ -85,8 +87,10 @@ void litmus_reschedule(int cpu)
 697  	if (scheduled_transition_ok) {
 698  		if (smp_processor_id() == cpu)
 699  			set_tsk_need_resched(current);
 700 -		else
 701 +		else {
 702 +			TS_SEND_RESCHED_START(cpu);
 703  			smp_send_reschedule(cpu);
 704 +		}
 705  	}
 706  
 707  	TRACE_STATE("%s picked-ok:%d sched-ok:%d\n",
 708 diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c
 709 index d0b7966..1683d38 100644
 710 --- a/litmus/rt_domain.c
 711 +++ b/litmus/rt_domain.c
 712 @@ -331,12 +331,7 @@ void __add_release_on(rt_domain_t* rt, struct task_struct *task,
 713  	list_add(&tsk_rt(task)->list, &rt->tobe_released);
 714  	task->rt_param.domain = rt;
 715  
 716 -	/* start release timer */
 717 -	TS_SCHED2_START(task);
 718 -
 719  	arm_release_timer_on(rt, target_cpu);
 720 -
 721 -	TS_SCHED2_END(task);
 722  }
 723  #endif
 724  
 725 @@ -349,11 +344,6 @@ void __add_release(rt_domain_t* rt, struct task_struct *task)
 726  	list_add(&tsk_rt(task)->list, &rt->tobe_released);
 727  	task->rt_param.domain = rt;
 728  
 729 -	/* start release timer */
 730 -	TS_SCHED2_START(task);
 731 -
 732  	arm_release_timer(rt);
 733 -
 734 -	TS_SCHED2_END(task);
 735  }
 736  
 737 diff --git a/litmus/sched_litmus.c b/litmus/sched_litmus.c
 738 index 5a15ce9..1de8873 100644
 739 --- a/litmus/sched_litmus.c
 740 +++ b/litmus/sched_litmus.c
 741 @@ -194,6 +194,9 @@ static void dequeue_task_litmus(struct rq *rq, struct task_struct *p,
 742  
 743  static void yield_task_litmus(struct rq *rq)
 744  {
 745 +	TS_SYSCALL_IN_START;
 746 +	TS_SYSCALL_IN_END;
 747 +
 748  	BUG_ON(rq->curr != current);
 749  	/* sched_yield() is called to trigger delayed preemptions.
 750  	 * Thus, mark the current task as needing to be rescheduled.
 751 @@ -202,6 +205,8 @@ static void yield_task_litmus(struct rq *rq)
 752  	 */
 753  	clear_exit_np(current);
 754  	litmus_reschedule_local();
 755 +
 756 +	TS_SYSCALL_OUT_START;
 757  }
 758  
 759  /* Plugins are responsible for this.
 760 diff --git a/litmus/sched_pfp.c b/litmus/sched_pfp.c
 761 index 62be699..5a301e7 100644
 762 --- a/litmus/sched_pfp.c
 763 +++ b/litmus/sched_pfp.c
 764 @@ -55,7 +55,7 @@ static void preempt(pfp_domain_t *pfp)
 765  
 766  static unsigned int priority_index(struct task_struct* t)
 767  {
 768 -#ifdef CONFIG_LOCKING
 769 +#ifdef CONFIG_LITMUS_LOCKING
 770  	if (unlikely(t->rt_param.inh_task))
 771  		/* use effective priority */
 772  		t = t->rt_param.inh_task;
 773 @@ -76,23 +76,31 @@ static void pfp_release_jobs(rt_domain_t* rt, struct bheap* tasks)
 774  	struct task_struct* t;
 775  	struct bheap_node* hn;
 776  
 777 -	raw_spin_lock_irqsave(&pfp->slock, flags);
 778 -
 779  	while (!bheap_empty(tasks)) {
 780 +		raw_spin_lock_irqsave(&pfp->slock, flags);
 781 +
 782  		hn = bheap_take(fp_ready_order, tasks);
 783  		t = bheap2task(hn);
 784  		TRACE_TASK(t, "released (part:%d prio:%d)\n",
 785  			   get_partition(t), get_priority(t));
 786  		fp_prio_add(&pfp->ready_queue, t, priority_index(t));
 787 -	}
 788  
 789 -	/* do we need to preempt? */
 790 -	if (fp_higher_prio(fp_prio_peek(&pfp->ready_queue), pfp->scheduled)) {
 791 -		TRACE_CUR("preempted by new release\n");
 792 -		preempt(pfp);
 793 +		if (bheap_empty(tasks)) {
 794 +			/* do we need to preempt? */
 795 +			if (fp_higher_prio(fp_prio_peek(&pfp->ready_queue), pfp->scheduled)) {
 796 +				TRACE_CUR("preempted by new release\n");
 797 +				preempt(pfp);
 798 +			}
 799 +		}
 800 +
 801 +		raw_spin_unlock_irqrestore(&pfp->slock, flags);
 802  	}
 803 +}
 804  
 805 -	raw_spin_unlock_irqrestore(&pfp->slock, flags);
 806 +static void pfp_preempt_check(pfp_domain_t *pfp)
 807 +{
 808 +	if (fp_higher_prio(fp_prio_peek(&pfp->ready_queue), pfp->scheduled))
 809 +		preempt(pfp);
 810  }
 811  
 812  static void pfp_domain_init(pfp_domain_t* pfp,
 813 @@ -106,8 +114,7 @@ static void pfp_domain_init(pfp_domain_t* pfp,
 814  
 815  static void requeue(struct task_struct* t, pfp_domain_t *pfp)
 816  {
 817 -	if (t->state != TASK_RUNNING)
 818 -		TRACE_TASK(t, "requeue: !TASK_RUNNING\n");
 819 +	BUG_ON(!is_running(t));
 820  
 821  	set_rt_flags(t, RT_F_RUNNING);
 822  	if (is_released(t, litmus_clock()))
 823 @@ -123,6 +130,8 @@ static void job_completion(struct task_struct* t, int forced)
 824  
 825  	set_rt_flags(t, RT_F_SLEEP);
 826  	prepare_for_next_period(t);
 827 +	if (is_released(t, litmus_clock()))
 828 +		sched_trace_task_release(t);
 829  }
 830  
 831  static void pfp_tick(struct task_struct *t)
 832 @@ -217,6 +226,26 @@ static struct task_struct* pfp_schedule(struct task_struct * prev)
 833  		if (pfp->scheduled && !blocks  && !migrate)
 834  			requeue(pfp->scheduled, pfp);
 835  		next = fp_prio_take(&pfp->ready_queue);
 836 +		if (next == prev) {
 837 +			struct task_struct *t = fp_prio_peek(&pfp->ready_queue);
 838 +			TRACE_TASK(next, "next==prev sleep=%d oot=%d np=%d preempt=%d migrate=%d "
 839 +				   "boost=%d empty=%d prio-idx=%u prio=%u\n",
 840 +				   sleep, out_of_time, np, preempt, migrate,
 841 +				   is_priority_boosted(next),
 842 +				   t == NULL,
 843 +				   priority_index(next),
 844 +				   get_priority(next));
 845 +			if (t)
 846 +				TRACE_TASK(t, "waiter boost=%d prio-idx=%u prio=%u\n",
 847 +					   is_priority_boosted(t),
 848 +					   priority_index(t),
 849 +					   get_priority(t));
 850 +		}
 851 +		/* If preempt is set, we should not see the same task again. */
 852 +		BUG_ON(preempt && next == prev);
 853 +		/* Similarly, if preempt is set, then next may not be NULL,
 854 +		 * unless it's a migration. */
 855 +		BUG_ON(preempt && !migrate && next == NULL);
 856  	} else
 857  		/* Only override Linux scheduler if we have a real-time task
 858  		 * scheduled that needs to continue.
 859 @@ -291,7 +320,7 @@ static void pfp_task_new(struct task_struct * t, int on_rq, int running)
 860  	} else {
 861  		requeue(t, pfp);
 862  		/* maybe we have to reschedule */
 863 -		preempt(pfp);
 864 +		pfp_preempt_check(pfp);
 865  	}
 866  	raw_spin_unlock_irqrestore(&pfp->slock, flags);
 867  }
 868 @@ -337,8 +366,10 @@ static void pfp_task_wake_up(struct task_struct *task)
 869  	 * and won. Also, don't requeue if it is still queued, which can
 870  	 * happen under the DPCP due wake-ups racing with migrations.
 871  	 */
 872 -	if (pfp->scheduled != task)
 873 +	if (pfp->scheduled != task) {
 874  		requeue(task, pfp);
 875 +		pfp_preempt_check(pfp);
 876 +	}
 877  
 878  out_unlock:
 879  	raw_spin_unlock_irqrestore(&pfp->slock, flags);
 880 @@ -371,13 +402,24 @@ static void pfp_task_exit(struct task_struct * t)
 881  	rt_domain_t*		dom;
 882  
 883  	raw_spin_lock_irqsave(&pfp->slock, flags);
 884 +
 885 +	printk(KERN_ERR "[%s] %s/%d is exiting (part:%d / cpu:%d).\n",
 886 +	       __FUNCTION__, t->comm, t->pid, get_partition(t),
 887 +	       smp_processor_id());
 888 +
 889 +	if (t != current) {
 890 +		/* This currently doesn't work. */
 891 +		printk(KERN_ERR "[%s] task %s/%d is not current!\n",
 892 +		       __FUNCTION__, t->comm, t->pid);
 893 +	}
 894  	if (is_queued(t)) {
 895 -		BUG(); /* This currently doesn't work. */
 896 -		/* dequeue */
 897 -		dom  = task_dom(t);
 898 -		remove(dom, t);
 899 +		/* This currently doesn't work. */
 900 +		printk(KERN_ERR "[%s] task %s/%d is still queued!\n",
 901 +		       __FUNCTION__, t->comm, t->pid);
 902  	}
 903  	if (pfp->scheduled == t) {
 904 +		printk(KERN_ERR "[%s] task %s/%d is scheduled!\n",
 905 +		       __FUNCTION__, t->comm, t->pid);
 906  		pfp->scheduled = NULL;
 907  		preempt(pfp);
 908  	}
 909 @@ -455,17 +497,10 @@ static void boost_priority(struct task_struct* t, lt_t priority_point)
 910  	/* tie-break by protocol-specific priority point */
 911  	tsk_rt(t)->boost_start_time = priority_point;
 912  
 913 -	if (pfp->scheduled != t) {
 914 -		/* holder may be queued: first stop queue changes */
 915 -		raw_spin_lock(&pfp->domain.release_lock);
 916 -		if (is_queued(t) &&
 917 -		    /* If it is queued, then we need to re-order. */
 918 -		    bheap_decrease(fp_ready_order, tsk_rt(t)->heap_node) &&
 919 -		    /* If we bubbled to the top, then we need to check for preemptions. */
 920 -		    fp_preemption_needed(&pfp->ready_queue, pfp->scheduled))
 921 -				preempt(pfp);
 922 -		raw_spin_unlock(&pfp->domain.release_lock);
 923 -	} /* else: nothing to do since the job is not queued while scheduled */
 924 +	/* Priority boosting currently only takes effect for already-scheduled
 925 +	 * tasks. This is sufficient since priority boosting only kicks in as
 926 +	 * part of lock acquisitions. */
 927 +	BUG_ON(pfp->scheduled != t);
 928  
 929  	raw_spin_unlock_irqrestore(&pfp->slock, flags);
 930  }
 931 @@ -1521,6 +1556,211 @@ static struct litmus_lock* pfp_new_dpcp(int on_cpu)
 932  }
 933  
 934  
 935 +/* ******************** DFLP support ********************** */
 936 +
 937 +struct dflp_semaphore {
 938 +	struct litmus_lock litmus_lock;
 939 +
 940 +	/* current resource holder */
 941 +	struct task_struct *owner;
 942 +	int owner_cpu;
 943 +
 944 +	/* FIFO queue of waiting tasks */
 945 +	wait_queue_head_t wait;
 946 +
 947 +	/* where is the resource assigned to */
 948 +	int on_cpu;
 949 +};
 950 +
 951 +static inline struct dflp_semaphore* dflp_from_lock(struct litmus_lock* lock)
 952 +{
 953 +	return container_of(lock, struct dflp_semaphore, litmus_lock);
 954 +}
 955 +
 956 +int pfp_dflp_lock(struct litmus_lock* l)
 957 +{
 958 +	struct task_struct* t = current;
 959 +	struct dflp_semaphore *sem = dflp_from_lock(l);
 960 +	int from  = get_partition(t);
 961 +	int to    = sem->on_cpu;
 962 +	unsigned long flags;
 963 +	wait_queue_t wait;
 964 +	lt_t time_of_request;
 965 +
 966 +	if (!is_realtime(t))
 967 +		return -EPERM;
 968 +
 969 +	preempt_disable();
 970 +
 971 +	/* tie-break by this point in time */
 972 +	time_of_request = litmus_clock();
 973 +
 974 +	/* Priority-boost ourself *before* we suspend so that
 975 +	 * our priority is boosted when we resume. */
 976 +	boost_priority(t, time_of_request);
 977 +
 978 +	pfp_migrate_to(to);
 979 +
 980 +	/* Now on the right CPU, preemptions still disabled. */
 981 +
 982 +	spin_lock_irqsave(&sem->wait.lock, flags);
 983 +
 984 +	if (sem->owner) {
 985 +		/* resource is not free => must suspend and wait */
 986 +
 987 +		init_waitqueue_entry(&wait, t);
 988 +
 989 +		/* FIXME: interruptible would be nice some day */
 990 +		set_task_state(t, TASK_UNINTERRUPTIBLE);
 991 +
 992 +		__add_wait_queue_tail_exclusive(&sem->wait, &wait);
 993 +
 994 +		TS_LOCK_SUSPEND;
 995 +
 996 +		/* release lock before sleeping */
 997 +		spin_unlock_irqrestore(&sem->wait.lock, flags);
 998 +
 999 +		/* We depend on the FIFO order.  Thus, we don't need to recheck
1000 +		 * when we wake up; we are guaranteed to have the lock since
1001 +		 * there is only one wake up per release.
1002 +		 */
1003 +
1004 +		preempt_enable_no_resched();
1005 +
1006 +		schedule();
1007 +
1008 +		preempt_disable();
1009 +
1010 +		TS_LOCK_RESUME;
1011 +
1012 +		/* Since we hold the lock, no other task will change
1013 +		 * ->owner. We can thus check it without acquiring the spin
1014 +		 * lock. */
1015 +		BUG_ON(sem->owner != t);
1016 +	} else {
1017 +		/* it's ours now */
1018 +		sem->owner = t;
1019 +
1020 +		spin_unlock_irqrestore(&sem->wait.lock, flags);
1021 +	}
1022 +
1023 +	sem->owner_cpu = from;
1024 +
1025 +	preempt_enable();
1026 +
1027 +	return 0;
1028 +}
1029 +
1030 +int pfp_dflp_unlock(struct litmus_lock* l)
1031 +{
1032 +	struct task_struct *t = current, *next;
1033 +	struct dflp_semaphore *sem = dflp_from_lock(l);
1034 +	int err = 0;
1035 +	int home;
1036 +	unsigned long flags;
1037 +
1038 +	preempt_disable();
1039 +
1040 +	spin_lock_irqsave(&sem->wait.lock, flags);
1041 +
1042 +	if (sem->owner != t) {
1043 +		err = -EINVAL;
1044 +		spin_unlock_irqrestore(&sem->wait.lock, flags);
1045 +		goto out;
1046 +	}
1047 +
1048 +	/* check if there are jobs waiting for this resource */
1049 +	next = __waitqueue_remove_first(&sem->wait);
1050 +	if (next) {
1051 +		/* next becomes the resouce holder */
1052 +		sem->owner = next;
1053 +
1054 +		/* Wake up next. The waiting job is already priority-boosted. */
1055 +		wake_up_process(next);
1056 +	} else
1057 +		/* resource becomes available */
1058 +		sem->owner = NULL;
1059 +
1060 +	home = sem->owner_cpu;
1061 +
1062 +	spin_unlock_irqrestore(&sem->wait.lock, flags);
1063 +
1064 +	/* we lose the benefit of priority boosting */
1065 +	unboost_priority(t);
1066 +
1067 +	pfp_migrate_to(home);
1068 +
1069 +out:
1070 +	preempt_enable();
1071 +
1072 +	return err;
1073 +}
1074 +
1075 +int pfp_dflp_open(struct litmus_lock* l, void* __user config)
1076 +{
1077 +	struct dflp_semaphore *sem = dflp_from_lock(l);
1078 +	int cpu;
1079 +
1080 +	if (get_user(cpu, (int*) config))
1081 +		return -EFAULT;
1082 +
1083 +	/* make sure the resource location matches */
1084 +	if (cpu != sem->on_cpu)
1085 +		return -EINVAL;
1086 +
1087 +	return 0;
1088 +}
1089 +
1090 +int pfp_dflp_close(struct litmus_lock* l)
1091 +{
1092 +	struct task_struct *t = current;
1093 +	struct dflp_semaphore *sem = dflp_from_lock(l);
1094 +	int owner = 0;
1095 +
1096 +	preempt_disable();
1097 +
1098 +	if (sem->on_cpu == smp_processor_id())
1099 +		owner = sem->owner == t;
1100 +
1101 +	preempt_enable();
1102 +
1103 +	if (owner)
1104 +		pfp_dflp_unlock(l);
1105 +
1106 +	return 0;
1107 +}
1108 +
1109 +void pfp_dflp_free(struct litmus_lock* lock)
1110 +{
1111 +	kfree(dflp_from_lock(lock));
1112 +}
1113 +
1114 +static struct litmus_lock_ops pfp_dflp_lock_ops = {
1115 +	.close  = pfp_dflp_close,
1116 +	.lock   = pfp_dflp_lock,
1117 +	.open	= pfp_dflp_open,
1118 +	.unlock = pfp_dflp_unlock,
1119 +	.deallocate = pfp_dflp_free,
1120 +};
1121 +
1122 +static struct litmus_lock* pfp_new_dflp(int on_cpu)
1123 +{
1124 +	struct dflp_semaphore* sem;
1125 +
1126 +	sem = kmalloc(sizeof(*sem), GFP_KERNEL);
1127 +	if (!sem)
1128 +		return NULL;
1129 +
1130 +	sem->litmus_lock.ops = &pfp_dflp_lock_ops;
1131 +	sem->owner_cpu = NO_CPU;
1132 +	sem->owner   = NULL;
1133 +	sem->on_cpu  = on_cpu;
1134 +	init_waitqueue_head(&sem->wait);
1135 +
1136 +	return &sem->litmus_lock;
1137 +}
1138 +
1139 +
1140  /* **** lock constructor **** */
1141  
1142  
1143 @@ -1575,6 +1815,21 @@ static long pfp_allocate_lock(struct litmus_lock **lock, int type,
1144  			err = -ENOMEM;
1145  		break;
1146  
1147 +	case DFLP_SEM:
1148 +		/* Distributed FIFO Locking Protocol */
1149 +		if (get_user(cpu, (int*) config))
1150 +			return -EFAULT;
1151 +
1152 +		if (!cpu_online(cpu))
1153 +			return -EINVAL;
1154 +
1155 +		*lock = pfp_new_dflp(cpu);
1156 +		if (*lock)
1157 +			err = 0;
1158 +		else
1159 +			err = -ENOMEM;
1160 +		break;
1161 +
1162  	case SRP_SEM:
1163  		/* Baker's Stack Resource Policy */
1164  		srp = allocate_srp_semaphore();
1165 diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c
1166 index b0c8126..8933e15 100644
1167 --- a/litmus/sched_psn_edf.c
1168 +++ b/litmus/sched_psn_edf.c
1169 @@ -133,6 +133,15 @@ static void unboost_priority(struct task_struct* t)
1170  
1171  #endif
1172  
1173 +static int psnedf_preempt_check(psnedf_domain_t *pedf)
1174 +{
1175 +	if (edf_preemption_needed(&pedf->domain, pedf->scheduled)) {
1176 +		preempt(pedf);
1177 +		return 1;
1178 +	} else
1179 +		return 0;
1180 +}
1181 +
1182  /* This check is trivial in partioned systems as we only have to consider
1183   * the CPU of the partition.
1184   */
1185 @@ -143,11 +152,7 @@ static int psnedf_check_resched(rt_domain_t *edf)
1186  	/* because this is a callback from rt_domain_t we already hold
1187  	 * the necessary lock for the ready queue
1188  	 */
1189 -	if (edf_preemption_needed(edf, pedf->scheduled)) {
1190 -		preempt(pedf);
1191 -		return 1;
1192 -	} else
1193 -		return 0;
1194 +	return psnedf_preempt_check(pedf);
1195  }
1196  
1197  static void job_completion(struct task_struct* t, int forced)
1198 @@ -299,7 +304,7 @@ static void psnedf_task_new(struct task_struct * t, int on_rq, int running)
1199  	} else {
1200  		requeue(t, edf);
1201  		/* maybe we have to reschedule */
1202 -		preempt(pedf);
1203 +		psnedf_preempt_check(pedf);
1204  	}
1205  	raw_spin_unlock_irqrestore(&pedf->slock, flags);
1206  }
1207 @@ -335,8 +340,10 @@ static void psnedf_task_wake_up(struct task_struct *task)
1208  	 * de-scheduling the task, i.e., wake_up() raced with schedule()
1209  	 * and won.
1210  	 */
1211 -	if (pedf->scheduled != task)
1212 +	if (pedf->scheduled != task) {
1213  		requeue(task, edf);
1214 +		psnedf_preempt_check(pedf);
1215 +	}
1216  
1217  	raw_spin_unlock_irqrestore(&pedf->slock, flags);
1218  	TRACE_TASK(task, "wake up done\n");
1219 diff --git a/litmus/sync.c b/litmus/sync.c
1220 index bf75fde..873b3ff 100644
1221 --- a/litmus/sync.c
1222 +++ b/litmus/sync.c
1223 @@ -16,63 +16,106 @@
1224  
1225  #include <litmus/sched_trace.h>
1226  
1227 -static DECLARE_COMPLETION(ts_release);
1228 +struct ts_release_wait {
1229 +	struct list_head list;
1230 +	struct completion completion;
1231 +	lt_t ts_release_time;
1232 +};
1233 +
1234 +#define DECLARE_TS_RELEASE_WAIT(symb)					\
1235 +	struct ts_release_wait symb =					\
1236 +	{								\
1237 +		LIST_HEAD_INIT(symb.list),				\
1238 +		COMPLETION_INITIALIZER_ONSTACK(symb.completion),	\
1239 +		0							\
1240 +	}
1241 +
1242 +static LIST_HEAD(task_release_list);
1243 +static DEFINE_MUTEX(task_release_lock);
1244  
1245  static long do_wait_for_ts_release(void)
1246  {
1247 -	long ret = 0;
1248 +	DECLARE_TS_RELEASE_WAIT(wait);
1249 +
1250 +	long ret = -ERESTARTSYS;
1251 +
1252 +	if (mutex_lock_interruptible(&task_release_lock))
1253 +		goto out;
1254 +
1255 +	list_add(&wait.list, &task_release_list);
1256  
1257 -	/* If the interruption races with a release, the completion object
1258 -	 * may have a non-zero counter. To avoid this problem, this should
1259 -	 * be replaced by wait_for_completion().
1260 -	 *
1261 -	 * For debugging purposes, this is interruptible for now.
1262 -	 */
1263 -	ret = wait_for_completion_interruptible(&ts_release);
1264 +	mutex_unlock(&task_release_lock);
1265  
1266 +	/* We are enqueued, now we wait for someone to wake us up. */
1267 +	ret = wait_for_completion_interruptible(&wait.completion);
1268 +
1269 +	if (!ret) {
1270 +		/* Completion succeeded, setup release. */
1271 +		litmus->release_at(current, wait.ts_release_time
1272 +				   + current->rt_param.task_params.phase
1273 +				   - current->rt_param.task_params.period);
1274 +		/* trigger advance to next job release at the programmed time */
1275 +		ret = complete_job();
1276 +	} else {
1277 +		/* We were interrupted, must cleanup list. */
1278 +		mutex_lock(&task_release_lock);
1279 +		if (!wait.completion.done)
1280 +			list_del(&wait.list);
1281 +		mutex_unlock(&task_release_lock);
1282 +	}
1283 +
1284 +out:
1285  	return ret;
1286  }
1287  
1288  int count_tasks_waiting_for_release(void)
1289  {
1290 -	unsigned long flags;
1291  	int task_count = 0;
1292  	struct list_head *pos;
1293  
1294 -	spin_lock_irqsave(&ts_release.wait.lock, flags);
1295 -	list_for_each(pos, &ts_release.wait.task_list) {
1296 +	mutex_lock(&task_release_lock);
1297 +
1298 +	list_for_each(pos, &task_release_list) {
1299  		task_count++;
1300  	}
1301 -	spin_unlock_irqrestore(&ts_release.wait.lock, flags);
1302 +
1303 +	mutex_unlock(&task_release_lock);
1304 +
1305  
1306  	return task_count;
1307  }
1308  
1309  static long do_release_ts(lt_t start)
1310  {
1311 -	int  task_count = 0;
1312 -	unsigned long flags;
1313 +	long  task_count = 0;
1314 +
1315  	struct list_head	*pos;
1316 -	struct task_struct 	*t;
1317 +	struct ts_release_wait	*wait;
1318  
1319 +	if (mutex_lock_interruptible(&task_release_lock)) {
1320 +		task_count = -ERESTARTSYS;
1321 +		goto out;
1322 +	}
1323  
1324 -	spin_lock_irqsave(&ts_release.wait.lock, flags);
1325  	TRACE("<<<<<< synchronous task system release >>>>>>\n");
1326 -
1327  	sched_trace_sys_release(&start);
1328 -	list_for_each(pos, &ts_release.wait.task_list) {
1329 -		t = (struct task_struct*) list_entry(pos,
1330 -						     struct __wait_queue,
1331 -						     task_list)->private;
1332 +
1333 +	task_count = 0;
1334 +	list_for_each(pos, &task_release_list) {
1335 +		wait = (struct ts_release_wait*)
1336 +			list_entry(pos, struct ts_release_wait, list);
1337 +
1338  		task_count++;
1339 -		litmus->release_at(t, start + t->rt_param.task_params.phase);
1340 -		sched_trace_task_release(t);
1341 +		wait->ts_release_time = start;
1342 +		complete(&wait->completion);
1343  	}
1344  
1345 -	spin_unlock_irqrestore(&ts_release.wait.lock, flags);
1346 +	/* clear stale list */
1347 +	INIT_LIST_HEAD(&task_release_list);
1348  
1349 -	complete_n(&ts_release, task_count);
1350 +	mutex_unlock(&task_release_lock);
1351  
1352 +out:
1353  	return task_count;
1354  }
1355  
1356 @@ -88,17 +131,22 @@ asmlinkage long sys_wait_for_ts_release(void)
1357  	return ret;
1358  }
1359  
1360 +#define ONE_MS 1000000
1361  
1362  asmlinkage long sys_release_ts(lt_t __user *__delay)
1363  {
1364  	long ret;
1365  	lt_t delay;
1366 +	lt_t start_time;
1367  
1368  	/* FIXME: check capabilities... */
1369  
1370  	ret = copy_from_user(&delay, __delay, sizeof(delay));
1371 -	if (ret == 0)
1372 -		ret = do_release_ts(litmus_clock() + delay);
1373 +	if (ret == 0) {
1374 +		/* round up to next larger integral millisecond */
1375 +		start_time = ((litmus_clock() / ONE_MS) + 1) * ONE_MS;
1376 +		ret = do_release_ts(start_time + delay);
1377 +	}
1378  
1379  	return ret;
1380  }
1381 diff --git a/litmus/trace.c b/litmus/trace.c
1382 index 3c35c52..7dbb98e 100644
1383 --- a/litmus/trace.c
1384 +++ b/litmus/trace.c
1385 @@ -18,6 +18,15 @@ static unsigned int ts_seq_no = 0;
1386  
1387  DEFINE_PER_CPU(atomic_t, irq_fired_count);
1388  
1389 +void ft_irq_fired(void)
1390 +{
1391 +	/* Only called with preemptions disabled.  */
1392 +	atomic_inc(&__get_cpu_var(irq_fired_count));
1393 +
1394 +	if (has_control_page(current))
1395 +		get_control_page(current)->irq_count++;
1396 +}
1397 +
1398  static inline void clear_irq_fired(void)
1399  {
1400  	atomic_set(&__raw_get_cpu_var(irq_fired_count), 0);
1401 @@ -34,77 +43,119 @@ static inline unsigned int get_and_clear_irq_fired(void)
1402  	return atomic_xchg(&__raw_get_cpu_var(irq_fired_count), 0);
1403  }
1404  
1405 -static inline void __save_irq_flags(struct timestamp *ts)
1406 +static inline void save_irq_flags(struct timestamp *ts, unsigned int irq_count)
1407  {
1408 -	unsigned int irq_count;
1409 -
1410 -	irq_count     = get_and_clear_irq_fired();
1411  	/* Store how many interrupts occurred. */
1412  	ts->irq_count = irq_count;
1413  	/* Extra flag because ts->irq_count overflows quickly. */
1414  	ts->irq_flag  = irq_count > 0;
1415 +
1416  }
1417  
1418 -static inline void __save_timestamp_cpu(unsigned long event,
1419 -					uint8_t type, uint8_t cpu)
1420 +static inline void write_timestamp(uint8_t event,
1421 +				   uint8_t type,
1422 +				   uint8_t cpu,
1423 +				   uint16_t pid_fragment,
1424 +				   unsigned int irq_count,
1425 +				   int record_irq,
1426 +				   int hide_irq,
1427 +				   uint64_t timestamp,
1428 +				   int record_timestamp)
1429  {
1430 +	unsigned long flags;
1431  	unsigned int seq_no;
1432  	struct timestamp *ts;
1433 +
1434 +	/* Avoid preemptions while recording the timestamp. This reduces the
1435 +	 * number of "out of order" timestamps in the stream and makes
1436 +	 * post-processing easier. */
1437 +
1438 +	local_irq_save(flags);
1439 +
1440  	seq_no = fetch_and_inc((int *) &ts_seq_no);
1441  	if (ft_buffer_start_write(trace_ts_buf, (void**)  &ts)) {
1442  		ts->event     = event;
1443  		ts->seq_no    = seq_no;
1444 -		ts->cpu       = cpu;
1445 +
1446  		ts->task_type = type;
1447 -		__save_irq_flags(ts);
1448 -		barrier();
1449 -		/* prevent re-ordering of ft_timestamp() */
1450 -		ts->timestamp = ft_timestamp();
1451 +		ts->pid	      = pid_fragment;
1452 +
1453 +		ts->cpu       = cpu;
1454 +
1455 +		if (record_irq)
1456 +			irq_count = get_and_clear_irq_fired();
1457 +
1458 +		save_irq_flags(ts, irq_count - hide_irq);
1459 +
1460 +		if (record_timestamp)
1461 +			timestamp = ft_timestamp();
1462 +
1463 +		ts->timestamp = timestamp;
1464  		ft_buffer_finish_write(trace_ts_buf, ts);
1465  	}
1466 +
1467 +	local_irq_restore(flags);
1468  }
1469  
1470  static void __add_timestamp_user(struct timestamp *pre_recorded)
1471  {
1472 +	unsigned long flags;
1473  	unsigned int seq_no;
1474  	struct timestamp *ts;
1475 +
1476 +
1477 +	local_irq_save(flags);
1478 +
1479  	seq_no = fetch_and_inc((int *) &ts_seq_no);
1480  	if (ft_buffer_start_write(trace_ts_buf, (void**)  &ts)) {
1481  		*ts = *pre_recorded;
1482  		ts->seq_no = seq_no;
1483 -		__save_irq_flags(ts);
1484 +		ts->cpu	   = raw_smp_processor_id();
1485 +	        save_irq_flags(ts, get_and_clear_irq_fired());
1486  		ft_buffer_finish_write(trace_ts_buf, ts);
1487  	}
1488 -}
1489  
1490 -static inline void __save_timestamp(unsigned long event,
1491 -				   uint8_t type)
1492 -{
1493 -	__save_timestamp_cpu(event, type, raw_smp_processor_id());
1494 +	local_irq_restore(flags);
1495  }
1496  
1497  feather_callback void save_timestamp(unsigned long event)
1498  {
1499 -	__save_timestamp(event, TSK_UNKNOWN);
1500 +	write_timestamp(event, TSK_UNKNOWN,
1501 +			raw_smp_processor_id(),
1502 +			current->pid,
1503 +			0, 1, 0,
1504 +			0, 1);
1505  }
1506  
1507  feather_callback void save_timestamp_def(unsigned long event,
1508  					 unsigned long type)
1509  {
1510 -	__save_timestamp(event, (uint8_t) type);
1511 +	write_timestamp(event, type,
1512 +			raw_smp_processor_id(),
1513 +			current->pid,
1514 +			0, 1, 0,
1515 +			0, 1);
1516  }
1517  
1518  feather_callback void save_timestamp_task(unsigned long event,
1519  					  unsigned long t_ptr)
1520  {
1521 -	int rt = is_realtime((struct task_struct *) t_ptr);
1522 -	__save_timestamp(event, rt ? TSK_RT : TSK_BE);
1523 +	struct task_struct *t = (struct task_struct *) t_ptr;
1524 +	int rt = is_realtime(t);
1525 +
1526 +	write_timestamp(event, rt ? TSK_RT : TSK_BE,
1527 +			raw_smp_processor_id(),
1528 +			t->pid,
1529 +			0, 1, 0,
1530 +			0, 1);
1531  }
1532  
1533  feather_callback void save_timestamp_cpu(unsigned long event,
1534  					 unsigned long cpu)
1535  {
1536 -	__save_timestamp_cpu(event, TSK_UNKNOWN, cpu);
1537 +	write_timestamp(event, TSK_UNKNOWN, cpu, current->pid,
1538 +			0, 1, 0,
1539 +			0, 1);
1540  }
1541  
1542  feather_callback void save_task_latency(unsigned long event,
1543 @@ -112,20 +163,44 @@ feather_callback void save_task_latency(unsigned long event,
1544  {
1545  	lt_t now = litmus_clock();
1546  	lt_t *when = (lt_t*) when_ptr;
1547 -	unsigned int seq_no;
1548 -	int cpu = raw_smp_processor_id();
1549 -	struct timestamp *ts;
1550  
1551 -	seq_no = fetch_and_inc((int *) &ts_seq_no);
1552 -	if (ft_buffer_start_write(trace_ts_buf, (void**)  &ts)) {
1553 -		ts->event     = event;
1554 -		ts->timestamp = now - *when;
1555 -		ts->seq_no    = seq_no;
1556 -		ts->cpu       = cpu;
1557 -		ts->task_type = TSK_RT;
1558 -		__save_irq_flags(ts);
1559 -		ft_buffer_finish_write(trace_ts_buf, ts);
1560 -	}
1561 +	write_timestamp(event, TSK_RT, raw_smp_processor_id(), 0,
1562 +			0, 1, 0,
1563 +			now - *when, 0);
1564 +}
1565 +
1566 +/* fake timestamp to user-reported time */
1567 +feather_callback void save_timestamp_time(unsigned long event,
1568 +			 unsigned long ptr)
1569 +{
1570 +	uint64_t* time = (uint64_t*) ptr;
1571 +
1572 +	write_timestamp(event, is_realtime(current) ? TSK_RT : TSK_BE,
1573 +			raw_smp_processor_id(), current->pid,
1574 +			0, 1, 0,
1575 +			*time, 0);
1576 +}
1577 +
1578 +/* Record user-reported IRQ count */
1579 +feather_callback void save_timestamp_irq(unsigned long event,
1580 +			unsigned long irq_counter_ptr)
1581 +{
1582 +	uint64_t* irqs = (uint64_t*) irq_counter_ptr;
1583 +
1584 +	write_timestamp(event, is_realtime(current) ? TSK_RT : TSK_BE,
1585 +			raw_smp_processor_id(), current->pid,
1586 +			*irqs, 0, 0,
1587 +			0, 1);
1588 +}
1589 +
1590 +/* Suppress one IRQ from the irq count. Used by TS_SEND_RESCHED_END, which is
1591 + * called from within an interrupt that is expected. */
1592 +feather_callback void save_timestamp_hide_irq(unsigned long event)
1593 +{
1594 +	write_timestamp(event, is_realtime(current) ? TSK_RT : TSK_BE,
1595 +			raw_smp_processor_id(), current->pid,
1596 +			0, 1, 1,
1597 +			0, 1);
1598  }
1599  
1600  /******************************************************************************/