attachment:feather-trace-patch-against-sched-deadline-v8.patch of Publications

   1 diff --git a/Makefile b/Makefile
   2 index cd11e88..90c4f57 100644
   3 --- a/Makefile
   4 +++ b/Makefile
   5 @@ -733,7 +733,7 @@ export mod_sign_cmd
   6  
   7  
   8  ifeq ($(KBUILD_EXTMOD),)
   9 -core-y		+= kernel/ mm/ fs/ ipc/ security/ crypto/ block/
  10 +core-y		+= kernel/ mm/ fs/ ipc/ security/ crypto/ block/ litmus/
  11  
  12  vmlinux-dirs	:= $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \
  13  		     $(core-y) $(core-m) $(drivers-y) $(drivers-m) \
  14 diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
  15 index 49d993c..8d27866 100644
  16 --- a/arch/arm/Kconfig
  17 +++ b/arch/arm/Kconfig
  18 @@ -2221,3 +2221,9 @@ source "crypto/Kconfig"
  19  source "lib/Kconfig"
  20  
  21  source "arch/arm/kvm/Kconfig"
  22 +
  23 +config ARCH_HAS_FEATHER_TRACE
  24 +	def_bool n
  25 +
  26 +source "litmus/Kconfig"
  27 +
  28 diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
  29 index 685692c..9487544 100644
  30 --- a/arch/x86/Kconfig
  31 +++ b/arch/x86/Kconfig
  32 @@ -2345,3 +2345,8 @@ source "crypto/Kconfig"
  33  source "arch/x86/kvm/Kconfig"
  34  
  35  source "lib/Kconfig"
  36 +
  37 +config ARCH_HAS_FEATHER_TRACE
  38 +	def_bool y
  39 +
  40 +source "litmus/Kconfig"
  41 diff --git a/arch/x86/include/asm/feather_trace.h b/arch/x86/include/asm/feather_trace.h
  42 new file mode 100644
  43 index 0000000..4fd3163
  44 --- /dev/null
  45 +++ b/arch/x86/include/asm/feather_trace.h
  46 @@ -0,0 +1,17 @@
  47 +#ifndef _ARCH_FEATHER_TRACE_H
  48 +#define _ARCH_FEATHER_TRACE_H
  49 +
  50 +#include <asm/msr.h>
  51 +
  52 +static inline unsigned long long ft_timestamp(void)
  53 +{
  54 +	return __native_read_tsc();
  55 +}
  56 +
  57 +#ifdef CONFIG_X86_32
  58 +#include "feather_trace_32.h"
  59 +#else
  60 +#include "feather_trace_64.h"
  61 +#endif
  62 +
  63 +#endif
  64 diff --git a/arch/x86/include/asm/feather_trace_32.h b/arch/x86/include/asm/feather_trace_32.h
  65 new file mode 100644
  66 index 0000000..75e81a9
  67 --- /dev/null
  68 +++ b/arch/x86/include/asm/feather_trace_32.h
  69 @@ -0,0 +1,115 @@
  70 +/* Copyright (c) 2007-2012 Björn Brandenburg, <bbb@mpi-sws.org>
  71 + *
  72 + * Permission is hereby granted, free of charge, to any person obtaining
  73 + * a copy of this software and associated documentation files (the
  74 + * "Software"), to deal in the Software without restriction, including
  75 + * without limitation the rights to use, copy, modify, merge, publish,
  76 + * distribute, sublicense, and/or sell copies of the Software, and to
  77 + * permit persons to whom the Software is furnished to do so, subject to
  78 + * the following conditions:
  79 + *
  80 + * The above copyright notice and this permission notice shall be
  81 + * included in all copies or substantial portions of the Software.
  82 + *
  83 + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  84 + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  85 + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  86 + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  87 + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  88 + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  89 + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  90 + * SOFTWARE.
  91 + */
  92 +
  93 +/* Do not directly include this file. Include feather_trace.h instead */
  94 +
  95 +#define feather_callback __attribute__((regparm(3)))  __attribute__((used))
  96 +
  97 +/*
  98 + * Make the compiler reload any register that is not saved in a cdecl function
  99 + * call (minus the registers that we explicitly clobber as output registers).
 100 + */
 101 +#define __FT_CLOBBER_LIST0 "memory", "cc", "eax", "edx", "ecx"
 102 +#define __FT_CLOBBER_LIST1 "memory", "cc", "eax", "ecx"
 103 +#define __FT_CLOBBER_LIST2 "memory", "cc", "eax"
 104 +#define __FT_CLOBBER_LIST3 "memory", "cc", "eax"
 105 +
 106 +#define __FT_TMP1(x) "=d" (x)
 107 +#define __FT_ARG1(x) "0" ((long) (x))
 108 +#define __FT_TMP2(x) "=c" (x)
 109 +#define __FT_ARG2(x) "1" ((long) (x))
 110 +
 111 +#define __FT_ARG3(x) "r" ((long) (x))
 112 +
 113 +#define ft_event(id, callback)                                  \
 114 +        __asm__ __volatile__(                                   \
 115 +            "1: jmp 2f                                    \n\t" \
 116 +	    " call " #callback "                          \n\t" \
 117 +            ".section __event_table, \"aw\"               \n\t" \
 118 +            ".long " #id  ", 0, 1b, 2f                    \n\t" \
 119 +            ".previous                                    \n\t" \
 120 +            "2:                                           \n\t" \
 121 +	    : : : __FT_CLOBBER_LIST0)
 122 +
 123 +#define ft_event0(id, callback)                                 \
 124 +        __asm__ __volatile__(                                   \
 125 +            "1: jmp 2f                                    \n\t" \
 126 +            " movl $" #id  ", %%eax                       \n\t" \
 127 +	    " call " #callback "                          \n\t" \
 128 +            ".section __event_table, \"aw\"               \n\t" \
 129 +            ".long " #id  ", 0, 1b, 2f                    \n\t" \
 130 +            ".previous                                    \n\t" \
 131 +            "2:                                           \n\t" \
 132 +	    : : : __FT_CLOBBER_LIST0)
 133 +
 134 +#define ft_event1(id, callback, param)				\
 135 +	do {							\
 136 +		long __ft_tmp1;					\
 137 +        __asm__ __volatile__(                                   \
 138 +            "1: jmp 2f                                    \n\t" \
 139 +            " movl $" #id  ", %%eax                       \n\t" \
 140 +	    " call " #callback "                          \n\t" \
 141 +            ".section __event_table, \"aw\"               \n\t" \
 142 +            ".long " #id  ", 0, 1b, 2f                    \n\t" \
 143 +            ".previous                                    \n\t" \
 144 +            "2:                                           \n\t" \
 145 +	    : __FT_TMP1(__ft_tmp1)				\
 146 +	    : __FT_ARG1(param)					\
 147 +	    : __FT_CLOBBER_LIST1);				\
 148 +	} while (0);
 149 +
 150 +#define ft_event2(id, callback, param, param2)                  \
 151 +	do {							\
 152 +		long __ft_tmp1, __ft_tmp2;			\
 153 +        __asm__ __volatile__(                                   \
 154 +            "1: jmp 2f                                    \n\t" \
 155 +            " movl $" #id  ", %%eax                       \n\t" \
 156 +	    " call " #callback "                          \n\t" \
 157 +            ".section __event_table, \"aw\"               \n\t" \
 158 +            ".long " #id  ", 0, 1b, 2f                    \n\t" \
 159 +            ".previous                                    \n\t" \
 160 +            "2:                                           \n\t" \
 161 +	    : __FT_TMP1(__ft_tmp1), __FT_TMP2(__ft_tmp2)	\
 162 +	    : __FT_ARG1(param), __FT_ARG2(param2)		\
 163 +	    : __FT_CLOBBER_LIST2);				\
 164 +	} while (0);
 165 +
 166 +
 167 +#define ft_event3(id, callback, param, param2, param3)		\
 168 +	do {							\
 169 +		long __ft_tmp1, __ft_tmp2;			\
 170 +        __asm__ __volatile__(                                   \
 171 +            "1: jmp 2f                                    \n\t" \
 172 +	    " subl $4, %%esp                              \n\t" \
 173 +            " movl $" #id  ", %%eax                       \n\t" \
 174 +	    " movl %2, (%%esp)                            \n\t" \
 175 +	    " call " #callback "                          \n\t" \
 176 +	    " addl $4, %%esp                              \n\t" \
 177 +            ".section __event_table, \"aw\"               \n\t" \
 178 +            ".long " #id  ", 0, 1b, 2f                    \n\t" \
 179 +            ".previous                                    \n\t" \
 180 +            "2:                                           \n\t" \
 181 +	    : __FT_TMP1(__ft_tmp1), __FT_TMP2(__ft_tmp2)	\
 182 +	    : __FT_ARG1(param), __FT_ARG2(param2), __FT_ARG3(param3)	\
 183 +	    : __FT_CLOBBER_LIST3);				\
 184 +	} while (0);
 185 diff --git a/arch/x86/include/asm/feather_trace_64.h b/arch/x86/include/asm/feather_trace_64.h
 186 new file mode 100644
 187 index 0000000..5ce49e2
 188 --- /dev/null
 189 +++ b/arch/x86/include/asm/feather_trace_64.h
 190 @@ -0,0 +1,124 @@
 191 +/* Copyright (c) 2010 Andrea Bastoni, <bastoni@cs.unc.edu>
 192 + * Copyright (c) 2012 Björn Brandenburg, <bbb@mpi-sws.org>
 193 + *
 194 + * Permission is hereby granted, free of charge, to any person obtaining
 195 + * a copy of this software and associated documentation files (the
 196 + * "Software"), to deal in the Software without restriction, including
 197 + * without limitation the rights to use, copy, modify, merge, publish,
 198 + * distribute, sublicense, and/or sell copies of the Software, and to
 199 + * permit persons to whom the Software is furnished to do so, subject to
 200 + * the following conditions:
 201 + *
 202 + * The above copyright notice and this permission notice shall be
 203 + * included in all copies or substantial portions of the Software.
 204 + *
 205 + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 206 + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 207 + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 208 + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 209 + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 210 + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 211 + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 212 + * SOFTWARE.
 213 + */
 214 +
 215 +/* Do not directly include this file. Include feather_trace.h instead */
 216 +
 217 +/* regparm is the default on x86_64 */
 218 +#define feather_callback  __attribute__((used))
 219 +
 220 +#define __FT_EVENT_TABLE(id,from,to) \
 221 +            ".section __event_table, \"aw\"\n\t" \
 222 +	    ".balign 8\n\t" \
 223 +            ".quad " #id  ", 0, " #from ", " #to " \n\t" \
 224 +            ".previous \n\t"
 225 +
 226 +/*
 227 + * x86_64 caller only owns rbp, rbx, r12-r15;
 228 + * the callee can freely modify the others.
 229 + */
 230 +#define __FT_CLOBBER_LIST0	"memory", "cc", "rdi", "rsi", "rdx", "rcx", \
 231 +			"r8", "r9", "r10", "r11", "rax"
 232 +
 233 +#define __FT_CLOBBER_LIST1	"memory", "cc", "rdi", "rdx", "rcx", \
 234 +			"r8", "r9", "r10", "r11", "rax"
 235 +
 236 +#define __FT_CLOBBER_LIST2	"memory", "cc", "rdi", "rcx", \
 237 +			"r8", "r9", "r10", "r11", "rax"
 238 +
 239 +#define __FT_CLOBBER_LIST3	"memory", "cc", "rdi", \
 240 +			"r8", "r9", "r10", "r11", "rax"
 241 +
 242 +/* The registers RDI, RSI, RDX, RCX, R8 and R9 are used for integer and pointer
 243 + * arguments. */
 244 +
 245 +/* RSI */
 246 +#define __FT_TMP1(x) "=S" (x)
 247 +#define __FT_ARG1(x) "0" ((long) (x))
 248 +
 249 +/* RDX */
 250 +#define __FT_TMP2(x) "=d" (x)
 251 +#define __FT_ARG2(x) "1" ((long) (x))
 252 +
 253 +/* RCX */
 254 +#define __FT_TMP3(x) "=c" (x)
 255 +#define __FT_ARG3(x) "2" ((long) (x))
 256 +
 257 +#define ft_event(id, callback)                                  \
 258 +        __asm__ __volatile__(                                   \
 259 +            "1: jmp 2f                                    \n\t" \
 260 +	    " call " #callback "                          \n\t" \
 261 +            __FT_EVENT_TABLE(id,1b,2f)				\
 262 +            "2:                                           \n\t" \
 263 +        : : : __FT_CLOBBER_LIST0)
 264 +
 265 +#define ft_event0(id, callback)                                 \
 266 +        __asm__ __volatile__(                                   \
 267 +            "1: jmp 2f                                    \n\t" \
 268 +	    " movq $" #id ", %%rdi			  \n\t" \
 269 +	    " call " #callback "                          \n\t" \
 270 +	    __FT_EVENT_TABLE(id,1b,2f)				\
 271 +            "2:                                           \n\t" \
 272 +        : :  : __FT_CLOBBER_LIST0)
 273 +
 274 +#define ft_event1(id, callback, param)                          \
 275 +	do {							\
 276 +		long __ft_tmp1;					\
 277 +	__asm__ __volatile__(                                   \
 278 +	    "1: jmp 2f                                    \n\t" \
 279 +	    " movq $" #id ", %%rdi			  \n\t" \
 280 +	    " call " #callback "                          \n\t" \
 281 +	    __FT_EVENT_TABLE(id,1b,2f)				\
 282 +	    "2:                                           \n\t" \
 283 +	    : __FT_TMP1(__ft_tmp1)				\
 284 +	    : __FT_ARG1(param)					\
 285 +	    : __FT_CLOBBER_LIST1);				\
 286 +	} while (0);
 287 +
 288 +#define ft_event2(id, callback, param, param2)                  \
 289 +	do {							\
 290 +		long __ft_tmp1, __ft_tmp2;			\
 291 +        __asm__ __volatile__(                                   \
 292 +            "1: jmp 2f                                    \n\t" \
 293 +	    " movq $" #id ", %%rdi			  \n\t" \
 294 +	    " call " #callback "                          \n\t" \
 295 +            __FT_EVENT_TABLE(id,1b,2f)				\
 296 +            "2:                                           \n\t" \
 297 +	    : __FT_TMP1(__ft_tmp1), __FT_TMP2(__ft_tmp2)	\
 298 +	    : __FT_ARG1(param), __FT_ARG2(param2)		\
 299 +	    : __FT_CLOBBER_LIST2);				\
 300 +	} while (0);
 301 +
 302 +#define ft_event3(id, callback, param, param2, param3)		\
 303 +	do {							\
 304 +		long __ft_tmp1, __ft_tmp2, __ft_tmp3;		\
 305 +        __asm__ __volatile__(                                   \
 306 +            "1: jmp 2f                                    \n\t" \
 307 +	    " movq $" #id ", %%rdi			  \n\t" \
 308 +	    " call " #callback "                          \n\t" \
 309 +            __FT_EVENT_TABLE(id,1b,2f)				\
 310 +            "2:                                           \n\t" \
 311 +	    : __FT_TMP1(__ft_tmp1), __FT_TMP2(__ft_tmp2), __FT_TMP3(__ft_tmp3) \
 312 +	    : __FT_ARG1(param), __FT_ARG2(param2), __FT_ARG3(param3)	\
 313 +	    : __FT_CLOBBER_LIST3);				\
 314 +	} while (0);
 315 diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
 316 index 7bd3bd3..d38a5a7 100644
 317 --- a/arch/x86/kernel/Makefile
 318 +++ b/arch/x86/kernel/Makefile
 319 @@ -103,6 +103,8 @@ obj-$(CONFIG_UPROBES)			+= uprobes.o
 320  
 321  obj-$(CONFIG_PERF_EVENTS)		+= perf_regs.o
 322  
 323 +obj-$(CONFIG_FEATHER_TRACE)	+= ft_event.o
 324 +
 325  ###
 326  # 64 bit specific files
 327  ifeq ($(CONFIG_X86_64),y)
 328 diff --git a/arch/x86/kernel/ft_event.c b/arch/x86/kernel/ft_event.c
 329 new file mode 100644
 330 index 0000000..37cc332
 331 --- /dev/null
 332 +++ b/arch/x86/kernel/ft_event.c
 333 @@ -0,0 +1,118 @@
 334 +#include <linux/types.h>
 335 +
 336 +#include <litmus/feather_trace.h>
 337 +
 338 +/* the feather trace management functions assume
 339 + * exclusive access to the event table
 340 + */
 341 +
 342 +#ifndef CONFIG_DEBUG_RODATA
 343 +
 344 +#define BYTE_JUMP      0xeb
 345 +#define BYTE_JUMP_LEN  0x02
 346 +
 347 +/* for each event, there is an entry in the event table */
 348 +struct trace_event {
 349 +	long 	id;
 350 +	long	count;
 351 +	long	start_addr;
 352 +	long	end_addr;
 353 +};
 354 +
 355 +extern struct trace_event  __start___event_table[];
 356 +extern struct trace_event  __stop___event_table[];
 357 +
 358 +/* Workaround: if no events are defined, then the event_table section does not
 359 + * exist and the above references cause linker errors. This could probably be
 360 + * fixed by adjusting the linker script, but it is easier to maintain for us if
 361 + * we simply create a dummy symbol in the event table section.
 362 + */
 363 +int __event_table_dummy[0] __attribute__ ((section("__event_table")));
 364 +
 365 +int ft_enable_event(unsigned long id)
 366 +{
 367 +	struct trace_event* te = __start___event_table;
 368 +	int count = 0;
 369 +	char* delta;
 370 +	unsigned char* instr;
 371 +
 372 +	while (te < __stop___event_table) {
 373 +		if (te->id == id && ++te->count == 1) {
 374 +			instr  = (unsigned char*) te->start_addr;
 375 +			/* make sure we don't clobber something wrong */
 376 +			if (*instr == BYTE_JUMP) {
 377 +				delta  = (((unsigned char*) te->start_addr) + 1);
 378 +				*delta = 0;
 379 +			}
 380 +		}
 381 +		if (te->id == id)
 382 +			count++;
 383 +		te++;
 384 +	}
 385 +
 386 +	printk(KERN_DEBUG "ft_enable_event: enabled %d events\n", count);
 387 +	return count;
 388 +}
 389 +
 390 +int ft_disable_event(unsigned long id)
 391 +{
 392 +	struct trace_event* te = __start___event_table;
 393 +	int count = 0;
 394 +	char* delta;
 395 +	unsigned char* instr;
 396 +
 397 +	while (te < __stop___event_table) {
 398 +		if (te->id == id && --te->count == 0) {
 399 +			instr  = (unsigned char*) te->start_addr;
 400 +			if (*instr == BYTE_JUMP) {
 401 +				delta  = (((unsigned char*) te->start_addr) + 1);
 402 +				*delta = te->end_addr - te->start_addr -
 403 +					BYTE_JUMP_LEN;
 404 +			}
 405 +		}
 406 +		if (te->id == id)
 407 +			count++;
 408 +		te++;
 409 +	}
 410 +
 411 +	printk(KERN_DEBUG "ft_disable_event: disabled %d events\n", count);
 412 +	return count;
 413 +}
 414 +
 415 +int ft_disable_all_events(void)
 416 +{
 417 +	struct trace_event* te = __start___event_table;
 418 +	int count = 0;
 419 +	char* delta;
 420 +	unsigned char* instr;
 421 +
 422 +	while (te < __stop___event_table) {
 423 +		if (te->count) {
 424 +			instr  = (unsigned char*) te->start_addr;
 425 +			if (*instr == BYTE_JUMP) {
 426 +				delta  = (((unsigned char*) te->start_addr)
 427 +					  + 1);
 428 +				*delta = te->end_addr - te->start_addr -
 429 +					BYTE_JUMP_LEN;
 430 +				te->count = 0;
 431 +				count++;
 432 +			}
 433 +		}
 434 +		te++;
 435 +	}
 436 +	return count;
 437 +}
 438 +
 439 +int ft_is_event_enabled(unsigned long id)
 440 +{
 441 +	struct trace_event* te = __start___event_table;
 442 +
 443 +	while (te < __stop___event_table) {
 444 +		if (te->id == id)
 445 +			return te->count;
 446 +		te++;
 447 +	}
 448 +	return 0;
 449 +}
 450 +
 451 +#endif
 452 diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
 453 index 48d2b7d..5919042 100644
 454 --- a/arch/x86/kernel/smp.c
 455 +++ b/arch/x86/kernel/smp.c
 456 @@ -24,6 +24,8 @@
 457  #include <linux/cpu.h>
 458  #include <linux/gfp.h>
 459  
 460 +#include <litmus/trace.h>
 461 +
 462  #include <asm/mtrr.h>
 463  #include <asm/tlbflush.h>
 464  #include <asm/mmu_context.h>
 465 @@ -123,6 +125,9 @@ static void native_smp_send_reschedule(int cpu)
 466  		WARN_ON(1);
 467  		return;
 468  	}
 469 +
 470 +    /* LITMUS^RT*/
 471 +    TS_SEND_RESCHED_START(cpu);
 472  	apic->send_IPI_mask(cpumask_of(cpu), RESCHEDULE_VECTOR);
 473  }
 474  
 475 @@ -257,6 +262,9 @@ void smp_reschedule_interrupt(struct pt_regs *regs)
 476  	/*
 477  	 * KVM uses this interrupt to force a cpu out of guest mode
 478  	 */
 479 +
 480 +    /* LITMUS^RT */
 481 +    TS_SEND_RESCHED_END;
 482  }
 483  
 484  void smp_call_function_interrupt(struct pt_regs *regs)
 485 diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
 486 index c1d6555..7ad5fd8 100644
 487 --- a/include/linux/hardirq.h
 488 +++ b/include/linux/hardirq.h
 489 @@ -7,6 +7,8 @@
 490  #include <linux/vtime.h>
 491  #include <asm/hardirq.h>
 492  
 493 +#include <litmus/trace_irq.h>
 494 +
 495  /*
 496   * We put the hardirq and softirq counter into the preemption
 497   * counter. The bitmask has the following meaning:
 498 @@ -154,6 +156,7 @@ extern void rcu_nmi_exit(void);
 499  		account_irq_enter_time(current);	\
 500  		add_preempt_count(HARDIRQ_OFFSET);	\
 501  		trace_hardirq_enter();			\
 502 +		ft_irq_fired();				\
 503  	} while (0)
 504  
 505  /*
 506 @@ -184,6 +187,7 @@ extern void irq_exit(void);
 507  		add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET);	\
 508  		rcu_nmi_enter();				\
 509  		trace_hardirq_enter();				\
 510 +		ft_irq_fired();					\
 511  	} while (0)
 512  
 513  #define nmi_exit()						\
 514 diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
 515 index d19a5c2..923a57a 100644
 516 --- a/include/linux/hrtimer.h
 517 +++ b/include/linux/hrtimer.h
 518 @@ -116,6 +116,8 @@ struct hrtimer {
 519  	void				*start_site;
 520  	char				start_comm[16];
 521  #endif
 522 +
 523 +	ktime_t				enqueued;
 524  };
 525  
 526  /**
 527 diff --git a/include/litmus/feather_buffer.h b/include/litmus/feather_buffer.h
 528 new file mode 100644
 529 index 0000000..38de95b
 530 --- /dev/null
 531 +++ b/include/litmus/feather_buffer.h
 532 @@ -0,0 +1,118 @@
 533 +#ifndef _FEATHER_BUFFER_H_
 534 +#define _FEATHER_BUFFER_H_
 535 +
 536 +/* requires UINT_MAX and memcpy */
 537 +
 538 +#define SLOT_FREE	0
 539 +#define	SLOT_BUSY 	1
 540 +#define	SLOT_READY	2
 541 +
 542 +struct ft_buffer {
 543 +	unsigned int	slot_count;
 544 +	unsigned int	slot_size;
 545 +
 546 +	int 		free_count;
 547 +	unsigned int 	write_idx;
 548 +	unsigned int 	read_idx;
 549 +
 550 +	char*		slots;
 551 +	void*		buffer_mem;
 552 +	unsigned int	failed_writes;
 553 +};
 554 +
 555 +static inline int init_ft_buffer(struct ft_buffer*	buf,
 556 +				 unsigned int 		slot_count,
 557 +				 unsigned int 		slot_size,
 558 +				 char*			slots,
 559 +				 void* 			buffer_mem)
 560 +{
 561 +	int i = 0;
 562 +	if (!slot_count || UINT_MAX % slot_count != slot_count - 1) {
 563 +		/* The slot count must divide UNIT_MAX + 1 so that when it
 564 +		 * wraps around the index correctly points to 0.
 565 +		 */
 566 +		return 0;
 567 +	} else {
 568 +		buf->slot_count    = slot_count;
 569 +		buf->slot_size     = slot_size;
 570 +		buf->slots         = slots;
 571 +		buf->buffer_mem    = buffer_mem;
 572 +		buf->free_count    = slot_count;
 573 +		buf->write_idx     = 0;
 574 +		buf->read_idx      = 0;
 575 +		buf->failed_writes = 0;
 576 +		for (i = 0; i < slot_count; i++)
 577 +			buf->slots[i] = SLOT_FREE;
 578 +		return 1;
 579 +	}
 580 +}
 581 +
 582 +static inline int ft_buffer_start_write(struct ft_buffer* buf, void **ptr)
 583 +{
 584 +	int free = fetch_and_dec(&buf->free_count);
 585 +	unsigned int idx;
 586 +	if (free <= 0) {
 587 +		fetch_and_inc(&buf->free_count);
 588 +		*ptr = 0;
 589 +		fetch_and_inc(&buf->failed_writes);
 590 +		return 0;
 591 +	} else {
 592 +		idx  = fetch_and_inc((int*) &buf->write_idx) % buf->slot_count;
 593 +		buf->slots[idx] = SLOT_BUSY;
 594 +		*ptr = ((char*) buf->buffer_mem) + idx * buf->slot_size;
 595 +		return 1;
 596 +	}
 597 +}
 598 +
 599 +/* For single writer scenarios, with fewer atomic ops. */
 600 +static inline int ft_buffer_start_single_write(struct ft_buffer* buf, void **ptr)
 601 +{
 602 +	unsigned int idx;
 603 +
 604 +	if (buf->free_count <= 0) {
 605 +		*ptr = 0;
 606 +		/* single writer: no atomicity needed */
 607 +		buf->failed_writes++;
 608 +		return 0;
 609 +	} else {
 610 +		/* free_count is positive, and can only increase since we are
 611 +		 * (by assumption) the only writer accessing the buffer.
 612 +		 */
 613 +
 614 +		idx  = buf->write_idx++ % buf->slot_count;
 615 +		buf->slots[idx] = SLOT_BUSY;
 616 +		*ptr = ((char*) buf->buffer_mem) + idx * buf->slot_size;
 617 +
 618 +		ft_atomic_dec(&buf->free_count);
 619 +		return 1;
 620 +	}
 621 +}
 622 +
 623 +static inline void ft_buffer_finish_write(struct ft_buffer* buf, void *ptr)
 624 +{
 625 +	unsigned int idx = ((char*) ptr - (char*) buf->buffer_mem) / buf->slot_size;
 626 +	buf->slots[idx]  = SLOT_READY;
 627 +}
 628 +
 629 +
 630 +/* exclusive reader access is assumed */
 631 +static inline int ft_buffer_read(struct ft_buffer* buf, void* dest)
 632 +{
 633 +	unsigned int idx;
 634 +	if (buf->free_count == buf->slot_count)
 635 +		/* nothing available */
 636 +		return 0;
 637 +	idx = buf->read_idx % buf->slot_count;
 638 +	if (buf->slots[idx] == SLOT_READY) {
 639 +		memcpy(dest, ((char*) buf->buffer_mem) + idx * buf->slot_size,
 640 +		       buf->slot_size);
 641 +		buf->slots[idx] = SLOT_FREE;
 642 +		buf->read_idx++;
 643 +		fetch_and_inc(&buf->free_count);
 644 +		return 1;
 645 +	} else
 646 +		return 0;
 647 +}
 648 +
 649 +
 650 +#endif
 651 diff --git a/include/litmus/feather_trace.h b/include/litmus/feather_trace.h
 652 new file mode 100644
 653 index 0000000..76e8004
 654 --- /dev/null
 655 +++ b/include/litmus/feather_trace.h
 656 @@ -0,0 +1,70 @@
 657 +#ifndef _FEATHER_TRACE_H_
 658 +#define _FEATHER_TRACE_H_
 659 +
 660 +#include <asm/atomic.h>
 661 +
 662 +int ft_enable_event(unsigned long id);
 663 +int ft_disable_event(unsigned long id);
 664 +int ft_is_event_enabled(unsigned long id);
 665 +int ft_disable_all_events(void);
 666 +
 667 +/* atomic_* funcitons are inline anyway */
 668 +static inline int fetch_and_inc(int *val)
 669 +{
 670 +	return atomic_add_return(1, (atomic_t*) val) - 1;
 671 +}
 672 +
 673 +static inline int fetch_and_dec(int *val)
 674 +{
 675 +	return atomic_sub_return(1, (atomic_t*) val) + 1;
 676 +}
 677 +
 678 +static inline void ft_atomic_dec(int *val)
 679 +{
 680 +	atomic_sub(1, (atomic_t*) val);
 681 +}
 682 +
 683 +/* Don't use rewriting implementation if kernel text pages are read-only.
 684 + * Ftrace gets around this by using the identity mapping, but that's more
 685 + * effort that is warrented right now for Feather-Trace.
 686 + * Eventually, it may make sense to replace Feather-Trace with ftrace.
 687 + */
 688 +#if defined(CONFIG_ARCH_HAS_FEATHER_TRACE) && !defined(CONFIG_DEBUG_RODATA)
 689 +
 690 +#include <asm/feather_trace.h>
 691 +
 692 +#else /* !__ARCH_HAS_FEATHER_TRACE */
 693 +
 694 +/* provide default implementation */
 695 +
 696 +#include <asm/timex.h> /* for get_cycles() */
 697 +
 698 +static inline unsigned long long ft_timestamp(void)
 699 +{
 700 +	return get_cycles();
 701 +}
 702 +
 703 +#define feather_callback
 704 +
 705 +#define MAX_EVENTS 1024
 706 +
 707 +extern int ft_events[MAX_EVENTS];
 708 +
 709 +#define ft_event(id, callback) \
 710 +	if (ft_events[id]) callback();
 711 +
 712 +#define ft_event0(id, callback) \
 713 +	if (ft_events[id]) callback(id);
 714 +
 715 +#define ft_event1(id, callback, param) \
 716 +	if (ft_events[id]) callback(id, param);
 717 +
 718 +#define ft_event2(id, callback, param, param2) \
 719 +	if (ft_events[id]) callback(id, param, param2);
 720 +
 721 +#define ft_event3(id, callback, p, p2, p3) \
 722 +	if (ft_events[id]) callback(id, p, p2, p3);
 723 +
 724 +#endif /* __ARCH_HAS_FEATHER_TRACE */
 725 +
 726 +#endif
 727 diff --git a/include/litmus/ftdev.h b/include/litmus/ftdev.h
 728 new file mode 100644
 729 index 0000000..a566b0b
 730 --- /dev/null
 731 +++ b/include/litmus/ftdev.h
 732 @@ -0,0 +1,58 @@
 733 +#ifndef _LITMUS_FTDEV_H_
 734 +#define	_LITMUS_FTDEV_H_
 735 +
 736 +#include <litmus/feather_trace.h>
 737 +#include <litmus/feather_buffer.h>
 738 +#include <linux/mutex.h>
 739 +#include <linux/cdev.h>
 740 +
 741 +#define FTDEV_ENABLE_CMD 	0
 742 +#define FTDEV_DISABLE_CMD 	1
 743 +#define FTDEV_CALIBRATE		0x1410
 744 +
 745 +struct ftdev;
 746 +
 747 +/* return 0 if buffer can be opened, otherwise -$REASON */
 748 +typedef int  (*ftdev_can_open_t)(struct ftdev* dev, unsigned int buf_no);
 749 +/* return 0 on success, otherwise -$REASON */
 750 +typedef int  (*ftdev_alloc_t)(struct ftdev* dev, unsigned int buf_no);
 751 +typedef void (*ftdev_free_t)(struct ftdev* dev, unsigned int buf_no);
 752 +typedef long (*ftdev_calibrate_t)(struct ftdev* dev, unsigned int buf_no, unsigned long user_arg);
 753 +/* Let devices handle writes from userspace. No synchronization provided. */
 754 +typedef ssize_t (*ftdev_write_t)(struct ft_buffer* buf, size_t len, const char __user *from);
 755 +
 756 +struct ftdev_event;
 757 +
 758 +struct ftdev_minor {
 759 +	struct ft_buffer*	buf;
 760 +	unsigned int		readers;
 761 +	struct mutex		lock;
 762 +	/* FIXME: filter for authorized events */
 763 +	struct ftdev_event*	events;
 764 +	struct device*		device;
 765 +	struct ftdev*		ftdev;
 766 +};
 767 +
 768 +struct ftdev {
 769 +	dev_t			major;
 770 +	struct cdev		cdev;
 771 +	struct class*		class;
 772 +	const char*		name;
 773 +	struct ftdev_minor*	minor;
 774 +	unsigned int		minor_cnt;
 775 +	ftdev_alloc_t		alloc;
 776 +	ftdev_free_t		free;
 777 +	ftdev_can_open_t	can_open;
 778 +	ftdev_write_t		write;
 779 +	ftdev_calibrate_t	calibrate;
 780 +};
 781 +
 782 +struct ft_buffer* alloc_ft_buffer(unsigned int count, size_t size);
 783 +void free_ft_buffer(struct ft_buffer* buf);
 784 +
 785 +int ftdev_init(	struct ftdev* ftdev, struct module* owner,
 786 +		const int minor_cnt, const char* name);
 787 +void ftdev_exit(struct ftdev* ftdev);
 788 +int register_ftdev(struct ftdev* ftdev);
 789 +
 790 +#endif
 791 diff --git a/include/litmus/trace.h b/include/litmus/trace.h
 792 new file mode 100644
 793 index 0000000..4f96dd8
 794 --- /dev/null
 795 +++ b/include/litmus/trace.h
 796 @@ -0,0 +1,166 @@
 797 +#ifndef _SYS_TRACE_H_
 798 +#define	_SYS_TRACE_H_
 799 +
 800 +#ifdef CONFIG_SCHED_OVERHEAD_TRACE
 801 +
 802 +
 803 +#include <litmus/feather_trace.h>
 804 +#include <litmus/feather_buffer.h>
 805 +
 806 +/* Litmus time type. */
 807 +typedef unsigned long long lt_t;
 808 +
 809 +/* Our notion of time within LITMUS: kernel monotonic time. */
 810 +static inline lt_t litmus_clock(void)
 811 +{
 812 +	return ktime_to_ns(ktime_get());
 813 +}
 814 +
 815 +/* Only capture traces of SCHED_DEADLINE tasks */
 816 +#define is_realtime(t)      ((t)->policy == SCHED_DEADLINE)
 817 +
 818 +/*********************** TIMESTAMPS ************************/
 819 +
 820 +enum task_type_marker {
 821 +	TSK_BE,
 822 +	TSK_RT,
 823 +	TSK_UNKNOWN
 824 +};
 825 +
 826 +struct timestamp {
 827 +	uint64_t		timestamp:48;
 828 +	uint64_t		pid:16;
 829 +	uint32_t		seq_no;
 830 +	uint8_t			cpu;
 831 +	uint8_t			event;
 832 +	uint8_t			task_type:2;
 833 +	uint8_t			irq_flag:1;
 834 +	uint8_t			irq_count:5;
 835 +};
 836 +
 837 +/* tracing callbacks */
 838 +feather_callback void save_timestamp(unsigned long event);
 839 +feather_callback void save_timestamp_def(unsigned long event, unsigned long type);
 840 +feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu);
 841 +feather_callback void save_timestamp_time(unsigned long event, unsigned long time_ptr);
 842 +feather_callback void save_timestamp_irq(unsigned long event, unsigned long irq_count_ptr);
 843 +feather_callback void save_timestamp_hide_irq(unsigned long event);
 844 +feather_callback void save_timestamp_hide_irq_task(unsigned long event, unsigned long t_ptr);
 845 +feather_callback void save_timestamp_hide_irq_cpu(unsigned long event, unsigned long cpu);
 846 +
 847 +feather_callback void save_cpu_timestamp(unsigned long event);
 848 +feather_callback void save_cpu_timestamp_task(unsigned long event, unsigned long t_ptr);
 849 +feather_callback void save_cpu_timestamp_def(unsigned long event, unsigned long type);
 850 +feather_callback void save_cpu_task_latency(unsigned long event, unsigned long when_ptr);
 851 +
 852 +#define TIMESTAMP(id) ft_event0(id, save_timestamp)
 853 +
 854 +#define DTIMESTAMP(id, def)  ft_event1(id, save_timestamp_def, (unsigned long) def)
 855 +
 856 +#define TIMESTAMP_CUR(id) DTIMESTAMP(id, is_realtime(current) ? TSK_RT : TSK_BE)
 857 +
 858 +#define CTIMESTAMP(id, cpu) \
 859 +	ft_event1(id, save_timestamp_cpu, (unsigned long) cpu)
 860 +
 861 +#define TIMESTAMP_TIME(id, time_ptr) \
 862 +	ft_event1(id, save_timestamp_time, (unsigned long) time_ptr)
 863 +
 864 +#define TIMESTAMP_IRQ(id, irq_count_ptr) \
 865 +	ft_event1(id, save_timestamp_irq, (unsigned long) irq_count_ptr)
 866 +
 867 +#define TIMESTAMP_IN_IRQ(id) \
 868 +	ft_event0(id, save_timestamp_hide_irq)
 869 +
 870 +#define CPU_TIMESTAMP(id) ft_event0(id, save_cpu_timestamp)
 871 +
 872 +#define CPU_DTIMESTAMP(id, def)  ft_event1(id, save_cpu_timestamp_def, (unsigned long) def)
 873 +
 874 +#define CPU_TTIMESTAMP(id, task) \
 875 +	ft_event1(id, save_cpu_timestamp_task, (unsigned long) task)
 876 +
 877 +#define CPU_LTIMESTAMP(id, task) \
 878 +	ft_event1(id, save_cpu_task_latency, (unsigned long) task)
 879 +
 880 +
 881 +#else /* !CONFIG_SCHED_OVERHEAD_TRACE */
 882 +
 883 +#define TIMESTAMP(id)        /* no tracing */
 884 +
 885 +#define DTIMESTAMP(id, def)  /* no tracing */
 886 +
 887 +#define TIMESTAMP_CUR(id)    /* no tracing */
 888 +
 889 +#define TTIMESTAMP(id, task) /* no tracing */
 890 +
 891 +#define CTIMESTAMP(id, cpu)  /* no tracing */
 892 +
 893 +#define LTIMESTAMP(id, when_ptr) /* no tracing */
 894 +
 895 +#define TIMESTAMP_TIME(id, time_ptr) /* no tracing */
 896 +
 897 +#define TIMESTAMP_IRQ(id, irq_count_ptr) /* no tracing */
 898 +
 899 +#define TIMESTAMP_IN_IRQ(id) /* no tracing */
 900 +
 901 +#endif
 902 +
 903 +
 904 +/* Convention for timestamps
 905 + * =========================
 906 + *
 907 + * In order to process the trace files with a common tool, we use the following
 908 + * convention to measure execution times: The end time id of a code segment is
 909 + * always the next number after the start time event id.
 910 + */
 911 +
 912 +#define __TS_SYSCALL_IN_START(p)	TIMESTAMP_TIME(10, p)
 913 +#define __TS_SYSCALL_IN_END(p)		TIMESTAMP_IRQ(11, p)
 914 +
 915 +#define TS_SYSCALL_OUT_START		TIMESTAMP_CUR(20)
 916 +#define TS_SYSCALL_OUT_END		TIMESTAMP_CUR(21)
 917 +
 918 +#define TS_LOCK_START			TIMESTAMP_CUR(30)
 919 +#define TS_LOCK_END			TIMESTAMP_CUR(31)
 920 +
 921 +#define TS_LOCK_SUSPEND			TIMESTAMP_CUR(38)
 922 +#define TS_LOCK_RESUME			TIMESTAMP_CUR(39)
 923 +
 924 +#define TS_UNLOCK_START			TIMESTAMP_CUR(40)
 925 +#define TS_UNLOCK_END			TIMESTAMP_CUR(41)
 926 +
 927 +#define TS_SCHED_START			CPU_DTIMESTAMP(100, TSK_UNKNOWN) /* we only
 928 +								      * care
 929 +								      * about
 930 +								      * next */
 931 +#define TS_SCHED_END(t)			CPU_TTIMESTAMP(101, t)
 932 +#define TS_SCHED2_START(t) 		CPU_TTIMESTAMP(102, t)
 933 +#define TS_SCHED2_END(t)       		CPU_TTIMESTAMP(103, t)
 934 +
 935 +#define TS_CXS_START(t)			CPU_TTIMESTAMP(104, t)
 936 +#define TS_CXS_END(t)			CPU_TTIMESTAMP(105, t)
 937 +
 938 +#define TS_RELEASE_START		CPU_DTIMESTAMP(106, TSK_RT)
 939 +#define TS_RELEASE_END			CPU_DTIMESTAMP(107, TSK_RT)
 940 +
 941 +#define TS_TICK_START(t)		CPU_TTIMESTAMP(110, t)
 942 +#define TS_TICK_END(t) 			CPU_TTIMESTAMP(111, t)
 943 +
 944 +
 945 +#define TS_PLUGIN_SCHED_START		/* TIMESTAMP(120) */  /* currently unused */
 946 +#define TS_PLUGIN_SCHED_END		/* TIMESTAMP(121) */
 947 +
 948 +#define TS_PLUGIN_TICK_START		/* TIMESTAMP(130) */
 949 +#define TS_PLUGIN_TICK_END		/* TIMESTAMP(131) */
 950 +
 951 +#define TS_ENTER_NP_START		CPU_TIMESTAMP(140)
 952 +#define TS_ENTER_NP_END			CPU_TIMESTAMP(141)
 953 +
 954 +#define TS_EXIT_NP_START		CPU_TIMESTAMP(150)
 955 +#define TS_EXIT_NP_END			CPU_TIMESTAMP(151)
 956 +
 957 +#define TS_SEND_RESCHED_START(c)	CTIMESTAMP(190, c)
 958 +#define TS_SEND_RESCHED_END		TIMESTAMP_IN_IRQ(191)
 959 +
 960 +#define TS_RELEASE_LATENCY(when)	CPU_LTIMESTAMP(208, &(when))
 961 +
 962 +#endif /* !_SYS_TRACE_H_ */
 963 diff --git a/include/litmus/trace_irq.h b/include/litmus/trace_irq.h
 964 new file mode 100644
 965 index 0000000..0d0c042
 966 --- /dev/null
 967 +++ b/include/litmus/trace_irq.h
 968 @@ -0,0 +1,14 @@
 969 +#ifndef _LITMUS_TRACE_IRQ_H_
 970 +#define	_LITMUS_TRACE_IRQ_H_
 971 +
 972 +#ifdef CONFIG_SCHED_OVERHEAD_TRACE
 973 +
 974 +void ft_irq_fired(void);
 975 +
 976 +#else
 977 +
 978 +#define ft_irq_fired() /* nothing to do */
 979 +
 980 +#endif
 981 +
 982 +#endif
 983 diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
 984 index 8dcf9d9..2dda0a1 100644
 985 --- a/kernel/hrtimer.c
 986 +++ b/kernel/hrtimer.c
 987 @@ -49,6 +49,8 @@
 988  #include <linux/sched/deadline.h>
 989  #include <linux/timer.h>
 990  
 991 +#include <litmus/trace.h>
 992 +
 993  #include <asm/uaccess.h>
 994  
 995  #include <trace/events/timer.h>
 996 @@ -885,6 +887,8 @@ static int enqueue_hrtimer(struct hrtimer *timer,
 997  	 */
 998  	timer->state |= HRTIMER_STATE_ENQUEUED;
 999  
1000 +	timer->enqueued = base->get_time();
1001 +
1002  	return (&timer->node == base->active.next);
1003  }
1004  
1005 @@ -1520,10 +1524,22 @@ static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
1006  	struct hrtimer_sleeper *t =
1007  		container_of(timer, struct hrtimer_sleeper, timer);
1008  	struct task_struct *task = t->task;
1009 +	lt_t expected;
1010  
1011  	t->task = NULL;
1012 -	if (task)
1013 +	if (task) {
1014 +		if (is_realtime(task) && timer->base->clockid == CLOCK_MONOTONIC
1015 +		    && in_irq()
1016 +		    && ktime_to_ns(timer->enqueued) < ktime_to_ns(hrtimer_get_softexpires(timer))) {
1017 +			/* LITMUS^RT: Only trace SCHED_DEADLINE tasks */
1018 +			expected = ktime_to_ns(hrtimer_get_softexpires(timer));
1019 +			TS_RELEASE_LATENCY(expected);
1020 +		}
1021 +
1022 +		TS_RELEASE_START;
1023  		wake_up_process(task);
1024 +		TS_RELEASE_END;
1025 +    }
1026  
1027  	return HRTIMER_NORESTART;
1028  }
1029 diff --git a/kernel/sched/core.c b/kernel/sched/core.c
1030 index 52245d3..6a05e3a 100644
1031 --- a/kernel/sched/core.c
1032 +++ b/kernel/sched/core.c
1033 @@ -86,6 +86,8 @@
1034  #include "../workqueue_internal.h"
1035  #include "../smpboot.h"
1036  
1037 +#include <litmus/trace.h>
1038 +
1039  #define CREATE_TRACE_POINTS
1040  #include <trace/events/sched.h>
1041  
1042 @@ -1405,7 +1407,12 @@ void scheduler_ipi(void)
1043  {
1044  	if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()
1045  	    && !tick_nohz_full_cpu(smp_processor_id()))
1046 +	{
1047 +		/* If we don't call irq_enter(), we need to triggger the IRQ
1048 +		 * tracing manually. */
1049 +		ft_irq_fired();
1050  		return;
1051 +	}
1052  
1053  	/*
1054  	 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
1055 @@ -2842,6 +2849,8 @@ void scheduler_tick(void)
1056  
1057  	sched_clock_tick();
1058  
1059 +	TS_TICK_START(current);
1060 +
1061  	raw_spin_lock(&rq->lock);
1062  	update_rq_clock(rq);
1063  	update_cpu_load_active(rq);
1064 @@ -2855,6 +2864,8 @@ void scheduler_tick(void)
1065  	trigger_load_balance(rq, cpu);
1066  #endif
1067  	rq_last_tick_reset(rq);
1068 +
1069 +	TS_TICK_END(current);
1070  }
1071  
1072  #ifdef CONFIG_NO_HZ_FULL
1073 @@ -3066,6 +3077,8 @@ need_resched:
1074  	rcu_note_context_switch(cpu);
1075  	prev = rq->curr;
1076  
1077 +	TS_SCHED_START;
1078 +
1079  	schedule_debug(prev);
1080  
1081  	if (sched_feat(HRTICK))
1082 @@ -3112,7 +3125,10 @@ need_resched:
1083  		rq->curr = next;
1084  		++*switch_count;
1085  
1086 +		TS_SCHED_END(next);
1087 +		TS_CXS_START(next);
1088  		context_switch(rq, prev, next); /* unlocks the rq */
1089 +		TS_CXS_END(current);
1090  		/*
1091  		 * The context switch have flipped the stack from under us
1092  		 * and restored the local variables which were saved when
1093 @@ -3121,12 +3137,19 @@ need_resched:
1094  		 */
1095  		cpu = smp_processor_id();
1096  		rq = cpu_rq(cpu);
1097 -	} else
1098 +	} else {
1099 +		TS_SCHED_END(prev);
1100  		raw_spin_unlock_irq(&rq->lock);
1101 +	}
1102 +
1103 +	TS_SCHED2_START(prev);
1104  
1105  	post_schedule(rq);
1106  
1107  	sched_preempt_enable_no_resched();
1108 +
1109 +	TS_SCHED2_END(prev);
1110 +
1111  	if (need_resched())
1112  		goto need_resched;
1113  }
1114 diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
1115 index 00b550b..bda3cef 100644
1116 --- a/kernel/sched/deadline.c
1117 +++ b/kernel/sched/deadline.c
1118 @@ -18,6 +18,8 @@
1119  
1120  #include <linux/slab.h>
1121  
1122 +#include <litmus/trace.h>
1123 +
1124  struct dl_bandwidth def_dl_bandwidth;
1125  
1126  static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)
1127 diff --git a/kernel/softirq.c b/kernel/softirq.c
1128 index b5197dc..28542ab 100644
1129 --- a/kernel/softirq.c
1130 +++ b/kernel/softirq.c
1131 @@ -220,6 +220,9 @@ asmlinkage void __do_softirq(void)
1132  	 */
1133  	current->flags &= ~PF_MEMALLOC;
1134  
1135 +	/* Mark Feather-Trace samples as "disturbed". */
1136 +	ft_irq_fired();
1137 +
1138  	pending = local_softirq_pending();
1139  	account_irq_enter_time(current);
1140  
1141 diff --git a/litmus/Kconfig b/litmus/Kconfig
1142 new file mode 100644
1143 index 0000000..0c7e06b
1144 --- /dev/null
1145 +++ b/litmus/Kconfig
1146 @@ -0,0 +1,54 @@
1147 +menu "LITMUS^RT"
1148 +
1149 +menu "Tracing"
1150 +
1151 +config FEATHER_TRACE
1152 +	bool "Feather-Trace Infrastructure"
1153 +	depends on !RELOCATABLE
1154 +	default y
1155 +	help
1156 +	  Feather-Trace basic tracing infrastructure. Includes device file
1157 +	  driver and instrumentation point support.
1158 +
1159 +	  There are actually two implementations of Feather-Trace.
1160 +	  1) A slower, but portable, default implementation.
1161 +	  2) Architecture-specific implementations that rewrite kernel .text at runtime.
1162 +
1163 +	  If enabled, Feather-Trace will be based on 2) if available (currently only for x86).
1164 +	  However, if DEBUG_RODATA=y, then Feather-Trace will choose option 1) in any case
1165 +	  to avoid problems with write-protected .text pages.
1166 +
1167 +	  Bottom line: to avoid increased overheads, choose DEBUG_RODATA=n.
1168 +
1169 +	  Note that this option only enables the basic Feather-Trace infrastructure;
1170 +	  you still need to enable SCHED_TASK_TRACE and/or SCHED_OVERHEAD_TRACE to
1171 +	  actually enable any events.
1172 +
1173 +config SCHED_OVERHEAD_TRACE
1174 +	bool "Record timestamps for overhead measurements"
1175 +	depends on FEATHER_TRACE
1176 +	default y
1177 +	help
1178 +	  Export event stream for overhead tracing.
1179 +	  Say Yes for overhead tracing.
1180 +
1181 +config SCHED_OVERHEAD_TRACE_SHIFT
1182 +       int "Buffer size for Feather-Trace overhead data"
1183 +       depends on SCHED_OVERHEAD_TRACE
1184 +       range 15 32
1185 +       default 22
1186 +       help
1187 +
1188 +         Select the buffer size for the Feather-Trace overhead tracing
1189 +         infrastructure (/dev/litmus/ft_trace0 & ftcat) as a power of two.  The
1190 +         larger the buffer, the less likely the chance of buffer overflows if
1191 +         the ftcat process is starved by real-time activity. In machines with
1192 +         large memories, large buffer sizes are recommended.
1193 +
1194 +	 Examples: 16 =>   2 MB
1195 +		   24 => 512 MB
1196 +		   26 =>  2G MB
1197 +
1198 +endmenu
1199 +
1200 +endmenu
1201 diff --git a/litmus/Makefile b/litmus/Makefile
1202 new file mode 100644
1203 index 0000000..99f90c3
1204 --- /dev/null
1205 +++ b/litmus/Makefile
1206 @@ -0,0 +1,6 @@
1207 +#
1208 +# Makefile for LITMUS^RT
1209 +#
1210 +
1211 +obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o
1212 +obj-$(CONFIG_SCHED_OVERHEAD_TRACE) += trace.o
1213 diff --git a/litmus/ft_event.c b/litmus/ft_event.c
1214 new file mode 100644
1215 index 0000000..399a07b
1216 --- /dev/null
1217 +++ b/litmus/ft_event.c
1218 @@ -0,0 +1,43 @@
1219 +#include <linux/types.h>
1220 +
1221 +#include <litmus/feather_trace.h>
1222 +
1223 +#if !defined(CONFIG_ARCH_HAS_FEATHER_TRACE) || defined(CONFIG_DEBUG_RODATA)
1224 +/* provide dummy implementation */
1225 +
1226 +int ft_events[MAX_EVENTS];
1227 +
1228 +int ft_enable_event(unsigned long id)
1229 +{
1230 +	if (id < MAX_EVENTS) {
1231 +		ft_events[id]++;
1232 +		return 1;
1233 +	} else
1234 +		return 0;
1235 +}
1236 +
1237 +int ft_disable_event(unsigned long id)
1238 +{
1239 +	if (id < MAX_EVENTS && ft_events[id]) {
1240 +		ft_events[id]--;
1241 +		return 1;
1242 +	} else
1243 +		return 0;
1244 +}
1245 +
1246 +int ft_disable_all_events(void)
1247 +{
1248 +	int i;
1249 +
1250 +	for (i = 0; i < MAX_EVENTS; i++)
1251 +		ft_events[i] = 0;
1252 +
1253 +	return MAX_EVENTS;
1254 +}
1255 +
1256 +int ft_is_event_enabled(unsigned long id)
1257 +{
1258 +	return 	id < MAX_EVENTS && ft_events[id];
1259 +}
1260 +
1261 +#endif
1262 diff --git a/litmus/ftdev.c b/litmus/ftdev.c
1263 new file mode 100644
1264 index 0000000..13f1d48
1265 --- /dev/null
1266 +++ b/litmus/ftdev.c
1267 @@ -0,0 +1,439 @@
1268 +#include <linux/sched.h>
1269 +#include <linux/fs.h>
1270 +#include <linux/slab.h>
1271 +#include <linux/cdev.h>
1272 +#include <asm/uaccess.h>
1273 +#include <linux/module.h>
1274 +#include <linux/device.h>
1275 +#include <linux/vmalloc.h>
1276 +
1277 +#include <litmus/feather_trace.h>
1278 +#include <litmus/ftdev.h>
1279 +
1280 +struct ft_buffer* alloc_ft_buffer(unsigned int count, size_t size)
1281 +{
1282 +	struct ft_buffer* buf;
1283 +	size_t total = (size + 1) * count;
1284 +	char* mem;
1285 +
1286 +	buf = kmalloc(sizeof(*buf), GFP_KERNEL);
1287 +	if (!buf)
1288 +		return NULL;
1289 +
1290 +
1291 +	mem = vmalloc(total);
1292 +
1293 +	if (!mem) {
1294 +		kfree(buf);
1295 +		return NULL;
1296 +	}
1297 +
1298 +	if (!init_ft_buffer(buf, count, size,
1299 +			    mem + (count * size),  /* markers at the end */
1300 +			    mem)) {                /* buffer objects     */
1301 +		vfree(mem);
1302 +		kfree(buf);
1303 +		return NULL;
1304 +	}
1305 +	return buf;
1306 +}
1307 +
1308 +void free_ft_buffer(struct ft_buffer* buf)
1309 +{
1310 +	if (buf) {
1311 +		vfree(buf->buffer_mem);
1312 +		kfree(buf);
1313 +	}
1314 +}
1315 +
1316 +struct ftdev_event {
1317 +	int id;
1318 +	struct ftdev_event* next;
1319 +};
1320 +
1321 +static int activate(struct ftdev_event** chain, int id)
1322 +{
1323 +	struct ftdev_event* ev = kmalloc(sizeof(*ev), GFP_KERNEL);
1324 +	if (ev) {
1325 +		printk(KERN_INFO
1326 +		       "Enabling feather-trace event %d.\n", (int) id);
1327 +		ft_enable_event(id);
1328 +		ev->id = id;
1329 +		ev->next = *chain;
1330 +		*chain    = ev;
1331 +	}
1332 +	return ev ? 0 : -ENOMEM;
1333 +}
1334 +
1335 +static void deactivate(struct ftdev_event** chain, int id)
1336 +{
1337 +	struct ftdev_event **cur = chain;
1338 +	struct ftdev_event *nxt;
1339 +	while (*cur) {
1340 +		if ((*cur)->id == id) {
1341 +			nxt   = (*cur)->next;
1342 +			kfree(*cur);
1343 +			*cur  = nxt;
1344 +			printk(KERN_INFO
1345 +			       "Disabling feather-trace event %d.\n", (int) id);
1346 +			ft_disable_event(id);
1347 +			break;
1348 +		}
1349 +		cur = &(*cur)->next;
1350 +	}
1351 +}
1352 +
1353 +static int ftdev_open(struct inode *in, struct file *filp)
1354 +{
1355 +	struct ftdev* ftdev;
1356 +	struct ftdev_minor* ftdm;
1357 +	unsigned int buf_idx = iminor(in);
1358 +	int err = 0;
1359 +
1360 +	ftdev = container_of(in->i_cdev, struct ftdev, cdev);
1361 +
1362 +	if (buf_idx >= ftdev->minor_cnt) {
1363 +		err = -ENODEV;
1364 +		goto out;
1365 +	}
1366 +	if (ftdev->can_open && (err = ftdev->can_open(ftdev, buf_idx)))
1367 +		goto out;
1368 +
1369 +	ftdm = ftdev->minor + buf_idx;
1370 +	ftdm->ftdev = ftdev;
1371 +	filp->private_data = ftdm;
1372 +
1373 +	if (mutex_lock_interruptible(&ftdm->lock)) {
1374 +		err = -ERESTARTSYS;
1375 +		goto out;
1376 +	}
1377 +
1378 +	if (!ftdm->readers && ftdev->alloc)
1379 +		err = ftdev->alloc(ftdev, buf_idx);
1380 +	if (0 == err)
1381 +		ftdm->readers++;
1382 +
1383 +	mutex_unlock(&ftdm->lock);
1384 +out:
1385 +	return err;
1386 +}
1387 +
1388 +static int ftdev_release(struct inode *in, struct file *filp)
1389 +{
1390 +	struct ftdev* ftdev;
1391 +	struct ftdev_minor* ftdm;
1392 +	unsigned int buf_idx = iminor(in);
1393 +	int err = 0;
1394 +
1395 +	ftdev = container_of(in->i_cdev, struct ftdev, cdev);
1396 +
1397 +	if (buf_idx >= ftdev->minor_cnt) {
1398 +		err = -ENODEV;
1399 +		goto out;
1400 +	}
1401 +	ftdm = ftdev->minor + buf_idx;
1402 +
1403 +	if (mutex_lock_interruptible(&ftdm->lock)) {
1404 +		err = -ERESTARTSYS;
1405 +		goto out;
1406 +	}
1407 +
1408 +	if (ftdm->readers == 1) {
1409 +		while (ftdm->events)
1410 +			deactivate(&ftdm->events, ftdm->events->id);
1411 +
1412 +		/* wait for any pending events to complete */
1413 +		set_current_state(TASK_UNINTERRUPTIBLE);
1414 +		schedule_timeout(HZ);
1415 +
1416 +		printk(KERN_ALERT "Failed trace writes: %u\n",
1417 +		       ftdm->buf->failed_writes);
1418 +
1419 +		if (ftdev->free)
1420 +			ftdev->free(ftdev, buf_idx);
1421 +	}
1422 +
1423 +	ftdm->readers--;
1424 +	mutex_unlock(&ftdm->lock);
1425 +out:
1426 +	return err;
1427 +}
1428 +
1429 +/* based on ft_buffer_read
1430 + * @returns < 0 : page fault
1431 + *          = 0 : no data available
1432 + *          = 1 : one slot copied
1433 + */
1434 +static int ft_buffer_copy_to_user(struct ft_buffer* buf, char __user *dest)
1435 +{
1436 +	unsigned int idx;
1437 +	int err = 0;
1438 +	if (buf->free_count != buf->slot_count) {
1439 +		/* data available */
1440 +		idx = buf->read_idx % buf->slot_count;
1441 +		if (buf->slots[idx] == SLOT_READY) {
1442 +			err = copy_to_user(dest, ((char*) buf->buffer_mem) +
1443 +					   idx * buf->slot_size,
1444 +					   buf->slot_size);
1445 +			if (err == 0) {
1446 +				/* copy ok */
1447 +				buf->slots[idx] = SLOT_FREE;
1448 +				buf->read_idx++;
1449 +				fetch_and_inc(&buf->free_count);
1450 +				err = 1;
1451 +			}
1452 +		}
1453 +	}
1454 +	return err;
1455 +}
1456 +
1457 +static ssize_t ftdev_read(struct file *filp,
1458 +			  char __user *to, size_t len, loff_t *f_pos)
1459 +{
1460 +	/* 	we ignore f_pos, this is strictly sequential */
1461 +
1462 +	ssize_t err = 0;
1463 +	size_t chunk;
1464 +	int copied;
1465 +	struct ftdev_minor* ftdm = filp->private_data;
1466 +
1467 +	if (mutex_lock_interruptible(&ftdm->lock)) {
1468 +		err = -ERESTARTSYS;
1469 +		goto out;
1470 +	}
1471 +
1472 +
1473 +	chunk = ftdm->buf->slot_size;
1474 +	while (len >= chunk) {
1475 +		copied = ft_buffer_copy_to_user(ftdm->buf, to);
1476 +		if (copied == 1) {
1477 +			len    -= chunk;
1478 +			to     += chunk;
1479 +			err    += chunk;
1480 +	        } else if (err == 0 && copied == 0 && ftdm->events) {
1481 +			/* Only wait if there are any events enabled and only
1482 +			 * if we haven't copied some data yet. We cannot wait
1483 +			 * here with copied data because that data would get
1484 +			 * lost if the task is interrupted (e.g., killed).
1485 +			 */
1486 +			mutex_unlock(&ftdm->lock);
1487 +			set_current_state(TASK_INTERRUPTIBLE);
1488 +
1489 +			schedule_timeout(50);
1490 +
1491 +			if (signal_pending(current)) {
1492 +				if (err == 0)
1493 +					/* nothing read yet, signal problem */
1494 +					err = -ERESTARTSYS;
1495 +				goto out;
1496 +			}
1497 +			if (mutex_lock_interruptible(&ftdm->lock)) {
1498 +				err = -ERESTARTSYS;
1499 +				goto out;
1500 +			}
1501 +		} else if (copied < 0) {
1502 +			/* page fault */
1503 +			err = copied;
1504 +			break;
1505 +		} else
1506 +			/* nothing left to get, return to user space */
1507 +			break;
1508 +	}
1509 +	mutex_unlock(&ftdm->lock);
1510 +out:
1511 +	return err;
1512 +}
1513 +
1514 +static long ftdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1515 +{
1516 +	long err = -ENOIOCTLCMD;
1517 +	struct ftdev_minor* ftdm = filp->private_data;
1518 +
1519 +	if (mutex_lock_interruptible(&ftdm->lock)) {
1520 +		err = -ERESTARTSYS;
1521 +		goto out;
1522 +	}
1523 +
1524 +	/* FIXME: check id against list of acceptable events */
1525 +
1526 +	switch (cmd) {
1527 +	case  FTDEV_ENABLE_CMD:
1528 +		if (activate(&ftdm->events, arg))
1529 +			err = -ENOMEM;
1530 +		else
1531 +			err = 0;
1532 +		break;
1533 +
1534 +	case FTDEV_DISABLE_CMD:
1535 +		deactivate(&ftdm->events, arg);
1536 +		err = 0;
1537 +		break;
1538 +
1539 +	case FTDEV_CALIBRATE:
1540 +		if (ftdm->ftdev->calibrate) {
1541 +			err = ftdm->ftdev->calibrate(ftdm->ftdev, iminor(filp->f_dentry->d_inode), arg);
1542 +		}
1543 +		break;
1544 +
1545 +	default:
1546 +		printk(KERN_DEBUG "ftdev: strange ioctl (%u, %lu)\n", cmd, arg);
1547 +	};
1548 +
1549 +	mutex_unlock(&ftdm->lock);
1550 +out:
1551 +	return err;
1552 +}
1553 +
1554 +static ssize_t ftdev_write(struct file *filp, const char __user *from,
1555 +			   size_t len, loff_t *f_pos)
1556 +{
1557 +	struct ftdev_minor* ftdm = filp->private_data;
1558 +	ssize_t err = -EINVAL;
1559 +	struct ftdev* ftdev = ftdm->ftdev;
1560 +
1561 +	/* dispatch write to buffer-specific code, if available */
1562 +	if (ftdev->write)
1563 +		err = ftdev->write(ftdm->buf, len, from);
1564 +
1565 +	return err;
1566 +}
1567 +
1568 +struct file_operations ftdev_fops = {
1569 +	.owner   = THIS_MODULE,
1570 +	.open    = ftdev_open,
1571 +	.release = ftdev_release,
1572 +	.write   = ftdev_write,
1573 +	.read    = ftdev_read,
1574 +	.unlocked_ioctl = ftdev_ioctl,
1575 +};
1576 +
1577 +int ftdev_init(	struct ftdev* ftdev, struct module* owner,
1578 +		const int minor_cnt, const char* name)
1579 +{
1580 +	int i, err;
1581 +
1582 +	BUG_ON(minor_cnt < 1);
1583 +
1584 +	cdev_init(&ftdev->cdev, &ftdev_fops);
1585 +	ftdev->name = name;
1586 +	ftdev->minor_cnt = minor_cnt;
1587 +	ftdev->cdev.owner = owner;
1588 +	ftdev->cdev.ops = &ftdev_fops;
1589 +	ftdev->alloc    = NULL;
1590 +	ftdev->free     = NULL;
1591 +	ftdev->can_open = NULL;
1592 +	ftdev->write	= NULL;
1593 +	ftdev->calibrate = NULL;
1594 +
1595 +	ftdev->minor = kcalloc(ftdev->minor_cnt, sizeof(*ftdev->minor),
1596 +			GFP_KERNEL);
1597 +	if (!ftdev->minor) {
1598 +		printk(KERN_WARNING "ftdev(%s): Could not allocate memory\n",
1599 +			ftdev->name);
1600 +		err = -ENOMEM;
1601 +		goto err_out;
1602 +	}
1603 +
1604 +	for (i = 0; i < ftdev->minor_cnt; i++) {
1605 +		mutex_init(&ftdev->minor[i].lock);
1606 +		ftdev->minor[i].readers = 0;
1607 +		ftdev->minor[i].buf     = NULL;
1608 +		ftdev->minor[i].events  = NULL;
1609 +	}
1610 +
1611 +	ftdev->class = class_create(owner, ftdev->name);
1612 +	if (IS_ERR(ftdev->class)) {
1613 +		err = PTR_ERR(ftdev->class);
1614 +		printk(KERN_WARNING "ftdev(%s): "
1615 +			"Could not create device class.\n", ftdev->name);
1616 +		goto err_dealloc;
1617 +	}
1618 +
1619 +	return 0;
1620 +
1621 +err_dealloc:
1622 +	kfree(ftdev->minor);
1623 +err_out:
1624 +	return err;
1625 +}
1626 +
1627 +/*
1628 + * Destroy minor devices up to, but not including, up_to.
1629 + */
1630 +static void ftdev_device_destroy(struct ftdev* ftdev, unsigned int up_to)
1631 +{
1632 +	dev_t minor_cntr;
1633 +
1634 +	if (up_to < 1)
1635 +		up_to = (ftdev->minor_cnt < 1) ? 0 : ftdev->minor_cnt;
1636 +
1637 +	for (minor_cntr = 0; minor_cntr < up_to; ++minor_cntr)
1638 +		device_destroy(ftdev->class, MKDEV(ftdev->major, minor_cntr));
1639 +}
1640 +
1641 +void ftdev_exit(struct ftdev* ftdev)
1642 +{
1643 +	printk("ftdev(%s): Exiting\n", ftdev->name);
1644 +	ftdev_device_destroy(ftdev, -1);
1645 +	cdev_del(&ftdev->cdev);
1646 +	unregister_chrdev_region(MKDEV(ftdev->major, 0), ftdev->minor_cnt);
1647 +	class_destroy(ftdev->class);
1648 +	kfree(ftdev->minor);
1649 +}
1650 +
1651 +int register_ftdev(struct ftdev* ftdev)
1652 +{
1653 +	struct device **device;
1654 +	dev_t trace_dev_tmp, minor_cntr;
1655 +	int err;
1656 +
1657 +	err = alloc_chrdev_region(&trace_dev_tmp, 0, ftdev->minor_cnt,
1658 +			ftdev->name);
1659 +	if (err) {
1660 +		printk(KERN_WARNING "ftdev(%s): "
1661 +		       "Could not allocate char. device region (%d minors)\n",
1662 +		       ftdev->name, ftdev->minor_cnt);
1663 +		goto err_out;
1664 +	}
1665 +
1666 +	ftdev->major = MAJOR(trace_dev_tmp);
1667 +
1668 +	err = cdev_add(&ftdev->cdev, trace_dev_tmp, ftdev->minor_cnt);
1669 +	if (err) {
1670 +		printk(KERN_WARNING "ftdev(%s): "
1671 +		       "Could not add cdev for major %u with %u minor(s).\n",
1672 +		       ftdev->name, ftdev->major, ftdev->minor_cnt);
1673 +		goto err_unregister;
1674 +	}
1675 +
1676 +	/* create the minor device(s) */
1677 +	for (minor_cntr = 0; minor_cntr < ftdev->minor_cnt; ++minor_cntr)
1678 +	{
1679 +		trace_dev_tmp = MKDEV(ftdev->major, minor_cntr);
1680 +		device = &ftdev->minor[minor_cntr].device;
1681 +
1682 +		*device = device_create(ftdev->class, NULL, trace_dev_tmp, NULL,
1683 +				"litmus/%s%d", ftdev->name, minor_cntr);
1684 +		if (IS_ERR(*device)) {
1685 +			err = PTR_ERR(*device);
1686 +			printk(KERN_WARNING "ftdev(%s): "
1687 +				"Could not create device major/minor number "
1688 +				"%u/%u\n", ftdev->name, ftdev->major,
1689 +				minor_cntr);
1690 +			printk(KERN_WARNING "ftdev(%s): "
1691 +				"will attempt deletion of allocated devices.\n",
1692 +				ftdev->name);
1693 +			goto err_minors;
1694 +		}
1695 +	}
1696 +
1697 +	return 0;
1698 +
1699 +err_minors:
1700 +	ftdev_device_destroy(ftdev, minor_cntr);
1701 +	cdev_del(&ftdev->cdev);
1702 +err_unregister:
1703 +	unregister_chrdev_region(MKDEV(ftdev->major, 0), ftdev->minor_cnt);
1704 +err_out:
1705 +	return err;
1706 +}
1707 diff --git a/litmus/trace.c b/litmus/trace.c
1708 new file mode 100644
1709 index 0000000..378b657
1710 --- /dev/null
1711 +++ b/litmus/trace.c
1712 @@ -0,0 +1,582 @@
1713 +#include <linux/sched.h>
1714 +#include <linux/module.h>
1715 +#include <linux/uaccess.h>
1716 +
1717 +#include <litmus/ftdev.h>
1718 +#include <litmus/trace.h>
1719 +
1720 +/******************************************************************************/
1721 +/*                          Allocation                                        */
1722 +/******************************************************************************/
1723 +
1724 +static struct ftdev overhead_dev;
1725 +static struct ftdev cpu_overhead_dev;
1726 +
1727 +#define trace_ts_buf overhead_dev.minor[0].buf
1728 +
1729 +#define cpu_trace_ts_buf(cpu) cpu_overhead_dev.minor[(cpu)].buf
1730 +
1731 +static unsigned int ts_seq_no = 0;
1732 +
1733 +DEFINE_PER_CPU(atomic_t, irq_fired_count;)
1734 +DEFINE_PER_CPU(atomic_t, cpu_irq_fired_count);
1735 +
1736 +static DEFINE_PER_CPU(unsigned int, cpu_ts_seq_no);
1737 +
1738 +static int64_t cycle_offset[NR_CPUS][NR_CPUS];
1739 +
1740 +void ft_irq_fired(void)
1741 +{
1742 +	/* Only called with preemptions disabled.  */
1743 +	atomic_inc(&__get_cpu_var(irq_fired_count));
1744 +	atomic_inc(&__get_cpu_var(cpu_irq_fired_count));
1745 +}
1746 +
1747 +static inline void clear_irq_fired(void)
1748 +{
1749 +	atomic_set(&__raw_get_cpu_var(irq_fired_count), 0);
1750 +}
1751 +
1752 +static inline unsigned int get_and_clear_irq_fired(void)
1753 +{
1754 +	/* This is potentially not atomic  since we might migrate if
1755 +	 * preemptions are not disabled. As a tradeoff between
1756 +	 * accuracy and tracing overheads, this seems acceptable.
1757 +	 * If it proves to be a problem, then one could add a callback
1758 +	 * from the migration code to invalidate irq_fired_count.
1759 +	 */
1760 +	return atomic_xchg(&__raw_get_cpu_var(irq_fired_count), 0);
1761 +}
1762 +
1763 +static inline unsigned int get_and_clear_irq_fired_for_cpu(int cpu)
1764 +{
1765 +	return atomic_xchg(&per_cpu(irq_fired_count, cpu), 0);
1766 +}
1767 +
1768 +static inline void cpu_clear_irq_fired(void)
1769 +{
1770 +	atomic_set(&__raw_get_cpu_var(cpu_irq_fired_count), 0);
1771 +}
1772 +
1773 +static inline unsigned int cpu_get_and_clear_irq_fired(void)
1774 +{
1775 +	return atomic_xchg(&__raw_get_cpu_var(cpu_irq_fired_count), 0);
1776 +}
1777 +
1778 +static inline void save_irq_flags(struct timestamp *ts, unsigned int irq_count)
1779 +{
1780 +	/* Store how many interrupts occurred. */
1781 +	ts->irq_count = irq_count;
1782 +	/* Extra flag because ts->irq_count overflows quickly. */
1783 +	ts->irq_flag  = irq_count > 0;
1784 +}
1785 +
1786 +#define NO_IRQ_COUNT 0
1787 +#define LOCAL_IRQ_COUNT 1
1788 +#define REMOTE_IRQ_COUNT 2
1789 +
1790 +#define DO_NOT_RECORD_TIMESTAMP 0
1791 +#define RECORD_LOCAL_TIMESTAMP 1
1792 +#define RECORD_OFFSET_TIMESTAMP 2
1793 +
1794 +static inline void write_timestamp(uint8_t event,
1795 +				   uint8_t type,
1796 +				   uint8_t cpu,
1797 +				   uint16_t pid_fragment,
1798 +				   unsigned int irq_count,
1799 +				   int record_irq,
1800 +				   int hide_irq,
1801 +				   uint64_t timestamp,
1802 +				   int record_timestamp)
1803 +{
1804 +	unsigned long flags;
1805 +	unsigned int seq_no;
1806 +	struct timestamp *ts;
1807 +
1808 +	/* Avoid preemptions while recording the timestamp. This reduces the
1809 +	 * number of "out of order" timestamps in the stream and makes
1810 +	 * post-processing easier. */
1811 +
1812 +	local_irq_save(flags);
1813 +
1814 +	seq_no = fetch_and_inc((int *) &ts_seq_no);
1815 +	if (ft_buffer_start_write(trace_ts_buf, (void**)  &ts)) {
1816 +		ts->event     = event;
1817 +		ts->seq_no    = seq_no;
1818 +
1819 +		ts->task_type = type;
1820 +		ts->pid	      = pid_fragment;
1821 +
1822 +		ts->cpu       = cpu;
1823 +
1824 +		if (likely(record_irq == LOCAL_IRQ_COUNT))
1825 +			irq_count = get_and_clear_irq_fired();
1826 +		else if (record_irq == REMOTE_IRQ_COUNT)
1827 +			irq_count = get_and_clear_irq_fired_for_cpu(cpu);
1828 +
1829 +		save_irq_flags(ts, irq_count - hide_irq);
1830 +
1831 +		if (record_timestamp)
1832 +			timestamp = ft_timestamp();
1833 +		if (record_timestamp == RECORD_OFFSET_TIMESTAMP)
1834 +			timestamp += cycle_offset[smp_processor_id()][cpu];
1835 +
1836 +		ts->timestamp = timestamp;
1837 +		ft_buffer_finish_write(trace_ts_buf, ts);
1838 +	}
1839 +
1840 +	local_irq_restore(flags);
1841 +}
1842 +
1843 +static inline void write_cpu_timestamp(
1844 +	uint8_t event,
1845 +	uint8_t type,
1846 +	uint16_t pid_fragment,
1847 +	unsigned int irq_count,
1848 +	int record_irq,
1849 +	int hide_irq,
1850 +	uint64_t timestamp,
1851 +	int record_timestamp)
1852 +{
1853 +	unsigned long flags;
1854 +	unsigned int seq_no;
1855 +	struct timestamp *ts;
1856 +	int cpu;
1857 +	struct ft_buffer* buf;
1858 +
1859 +	local_irq_save(flags);
1860 +	cpu = smp_processor_id();
1861 +
1862 +	seq_no = __get_cpu_var(cpu_ts_seq_no)++;
1863 +
1864 +	buf = cpu_trace_ts_buf(cpu);
1865 +	/* If buf is non-NULL here, then the buffer cannot be deallocated until
1866 +	 * we turn interrupts on again. This is because free_timestamp_buffer()
1867 +	 * indirectly causes TLB invalidations due to modifications of the
1868 +	 * kernel address space, namely via vfree() in free_ft_buffer(), which
1869 +	 * cannot be processed until we turn on interrupts again.
1870 +	 */
1871 +
1872 +	if (buf && ft_buffer_start_single_write(buf, (void**)  &ts)) {
1873 +		ts->event     = event;
1874 +		ts->seq_no    = seq_no;
1875 +
1876 +		ts->task_type = type;
1877 +		ts->pid	      = pid_fragment;
1878 +
1879 +		ts->cpu       = cpu;
1880 +
1881 +		if (record_irq)
1882 +			irq_count = cpu_get_and_clear_irq_fired();
1883 +
1884 +		save_irq_flags(ts, irq_count - hide_irq);
1885 +
1886 +		if (record_timestamp)
1887 +			timestamp = ft_timestamp();
1888 +
1889 +		ts->timestamp = timestamp;
1890 +		ft_buffer_finish_write(buf, ts);
1891 +	}
1892 +
1893 +	local_irq_restore(flags);
1894 +}
1895 +
1896 +static void __add_timestamp_user(struct timestamp *pre_recorded)
1897 +{
1898 +	unsigned long flags;
1899 +	unsigned int seq_no;
1900 +	struct timestamp *ts;
1901 +
1902 +
1903 +	local_irq_save(flags);
1904 +
1905 +	seq_no = fetch_and_inc((int *) &ts_seq_no);
1906 +	if (ft_buffer_start_write(trace_ts_buf, (void**)  &ts)) {
1907 +		*ts = *pre_recorded;
1908 +		ts->seq_no = seq_no;
1909 +		ts->cpu	   = raw_smp_processor_id();
1910 +	        save_irq_flags(ts, get_and_clear_irq_fired());
1911 +		ft_buffer_finish_write(trace_ts_buf, ts);
1912 +	}
1913 +
1914 +	local_irq_restore(flags);
1915 +}
1916 +
1917 +feather_callback void save_timestamp(unsigned long event)
1918 +{
1919 +	write_timestamp(event, TSK_UNKNOWN,
1920 +			raw_smp_processor_id(),
1921 +			current->pid,
1922 +			0, 1, 0,
1923 +			0, 1);
1924 +}
1925 +
1926 +
1927 +feather_callback void save_cpu_timestamp(unsigned long event)
1928 +{
1929 +	write_cpu_timestamp(event, TSK_UNKNOWN,
1930 +			    current->pid,
1931 +			    0, 1, 0,
1932 +			    0, 1);
1933 +}
1934 +
1935 +feather_callback void save_timestamp_def(unsigned long event,
1936 +					 unsigned long type)
1937 +{
1938 +	write_timestamp(event, type,
1939 +			raw_smp_processor_id(),
1940 +			current->pid,
1941 +			0, 1, 0,
1942 +			0, 1);
1943 +}
1944 +
1945 +feather_callback void save_cpu_timestamp_def(unsigned long event,
1946 +					     unsigned long type)
1947 +{
1948 +	write_cpu_timestamp(event, type,
1949 +			    current->pid,
1950 +			    0, 1, 0,
1951 +			    0, 1);
1952 +}
1953 +
1954 +feather_callback void save_timestamp_task(unsigned long event,
1955 +					  unsigned long t_ptr)
1956 +{
1957 +	struct task_struct *t = (struct task_struct *) t_ptr;
1958 +	int rt = is_realtime(t);
1959 +
1960 +	write_timestamp(event, rt ? TSK_RT : TSK_BE,
1961 +			raw_smp_processor_id(),
1962 +			t->pid,
1963 +			0, 1, 0,
1964 +			0, 1);
1965 +}
1966 +
1967 +feather_callback void save_cpu_timestamp_task(unsigned long event,
1968 +					      unsigned long t_ptr)
1969 +{
1970 +	struct task_struct *t = (struct task_struct *) t_ptr;
1971 +	int rt = is_realtime(t);
1972 +
1973 +	write_cpu_timestamp(event, rt ? TSK_RT : TSK_BE,
1974 +			    t->pid,
1975 +			    0, 1, 0,
1976 +			    0, 1);
1977 +}
1978 +
1979 +feather_callback void save_timestamp_cpu(unsigned long event,
1980 +					 unsigned long cpu)
1981 +{
1982 +	write_timestamp(event, TSK_UNKNOWN, cpu, current->pid,
1983 +			0, REMOTE_IRQ_COUNT, 0,
1984 +			0, RECORD_OFFSET_TIMESTAMP);
1985 +}
1986 +
1987 +feather_callback void save_cpu_task_latency(unsigned long event,
1988 +					    unsigned long when_ptr)
1989 +{
1990 +	lt_t now = litmus_clock();
1991 +	lt_t *when = (lt_t*) when_ptr;
1992 +
1993 +	write_cpu_timestamp(event, TSK_RT,
1994 +			    0,
1995 +			    0, 1, 0,
1996 +			    now - *when, 0);
1997 +}
1998 +
1999 +/* fake timestamp to user-reported time */
2000 +feather_callback void save_timestamp_time(unsigned long event,
2001 +			 unsigned long ptr)
2002 +{
2003 +	uint64_t* time = (uint64_t*) ptr;
2004 +
2005 +	write_timestamp(event, is_realtime(current) ? TSK_RT : TSK_BE,
2006 +			raw_smp_processor_id(), current->pid,
2007 +			0, 1, 0,
2008 +			*time, 0);
2009 +}
2010 +
2011 +/* Record user-reported IRQ count */
2012 +feather_callback void save_timestamp_irq(unsigned long event,
2013 +			unsigned long irq_counter_ptr)
2014 +{
2015 +	uint64_t* irqs = (uint64_t*) irq_counter_ptr;
2016 +
2017 +	write_timestamp(event, is_realtime(current) ? TSK_RT : TSK_BE,
2018 +			raw_smp_processor_id(), current->pid,
2019 +			*irqs, 0, 0,
2020 +			0, 1);
2021 +}
2022 +
2023 +/* Suppress one IRQ from the irq count. Used by TS_SEND_RESCHED_END, which is
2024 + * called from within an interrupt that is expected. */
2025 +feather_callback void save_timestamp_hide_irq(unsigned long event)
2026 +{
2027 +	write_timestamp(event, is_realtime(current) ? TSK_RT : TSK_BE,
2028 +			raw_smp_processor_id(), current->pid,
2029 +			0, LOCAL_IRQ_COUNT, 1,
2030 +			0, 1);
2031 +}
2032 +
2033 +/* Suppress one IRQ from the irq count. */
2034 +feather_callback void save_timestamp_hide_irq_task(unsigned long event,
2035 +					  unsigned long t_ptr)
2036 +{
2037 +	struct task_struct *t = (struct task_struct *) t_ptr;
2038 +	int rt = is_realtime(t);
2039 +
2040 +	write_timestamp(event, rt ? TSK_RT : TSK_BE,
2041 +			raw_smp_processor_id(), t->pid,
2042 +			0, 1, 1,
2043 +			0, 1);
2044 +
2045 +}
2046 +
2047 +/* Suppress one IRQ from the irq count. */
2048 +feather_callback void save_timestamp_hide_irq_cpu(unsigned long event,
2049 +					 unsigned long cpu)
2050 +{
2051 +	write_timestamp(event, TSK_UNKNOWN, cpu, current->pid,
2052 +			0, 1, 1,
2053 +			0, 1);
2054 +}
2055 +
2056 +/******************************************************************************/
2057 +/*                        DEVICE FILE DRIVER                                  */
2058 +/******************************************************************************/
2059 +
2060 +struct calibrate_info {
2061 +	atomic_t ready;
2062 +
2063 +	uint64_t cycle_count;
2064 +};
2065 +
2066 +static void calibrate_helper(void *_info)
2067 +{
2068 +	struct calibrate_info *info = _info;
2069 +	/* check in with master */
2070 +	atomic_inc(&info->ready);
2071 +
2072 +	/* wait for master to signal start */
2073 +	while (atomic_read(&info->ready))
2074 +		cpu_relax();
2075 +
2076 +	/* report time stamp */
2077 +	info->cycle_count = ft_timestamp();
2078 +
2079 +	/* tell master that we are done */
2080 +	atomic_inc(&info->ready);
2081 +}
2082 +
2083 +
2084 +static int64_t calibrate_cpu(int cpu)
2085 +{
2086 +	uint64_t cycles;
2087 +	struct calibrate_info info;
2088 +	unsigned long flags;
2089 +	int64_t  delta;
2090 +
2091 +	atomic_set(&info.ready, 0);
2092 +	info.cycle_count = 0;
2093 +	smp_wmb();
2094 +
2095 +	smp_call_function_single(cpu, calibrate_helper, &info, 0);
2096 +
2097 +	/* wait for helper to become active */
2098 +	while (!atomic_read(&info.ready))
2099 +		cpu_relax();
2100 +
2101 +	/* avoid interrupt interference */
2102 +	local_irq_save(flags);
2103 +
2104 +	/* take measurement */
2105 +	atomic_set(&info.ready, 0);
2106 +	smp_wmb();
2107 +	cycles = ft_timestamp();
2108 +
2109 +	/* wait for helper reading */
2110 +	while (!atomic_read(&info.ready))
2111 +		cpu_relax();
2112 +
2113 +	/* positive offset: the other guy is ahead of us */
2114 +	delta  = (int64_t) info.cycle_count;
2115 +	delta -= (int64_t) cycles;
2116 +
2117 +	local_irq_restore(flags);
2118 +
2119 +	return delta;
2120 +}
2121 +
2122 +#define NUM_SAMPLES 10
2123 +
2124 +static long calibrate_tsc_offsets(struct ftdev* ftdev, unsigned int idx,
2125 +				  unsigned long uarg)
2126 +{
2127 +	int cpu, self, i;
2128 +	int64_t delta, sample;
2129 +
2130 +	preempt_disable();
2131 +	self = smp_processor_id();
2132 +
2133 +	if (uarg)
2134 +		printk(KERN_INFO "Feather-Trace: determining TSC offsets for P%d\n", self);
2135 +
2136 +	for_each_online_cpu(cpu)
2137 +		if (cpu != self) {
2138 +			delta = calibrate_cpu(cpu);
2139 +			for (i = 1; i < NUM_SAMPLES; i++) {
2140 +			        sample = calibrate_cpu(cpu);
2141 +				delta = sample < delta ? sample : delta;
2142 +			}
2143 +
2144 +			cycle_offset[self][cpu] = delta;
2145 +
2146 +			if (uarg)
2147 +				printk(KERN_INFO "Feather-Trace: TSC offset for P%d->P%d is %lld cycles.\n",
2148 +				       self, cpu, cycle_offset[self][cpu]);
2149 +		}
2150 +
2151 +	preempt_enable();
2152 +	return 0;
2153 +}
2154 +
2155 +#define NO_TIMESTAMPS (2 << CONFIG_SCHED_OVERHEAD_TRACE_SHIFT)
2156 +
2157 +static int alloc_timestamp_buffer(struct ftdev* ftdev, unsigned int idx)
2158 +{
2159 +	unsigned int count = NO_TIMESTAMPS;
2160 +
2161 +	/* An overhead-tracing timestamp should be exactly 16 bytes long. */
2162 +	BUILD_BUG_ON(sizeof(struct timestamp) != 16);
2163 +
2164 +	while (count && !ftdev->minor[idx].buf) {
2165 +		printk("time stamp buffer: trying to allocate %u time stamps for minor=%u.\n", count, idx);
2166 +		ftdev->minor[idx].buf = alloc_ft_buffer(count, sizeof(struct timestamp));
2167 +		count /= 2;
2168 +	}
2169 +	return ftdev->minor[idx].buf ? 0 : -ENOMEM;
2170 +}
2171 +
2172 +static void free_timestamp_buffer(struct ftdev* ftdev, unsigned int idx)
2173 +{
2174 +	ftdev->minor[idx].buf = NULL;
2175 +	/* Make sure all cores have actually seen buf == NULL before
2176 +	 * yanking out the mappings from underneath them. */
2177 +	smp_wmb();
2178 +	free_ft_buffer(ftdev->minor[idx].buf);
2179 +}
2180 +
2181 +static ssize_t write_timestamp_from_user(struct ft_buffer* buf, size_t len,
2182 +					 const char __user *from)
2183 +{
2184 +	ssize_t consumed = 0;
2185 +	struct timestamp ts;
2186 +
2187 +	/* don't give us partial timestamps */
2188 +	if (len % sizeof(ts))
2189 +		return -EINVAL;
2190 +
2191 +	while (len >= sizeof(ts)) {
2192 +		if (copy_from_user(&ts, from, sizeof(ts))) {
2193 +			consumed = -EFAULT;
2194 +			goto out;
2195 +		}
2196 +		len  -= sizeof(ts);
2197 +		from += sizeof(ts);
2198 +		consumed += sizeof(ts);
2199 +
2200 +		__add_timestamp_user(&ts);
2201 +	}
2202 +
2203 +out:
2204 +	return consumed;
2205 +}
2206 +
2207 +
2208 +static int __init init_global_ft_overhead_trace(void)
2209 +{
2210 +	int err;
2211 +
2212 +	printk("Initializing Feather-Trace overhead tracing device.\n");
2213 +	err = ftdev_init(&overhead_dev, THIS_MODULE, 1, "ft_trace");
2214 +	if (err)
2215 +		goto err_out;
2216 +
2217 +	overhead_dev.alloc = alloc_timestamp_buffer;
2218 +	overhead_dev.free  = free_timestamp_buffer;
2219 +	overhead_dev.write = write_timestamp_from_user;
2220 +	overhead_dev.calibrate = calibrate_tsc_offsets;
2221 +
2222 +	err = register_ftdev(&overhead_dev);
2223 +	if (err)
2224 +		goto err_dealloc;
2225 +
2226 +	return 0;
2227 +
2228 +err_dealloc:
2229 +	ftdev_exit(&overhead_dev);
2230 +err_out:
2231 +	printk(KERN_WARNING "Could not register ft_trace module.\n");
2232 +	return err;
2233 +}
2234 +
2235 +static int __init init_cpu_ft_overhead_trace(void)
2236 +{
2237 +	int err, cpu;
2238 +
2239 +	printk("Initializing Feather-Trace per-cpu overhead tracing device.\n");
2240 +	err = ftdev_init(&cpu_overhead_dev, THIS_MODULE,
2241 +			 num_online_cpus(), "ft_cpu_trace");
2242 +	if (err)
2243 +		goto err_out;
2244 +
2245 +	cpu_overhead_dev.alloc = alloc_timestamp_buffer;
2246 +	cpu_overhead_dev.free  = free_timestamp_buffer;
2247 +
2248 +	err = register_ftdev(&cpu_overhead_dev);
2249 +	if (err)
2250 +		goto err_dealloc;
2251 +
2252 +	for (cpu = 0; cpu < NR_CPUS; cpu++)  {
2253 +		per_cpu(cpu_ts_seq_no, cpu) = 0;
2254 +	}
2255 +
2256 +	return 0;
2257 +
2258 +err_dealloc:
2259 +	ftdev_exit(&cpu_overhead_dev);
2260 +err_out:
2261 +	printk(KERN_WARNING "Could not register per-cpu ft_trace module.\n");
2262 +	return err;
2263 +}
2264 +
2265 +
2266 +static int __init init_ft_overhead_trace(void)
2267 +{
2268 +	int err, i, j;
2269 +
2270 +	for (i = 0; i < NR_CPUS; i++)
2271 +		for (j = 0; j < NR_CPUS; j++)
2272 +			cycle_offset[i][j] = 0;
2273 +
2274 +	err = init_global_ft_overhead_trace();
2275 +	if (err)
2276 +		return err;
2277 +
2278 +	err = init_cpu_ft_overhead_trace();
2279 +	if (err) {
2280 +		ftdev_exit(&overhead_dev);
2281 +		return err;
2282 +	}
2283 +
2284 +	return 0;
2285 +}
2286 +
2287 +static void __exit exit_ft_overhead_trace(void)
2288 +{
2289 +	ftdev_exit(&overhead_dev);
2290 +	ftdev_exit(&cpu_overhead_dev);
2291 +}
2292 +
2293 +module_init(init_ft_overhead_trace);
2294 +module_exit(exit_ft_overhead_trace);
2295