diff --git a/Makefile b/Makefile index cd11e88..90c4f57 100644 --- a/Makefile +++ b/Makefile @@ -733,7 +733,7 @@ export mod_sign_cmd ifeq ($(KBUILD_EXTMOD),) -core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ +core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ litmus/ vmlinux-dirs := $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \ $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 49d993c..8d27866 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -2221,3 +2221,9 @@ source "crypto/Kconfig" source "lib/Kconfig" source "arch/arm/kvm/Kconfig" + +config ARCH_HAS_FEATHER_TRACE + def_bool n + +source "litmus/Kconfig" + diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 685692c..9487544 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2345,3 +2345,8 @@ source "crypto/Kconfig" source "arch/x86/kvm/Kconfig" source "lib/Kconfig" + +config ARCH_HAS_FEATHER_TRACE + def_bool y + +source "litmus/Kconfig" diff --git a/arch/x86/include/asm/feather_trace.h b/arch/x86/include/asm/feather_trace.h new file mode 100644 index 0000000..4fd3163 --- /dev/null +++ b/arch/x86/include/asm/feather_trace.h @@ -0,0 +1,17 @@ +#ifndef _ARCH_FEATHER_TRACE_H +#define _ARCH_FEATHER_TRACE_H + +#include + +static inline unsigned long long ft_timestamp(void) +{ + return __native_read_tsc(); +} + +#ifdef CONFIG_X86_32 +#include "feather_trace_32.h" +#else +#include "feather_trace_64.h" +#endif + +#endif diff --git a/arch/x86/include/asm/feather_trace_32.h b/arch/x86/include/asm/feather_trace_32.h new file mode 100644 index 0000000..75e81a9 --- /dev/null +++ b/arch/x86/include/asm/feather_trace_32.h @@ -0,0 +1,115 @@ +/* Copyright (c) 2007-2012 Björn Brandenburg, + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Do not directly include this file. Include feather_trace.h instead */ + +#define feather_callback __attribute__((regparm(3))) __attribute__((used)) + +/* + * Make the compiler reload any register that is not saved in a cdecl function + * call (minus the registers that we explicitly clobber as output registers). + */ +#define __FT_CLOBBER_LIST0 "memory", "cc", "eax", "edx", "ecx" +#define __FT_CLOBBER_LIST1 "memory", "cc", "eax", "ecx" +#define __FT_CLOBBER_LIST2 "memory", "cc", "eax" +#define __FT_CLOBBER_LIST3 "memory", "cc", "eax" + +#define __FT_TMP1(x) "=d" (x) +#define __FT_ARG1(x) "0" ((long) (x)) +#define __FT_TMP2(x) "=c" (x) +#define __FT_ARG2(x) "1" ((long) (x)) + +#define __FT_ARG3(x) "r" ((long) (x)) + +#define ft_event(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " call " #callback " \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : : __FT_CLOBBER_LIST0) + +#define ft_event0(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movl $" #id ", %%eax \n\t" \ + " call " #callback " \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : : __FT_CLOBBER_LIST0) + +#define ft_event1(id, callback, param) \ + do { \ + long __ft_tmp1; \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movl $" #id ", %%eax \n\t" \ + " call " #callback " \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : __FT_TMP1(__ft_tmp1) \ + : __FT_ARG1(param) \ + : __FT_CLOBBER_LIST1); \ + } while (0); + +#define ft_event2(id, callback, param, param2) \ + do { \ + long __ft_tmp1, __ft_tmp2; \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movl $" #id ", %%eax \n\t" \ + " call " #callback " \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : __FT_TMP1(__ft_tmp1), __FT_TMP2(__ft_tmp2) \ + : __FT_ARG1(param), __FT_ARG2(param2) \ + : __FT_CLOBBER_LIST2); \ + } while (0); + + +#define ft_event3(id, callback, param, param2, param3) \ + do { \ + long __ft_tmp1, __ft_tmp2; \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " subl $4, %%esp \n\t" \ + " movl $" #id ", %%eax \n\t" \ + " movl %2, (%%esp) \n\t" \ + " call " #callback " \n\t" \ + " addl $4, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : __FT_TMP1(__ft_tmp1), __FT_TMP2(__ft_tmp2) \ + : __FT_ARG1(param), __FT_ARG2(param2), __FT_ARG3(param3) \ + : __FT_CLOBBER_LIST3); \ + } while (0); diff --git a/arch/x86/include/asm/feather_trace_64.h b/arch/x86/include/asm/feather_trace_64.h new file mode 100644 index 0000000..5ce49e2 --- /dev/null +++ b/arch/x86/include/asm/feather_trace_64.h @@ -0,0 +1,124 @@ +/* Copyright (c) 2010 Andrea Bastoni, + * Copyright (c) 2012 Björn Brandenburg, + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Do not directly include this file. Include feather_trace.h instead */ + +/* regparm is the default on x86_64 */ +#define feather_callback __attribute__((used)) + +#define __FT_EVENT_TABLE(id,from,to) \ + ".section __event_table, \"aw\"\n\t" \ + ".balign 8\n\t" \ + ".quad " #id ", 0, " #from ", " #to " \n\t" \ + ".previous \n\t" + +/* + * x86_64 caller only owns rbp, rbx, r12-r15; + * the callee can freely modify the others. + */ +#define __FT_CLOBBER_LIST0 "memory", "cc", "rdi", "rsi", "rdx", "rcx", \ + "r8", "r9", "r10", "r11", "rax" + +#define __FT_CLOBBER_LIST1 "memory", "cc", "rdi", "rdx", "rcx", \ + "r8", "r9", "r10", "r11", "rax" + +#define __FT_CLOBBER_LIST2 "memory", "cc", "rdi", "rcx", \ + "r8", "r9", "r10", "r11", "rax" + +#define __FT_CLOBBER_LIST3 "memory", "cc", "rdi", \ + "r8", "r9", "r10", "r11", "rax" + +/* The registers RDI, RSI, RDX, RCX, R8 and R9 are used for integer and pointer + * arguments. */ + +/* RSI */ +#define __FT_TMP1(x) "=S" (x) +#define __FT_ARG1(x) "0" ((long) (x)) + +/* RDX */ +#define __FT_TMP2(x) "=d" (x) +#define __FT_ARG2(x) "1" ((long) (x)) + +/* RCX */ +#define __FT_TMP3(x) "=c" (x) +#define __FT_ARG3(x) "2" ((long) (x)) + +#define ft_event(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " call " #callback " \n\t" \ + __FT_EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : : : __FT_CLOBBER_LIST0) + +#define ft_event0(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movq $" #id ", %%rdi \n\t" \ + " call " #callback " \n\t" \ + __FT_EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : : : __FT_CLOBBER_LIST0) + +#define ft_event1(id, callback, param) \ + do { \ + long __ft_tmp1; \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movq $" #id ", %%rdi \n\t" \ + " call " #callback " \n\t" \ + __FT_EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : __FT_TMP1(__ft_tmp1) \ + : __FT_ARG1(param) \ + : __FT_CLOBBER_LIST1); \ + } while (0); + +#define ft_event2(id, callback, param, param2) \ + do { \ + long __ft_tmp1, __ft_tmp2; \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movq $" #id ", %%rdi \n\t" \ + " call " #callback " \n\t" \ + __FT_EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : __FT_TMP1(__ft_tmp1), __FT_TMP2(__ft_tmp2) \ + : __FT_ARG1(param), __FT_ARG2(param2) \ + : __FT_CLOBBER_LIST2); \ + } while (0); + +#define ft_event3(id, callback, param, param2, param3) \ + do { \ + long __ft_tmp1, __ft_tmp2, __ft_tmp3; \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movq $" #id ", %%rdi \n\t" \ + " call " #callback " \n\t" \ + __FT_EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : __FT_TMP1(__ft_tmp1), __FT_TMP2(__ft_tmp2), __FT_TMP3(__ft_tmp3) \ + : __FT_ARG1(param), __FT_ARG2(param2), __FT_ARG3(param3) \ + : __FT_CLOBBER_LIST3); \ + } while (0); diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 7bd3bd3..d38a5a7 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -103,6 +103,8 @@ obj-$(CONFIG_UPROBES) += uprobes.o obj-$(CONFIG_PERF_EVENTS) += perf_regs.o +obj-$(CONFIG_FEATHER_TRACE) += ft_event.o + ### # 64 bit specific files ifeq ($(CONFIG_X86_64),y) diff --git a/arch/x86/kernel/ft_event.c b/arch/x86/kernel/ft_event.c new file mode 100644 index 0000000..37cc332 --- /dev/null +++ b/arch/x86/kernel/ft_event.c @@ -0,0 +1,118 @@ +#include + +#include + +/* the feather trace management functions assume + * exclusive access to the event table + */ + +#ifndef CONFIG_DEBUG_RODATA + +#define BYTE_JUMP 0xeb +#define BYTE_JUMP_LEN 0x02 + +/* for each event, there is an entry in the event table */ +struct trace_event { + long id; + long count; + long start_addr; + long end_addr; +}; + +extern struct trace_event __start___event_table[]; +extern struct trace_event __stop___event_table[]; + +/* Workaround: if no events are defined, then the event_table section does not + * exist and the above references cause linker errors. This could probably be + * fixed by adjusting the linker script, but it is easier to maintain for us if + * we simply create a dummy symbol in the event table section. + */ +int __event_table_dummy[0] __attribute__ ((section("__event_table"))); + +int ft_enable_event(unsigned long id) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->id == id && ++te->count == 1) { + instr = (unsigned char*) te->start_addr; + /* make sure we don't clobber something wrong */ + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + 1); + *delta = 0; + } + } + if (te->id == id) + count++; + te++; + } + + printk(KERN_DEBUG "ft_enable_event: enabled %d events\n", count); + return count; +} + +int ft_disable_event(unsigned long id) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->id == id && --te->count == 0) { + instr = (unsigned char*) te->start_addr; + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + 1); + *delta = te->end_addr - te->start_addr - + BYTE_JUMP_LEN; + } + } + if (te->id == id) + count++; + te++; + } + + printk(KERN_DEBUG "ft_disable_event: disabled %d events\n", count); + return count; +} + +int ft_disable_all_events(void) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->count) { + instr = (unsigned char*) te->start_addr; + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + + 1); + *delta = te->end_addr - te->start_addr - + BYTE_JUMP_LEN; + te->count = 0; + count++; + } + } + te++; + } + return count; +} + +int ft_is_event_enabled(unsigned long id) +{ + struct trace_event* te = __start___event_table; + + while (te < __stop___event_table) { + if (te->id == id) + return te->count; + te++; + } + return 0; +} + +#endif diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index 48d2b7d..5919042 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -24,6 +24,8 @@ #include #include +#include + #include #include #include @@ -123,6 +125,9 @@ static void native_smp_send_reschedule(int cpu) WARN_ON(1); return; } + + /* LITMUS^RT*/ + TS_SEND_RESCHED_START(cpu); apic->send_IPI_mask(cpumask_of(cpu), RESCHEDULE_VECTOR); } @@ -257,6 +262,9 @@ void smp_reschedule_interrupt(struct pt_regs *regs) /* * KVM uses this interrupt to force a cpu out of guest mode */ + + /* LITMUS^RT */ + TS_SEND_RESCHED_END; } void smp_call_function_interrupt(struct pt_regs *regs) diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index c1d6555..7ad5fd8 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -7,6 +7,8 @@ #include #include +#include + /* * We put the hardirq and softirq counter into the preemption * counter. The bitmask has the following meaning: @@ -154,6 +156,7 @@ extern void rcu_nmi_exit(void); account_irq_enter_time(current); \ add_preempt_count(HARDIRQ_OFFSET); \ trace_hardirq_enter(); \ + ft_irq_fired(); \ } while (0) /* @@ -184,6 +187,7 @@ extern void irq_exit(void); add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \ rcu_nmi_enter(); \ trace_hardirq_enter(); \ + ft_irq_fired(); \ } while (0) #define nmi_exit() \ diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index d19a5c2..923a57a 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -116,6 +116,8 @@ struct hrtimer { void *start_site; char start_comm[16]; #endif + + ktime_t enqueued; }; /** diff --git a/include/litmus/feather_buffer.h b/include/litmus/feather_buffer.h new file mode 100644 index 0000000..38de95b --- /dev/null +++ b/include/litmus/feather_buffer.h @@ -0,0 +1,118 @@ +#ifndef _FEATHER_BUFFER_H_ +#define _FEATHER_BUFFER_H_ + +/* requires UINT_MAX and memcpy */ + +#define SLOT_FREE 0 +#define SLOT_BUSY 1 +#define SLOT_READY 2 + +struct ft_buffer { + unsigned int slot_count; + unsigned int slot_size; + + int free_count; + unsigned int write_idx; + unsigned int read_idx; + + char* slots; + void* buffer_mem; + unsigned int failed_writes; +}; + +static inline int init_ft_buffer(struct ft_buffer* buf, + unsigned int slot_count, + unsigned int slot_size, + char* slots, + void* buffer_mem) +{ + int i = 0; + if (!slot_count || UINT_MAX % slot_count != slot_count - 1) { + /* The slot count must divide UNIT_MAX + 1 so that when it + * wraps around the index correctly points to 0. + */ + return 0; + } else { + buf->slot_count = slot_count; + buf->slot_size = slot_size; + buf->slots = slots; + buf->buffer_mem = buffer_mem; + buf->free_count = slot_count; + buf->write_idx = 0; + buf->read_idx = 0; + buf->failed_writes = 0; + for (i = 0; i < slot_count; i++) + buf->slots[i] = SLOT_FREE; + return 1; + } +} + +static inline int ft_buffer_start_write(struct ft_buffer* buf, void **ptr) +{ + int free = fetch_and_dec(&buf->free_count); + unsigned int idx; + if (free <= 0) { + fetch_and_inc(&buf->free_count); + *ptr = 0; + fetch_and_inc(&buf->failed_writes); + return 0; + } else { + idx = fetch_and_inc((int*) &buf->write_idx) % buf->slot_count; + buf->slots[idx] = SLOT_BUSY; + *ptr = ((char*) buf->buffer_mem) + idx * buf->slot_size; + return 1; + } +} + +/* For single writer scenarios, with fewer atomic ops. */ +static inline int ft_buffer_start_single_write(struct ft_buffer* buf, void **ptr) +{ + unsigned int idx; + + if (buf->free_count <= 0) { + *ptr = 0; + /* single writer: no atomicity needed */ + buf->failed_writes++; + return 0; + } else { + /* free_count is positive, and can only increase since we are + * (by assumption) the only writer accessing the buffer. + */ + + idx = buf->write_idx++ % buf->slot_count; + buf->slots[idx] = SLOT_BUSY; + *ptr = ((char*) buf->buffer_mem) + idx * buf->slot_size; + + ft_atomic_dec(&buf->free_count); + return 1; + } +} + +static inline void ft_buffer_finish_write(struct ft_buffer* buf, void *ptr) +{ + unsigned int idx = ((char*) ptr - (char*) buf->buffer_mem) / buf->slot_size; + buf->slots[idx] = SLOT_READY; +} + + +/* exclusive reader access is assumed */ +static inline int ft_buffer_read(struct ft_buffer* buf, void* dest) +{ + unsigned int idx; + if (buf->free_count == buf->slot_count) + /* nothing available */ + return 0; + idx = buf->read_idx % buf->slot_count; + if (buf->slots[idx] == SLOT_READY) { + memcpy(dest, ((char*) buf->buffer_mem) + idx * buf->slot_size, + buf->slot_size); + buf->slots[idx] = SLOT_FREE; + buf->read_idx++; + fetch_and_inc(&buf->free_count); + return 1; + } else + return 0; +} + + +#endif diff --git a/include/litmus/feather_trace.h b/include/litmus/feather_trace.h new file mode 100644 index 0000000..76e8004 --- /dev/null +++ b/include/litmus/feather_trace.h @@ -0,0 +1,70 @@ +#ifndef _FEATHER_TRACE_H_ +#define _FEATHER_TRACE_H_ + +#include + +int ft_enable_event(unsigned long id); +int ft_disable_event(unsigned long id); +int ft_is_event_enabled(unsigned long id); +int ft_disable_all_events(void); + +/* atomic_* funcitons are inline anyway */ +static inline int fetch_and_inc(int *val) +{ + return atomic_add_return(1, (atomic_t*) val) - 1; +} + +static inline int fetch_and_dec(int *val) +{ + return atomic_sub_return(1, (atomic_t*) val) + 1; +} + +static inline void ft_atomic_dec(int *val) +{ + atomic_sub(1, (atomic_t*) val); +} + +/* Don't use rewriting implementation if kernel text pages are read-only. + * Ftrace gets around this by using the identity mapping, but that's more + * effort that is warrented right now for Feather-Trace. + * Eventually, it may make sense to replace Feather-Trace with ftrace. + */ +#if defined(CONFIG_ARCH_HAS_FEATHER_TRACE) && !defined(CONFIG_DEBUG_RODATA) + +#include + +#else /* !__ARCH_HAS_FEATHER_TRACE */ + +/* provide default implementation */ + +#include /* for get_cycles() */ + +static inline unsigned long long ft_timestamp(void) +{ + return get_cycles(); +} + +#define feather_callback + +#define MAX_EVENTS 1024 + +extern int ft_events[MAX_EVENTS]; + +#define ft_event(id, callback) \ + if (ft_events[id]) callback(); + +#define ft_event0(id, callback) \ + if (ft_events[id]) callback(id); + +#define ft_event1(id, callback, param) \ + if (ft_events[id]) callback(id, param); + +#define ft_event2(id, callback, param, param2) \ + if (ft_events[id]) callback(id, param, param2); + +#define ft_event3(id, callback, p, p2, p3) \ + if (ft_events[id]) callback(id, p, p2, p3); + +#endif /* __ARCH_HAS_FEATHER_TRACE */ + +#endif diff --git a/include/litmus/ftdev.h b/include/litmus/ftdev.h new file mode 100644 index 0000000..a566b0b --- /dev/null +++ b/include/litmus/ftdev.h @@ -0,0 +1,58 @@ +#ifndef _LITMUS_FTDEV_H_ +#define _LITMUS_FTDEV_H_ + +#include +#include +#include +#include + +#define FTDEV_ENABLE_CMD 0 +#define FTDEV_DISABLE_CMD 1 +#define FTDEV_CALIBRATE 0x1410 + +struct ftdev; + +/* return 0 if buffer can be opened, otherwise -$REASON */ +typedef int (*ftdev_can_open_t)(struct ftdev* dev, unsigned int buf_no); +/* return 0 on success, otherwise -$REASON */ +typedef int (*ftdev_alloc_t)(struct ftdev* dev, unsigned int buf_no); +typedef void (*ftdev_free_t)(struct ftdev* dev, unsigned int buf_no); +typedef long (*ftdev_calibrate_t)(struct ftdev* dev, unsigned int buf_no, unsigned long user_arg); +/* Let devices handle writes from userspace. No synchronization provided. */ +typedef ssize_t (*ftdev_write_t)(struct ft_buffer* buf, size_t len, const char __user *from); + +struct ftdev_event; + +struct ftdev_minor { + struct ft_buffer* buf; + unsigned int readers; + struct mutex lock; + /* FIXME: filter for authorized events */ + struct ftdev_event* events; + struct device* device; + struct ftdev* ftdev; +}; + +struct ftdev { + dev_t major; + struct cdev cdev; + struct class* class; + const char* name; + struct ftdev_minor* minor; + unsigned int minor_cnt; + ftdev_alloc_t alloc; + ftdev_free_t free; + ftdev_can_open_t can_open; + ftdev_write_t write; + ftdev_calibrate_t calibrate; +}; + +struct ft_buffer* alloc_ft_buffer(unsigned int count, size_t size); +void free_ft_buffer(struct ft_buffer* buf); + +int ftdev_init( struct ftdev* ftdev, struct module* owner, + const int minor_cnt, const char* name); +void ftdev_exit(struct ftdev* ftdev); +int register_ftdev(struct ftdev* ftdev); + +#endif diff --git a/include/litmus/trace.h b/include/litmus/trace.h new file mode 100644 index 0000000..4f96dd8 --- /dev/null +++ b/include/litmus/trace.h @@ -0,0 +1,166 @@ +#ifndef _SYS_TRACE_H_ +#define _SYS_TRACE_H_ + +#ifdef CONFIG_SCHED_OVERHEAD_TRACE + + +#include +#include + +/* Litmus time type. */ +typedef unsigned long long lt_t; + +/* Our notion of time within LITMUS: kernel monotonic time. */ +static inline lt_t litmus_clock(void) +{ + return ktime_to_ns(ktime_get()); +} + +/* Only capture traces of SCHED_DEADLINE tasks */ +#define is_realtime(t) ((t)->policy == SCHED_DEADLINE) + +/*********************** TIMESTAMPS ************************/ + +enum task_type_marker { + TSK_BE, + TSK_RT, + TSK_UNKNOWN +}; + +struct timestamp { + uint64_t timestamp:48; + uint64_t pid:16; + uint32_t seq_no; + uint8_t cpu; + uint8_t event; + uint8_t task_type:2; + uint8_t irq_flag:1; + uint8_t irq_count:5; +}; + +/* tracing callbacks */ +feather_callback void save_timestamp(unsigned long event); +feather_callback void save_timestamp_def(unsigned long event, unsigned long type); +feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu); +feather_callback void save_timestamp_time(unsigned long event, unsigned long time_ptr); +feather_callback void save_timestamp_irq(unsigned long event, unsigned long irq_count_ptr); +feather_callback void save_timestamp_hide_irq(unsigned long event); +feather_callback void save_timestamp_hide_irq_task(unsigned long event, unsigned long t_ptr); +feather_callback void save_timestamp_hide_irq_cpu(unsigned long event, unsigned long cpu); + +feather_callback void save_cpu_timestamp(unsigned long event); +feather_callback void save_cpu_timestamp_task(unsigned long event, unsigned long t_ptr); +feather_callback void save_cpu_timestamp_def(unsigned long event, unsigned long type); +feather_callback void save_cpu_task_latency(unsigned long event, unsigned long when_ptr); + +#define TIMESTAMP(id) ft_event0(id, save_timestamp) + +#define DTIMESTAMP(id, def) ft_event1(id, save_timestamp_def, (unsigned long) def) + +#define TIMESTAMP_CUR(id) DTIMESTAMP(id, is_realtime(current) ? TSK_RT : TSK_BE) + +#define CTIMESTAMP(id, cpu) \ + ft_event1(id, save_timestamp_cpu, (unsigned long) cpu) + +#define TIMESTAMP_TIME(id, time_ptr) \ + ft_event1(id, save_timestamp_time, (unsigned long) time_ptr) + +#define TIMESTAMP_IRQ(id, irq_count_ptr) \ + ft_event1(id, save_timestamp_irq, (unsigned long) irq_count_ptr) + +#define TIMESTAMP_IN_IRQ(id) \ + ft_event0(id, save_timestamp_hide_irq) + +#define CPU_TIMESTAMP(id) ft_event0(id, save_cpu_timestamp) + +#define CPU_DTIMESTAMP(id, def) ft_event1(id, save_cpu_timestamp_def, (unsigned long) def) + +#define CPU_TTIMESTAMP(id, task) \ + ft_event1(id, save_cpu_timestamp_task, (unsigned long) task) + +#define CPU_LTIMESTAMP(id, task) \ + ft_event1(id, save_cpu_task_latency, (unsigned long) task) + + +#else /* !CONFIG_SCHED_OVERHEAD_TRACE */ + +#define TIMESTAMP(id) /* no tracing */ + +#define DTIMESTAMP(id, def) /* no tracing */ + +#define TIMESTAMP_CUR(id) /* no tracing */ + +#define TTIMESTAMP(id, task) /* no tracing */ + +#define CTIMESTAMP(id, cpu) /* no tracing */ + +#define LTIMESTAMP(id, when_ptr) /* no tracing */ + +#define TIMESTAMP_TIME(id, time_ptr) /* no tracing */ + +#define TIMESTAMP_IRQ(id, irq_count_ptr) /* no tracing */ + +#define TIMESTAMP_IN_IRQ(id) /* no tracing */ + +#endif + + +/* Convention for timestamps + * ========================= + * + * In order to process the trace files with a common tool, we use the following + * convention to measure execution times: The end time id of a code segment is + * always the next number after the start time event id. + */ + +#define __TS_SYSCALL_IN_START(p) TIMESTAMP_TIME(10, p) +#define __TS_SYSCALL_IN_END(p) TIMESTAMP_IRQ(11, p) + +#define TS_SYSCALL_OUT_START TIMESTAMP_CUR(20) +#define TS_SYSCALL_OUT_END TIMESTAMP_CUR(21) + +#define TS_LOCK_START TIMESTAMP_CUR(30) +#define TS_LOCK_END TIMESTAMP_CUR(31) + +#define TS_LOCK_SUSPEND TIMESTAMP_CUR(38) +#define TS_LOCK_RESUME TIMESTAMP_CUR(39) + +#define TS_UNLOCK_START TIMESTAMP_CUR(40) +#define TS_UNLOCK_END TIMESTAMP_CUR(41) + +#define TS_SCHED_START CPU_DTIMESTAMP(100, TSK_UNKNOWN) /* we only + * care + * about + * next */ +#define TS_SCHED_END(t) CPU_TTIMESTAMP(101, t) +#define TS_SCHED2_START(t) CPU_TTIMESTAMP(102, t) +#define TS_SCHED2_END(t) CPU_TTIMESTAMP(103, t) + +#define TS_CXS_START(t) CPU_TTIMESTAMP(104, t) +#define TS_CXS_END(t) CPU_TTIMESTAMP(105, t) + +#define TS_RELEASE_START CPU_DTIMESTAMP(106, TSK_RT) +#define TS_RELEASE_END CPU_DTIMESTAMP(107, TSK_RT) + +#define TS_TICK_START(t) CPU_TTIMESTAMP(110, t) +#define TS_TICK_END(t) CPU_TTIMESTAMP(111, t) + + +#define TS_PLUGIN_SCHED_START /* TIMESTAMP(120) */ /* currently unused */ +#define TS_PLUGIN_SCHED_END /* TIMESTAMP(121) */ + +#define TS_PLUGIN_TICK_START /* TIMESTAMP(130) */ +#define TS_PLUGIN_TICK_END /* TIMESTAMP(131) */ + +#define TS_ENTER_NP_START CPU_TIMESTAMP(140) +#define TS_ENTER_NP_END CPU_TIMESTAMP(141) + +#define TS_EXIT_NP_START CPU_TIMESTAMP(150) +#define TS_EXIT_NP_END CPU_TIMESTAMP(151) + +#define TS_SEND_RESCHED_START(c) CTIMESTAMP(190, c) +#define TS_SEND_RESCHED_END TIMESTAMP_IN_IRQ(191) + +#define TS_RELEASE_LATENCY(when) CPU_LTIMESTAMP(208, &(when)) + +#endif /* !_SYS_TRACE_H_ */ diff --git a/include/litmus/trace_irq.h b/include/litmus/trace_irq.h new file mode 100644 index 0000000..0d0c042 --- /dev/null +++ b/include/litmus/trace_irq.h @@ -0,0 +1,14 @@ +#ifndef _LITMUS_TRACE_IRQ_H_ +#define _LITMUS_TRACE_IRQ_H_ + +#ifdef CONFIG_SCHED_OVERHEAD_TRACE + +void ft_irq_fired(void); + +#else + +#define ft_irq_fired() /* nothing to do */ + +#endif + +#endif diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 8dcf9d9..2dda0a1 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -49,6 +49,8 @@ #include #include +#include + #include #include @@ -885,6 +887,8 @@ static int enqueue_hrtimer(struct hrtimer *timer, */ timer->state |= HRTIMER_STATE_ENQUEUED; + timer->enqueued = base->get_time(); + return (&timer->node == base->active.next); } @@ -1520,10 +1524,22 @@ static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer); struct task_struct *task = t->task; + lt_t expected; t->task = NULL; - if (task) + if (task) { + if (is_realtime(task) && timer->base->clockid == CLOCK_MONOTONIC + && in_irq() + && ktime_to_ns(timer->enqueued) < ktime_to_ns(hrtimer_get_softexpires(timer))) { + /* LITMUS^RT: Only trace SCHED_DEADLINE tasks */ + expected = ktime_to_ns(hrtimer_get_softexpires(timer)); + TS_RELEASE_LATENCY(expected); + } + + TS_RELEASE_START; wake_up_process(task); + TS_RELEASE_END; + } return HRTIMER_NORESTART; } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 52245d3..6a05e3a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -86,6 +86,8 @@ #include "../workqueue_internal.h" #include "../smpboot.h" +#include + #define CREATE_TRACE_POINTS #include @@ -1405,7 +1407,12 @@ void scheduler_ipi(void) { if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick() && !tick_nohz_full_cpu(smp_processor_id())) + { + /* If we don't call irq_enter(), we need to triggger the IRQ + * tracing manually. */ + ft_irq_fired(); return; + } /* * Not all reschedule IPI handlers call irq_enter/irq_exit, since @@ -2842,6 +2849,8 @@ void scheduler_tick(void) sched_clock_tick(); + TS_TICK_START(current); + raw_spin_lock(&rq->lock); update_rq_clock(rq); update_cpu_load_active(rq); @@ -2855,6 +2864,8 @@ void scheduler_tick(void) trigger_load_balance(rq, cpu); #endif rq_last_tick_reset(rq); + + TS_TICK_END(current); } #ifdef CONFIG_NO_HZ_FULL @@ -3066,6 +3077,8 @@ need_resched: rcu_note_context_switch(cpu); prev = rq->curr; + TS_SCHED_START; + schedule_debug(prev); if (sched_feat(HRTICK)) @@ -3112,7 +3125,10 @@ need_resched: rq->curr = next; ++*switch_count; + TS_SCHED_END(next); + TS_CXS_START(next); context_switch(rq, prev, next); /* unlocks the rq */ + TS_CXS_END(current); /* * The context switch have flipped the stack from under us * and restored the local variables which were saved when @@ -3121,12 +3137,19 @@ need_resched: */ cpu = smp_processor_id(); rq = cpu_rq(cpu); - } else + } else { + TS_SCHED_END(prev); raw_spin_unlock_irq(&rq->lock); + } + + TS_SCHED2_START(prev); post_schedule(rq); sched_preempt_enable_no_resched(); + + TS_SCHED2_END(prev); + if (need_resched()) goto need_resched; } diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 00b550b..bda3cef 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -18,6 +18,8 @@ #include +#include + struct dl_bandwidth def_dl_bandwidth; static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) diff --git a/kernel/softirq.c b/kernel/softirq.c index b5197dc..28542ab 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -220,6 +220,9 @@ asmlinkage void __do_softirq(void) */ current->flags &= ~PF_MEMALLOC; + /* Mark Feather-Trace samples as "disturbed". */ + ft_irq_fired(); + pending = local_softirq_pending(); account_irq_enter_time(current); diff --git a/litmus/Kconfig b/litmus/Kconfig new file mode 100644 index 0000000..0c7e06b --- /dev/null +++ b/litmus/Kconfig @@ -0,0 +1,54 @@ +menu "LITMUS^RT" + +menu "Tracing" + +config FEATHER_TRACE + bool "Feather-Trace Infrastructure" + depends on !RELOCATABLE + default y + help + Feather-Trace basic tracing infrastructure. Includes device file + driver and instrumentation point support. + + There are actually two implementations of Feather-Trace. + 1) A slower, but portable, default implementation. + 2) Architecture-specific implementations that rewrite kernel .text at runtime. + + If enabled, Feather-Trace will be based on 2) if available (currently only for x86). + However, if DEBUG_RODATA=y, then Feather-Trace will choose option 1) in any case + to avoid problems with write-protected .text pages. + + Bottom line: to avoid increased overheads, choose DEBUG_RODATA=n. + + Note that this option only enables the basic Feather-Trace infrastructure; + you still need to enable SCHED_TASK_TRACE and/or SCHED_OVERHEAD_TRACE to + actually enable any events. + +config SCHED_OVERHEAD_TRACE + bool "Record timestamps for overhead measurements" + depends on FEATHER_TRACE + default y + help + Export event stream for overhead tracing. + Say Yes for overhead tracing. + +config SCHED_OVERHEAD_TRACE_SHIFT + int "Buffer size for Feather-Trace overhead data" + depends on SCHED_OVERHEAD_TRACE + range 15 32 + default 22 + help + + Select the buffer size for the Feather-Trace overhead tracing + infrastructure (/dev/litmus/ft_trace0 & ftcat) as a power of two. The + larger the buffer, the less likely the chance of buffer overflows if + the ftcat process is starved by real-time activity. In machines with + large memories, large buffer sizes are recommended. + + Examples: 16 => 2 MB + 24 => 512 MB + 26 => 2G MB + +endmenu + +endmenu diff --git a/litmus/Makefile b/litmus/Makefile new file mode 100644 index 0000000..99f90c3 --- /dev/null +++ b/litmus/Makefile @@ -0,0 +1,6 @@ +# +# Makefile for LITMUS^RT +# + +obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o +obj-$(CONFIG_SCHED_OVERHEAD_TRACE) += trace.o diff --git a/litmus/ft_event.c b/litmus/ft_event.c new file mode 100644 index 0000000..399a07b --- /dev/null +++ b/litmus/ft_event.c @@ -0,0 +1,43 @@ +#include + +#include + +#if !defined(CONFIG_ARCH_HAS_FEATHER_TRACE) || defined(CONFIG_DEBUG_RODATA) +/* provide dummy implementation */ + +int ft_events[MAX_EVENTS]; + +int ft_enable_event(unsigned long id) +{ + if (id < MAX_EVENTS) { + ft_events[id]++; + return 1; + } else + return 0; +} + +int ft_disable_event(unsigned long id) +{ + if (id < MAX_EVENTS && ft_events[id]) { + ft_events[id]--; + return 1; + } else + return 0; +} + +int ft_disable_all_events(void) +{ + int i; + + for (i = 0; i < MAX_EVENTS; i++) + ft_events[i] = 0; + + return MAX_EVENTS; +} + +int ft_is_event_enabled(unsigned long id) +{ + return id < MAX_EVENTS && ft_events[id]; +} + +#endif diff --git a/litmus/ftdev.c b/litmus/ftdev.c new file mode 100644 index 0000000..13f1d48 --- /dev/null +++ b/litmus/ftdev.c @@ -0,0 +1,439 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct ft_buffer* alloc_ft_buffer(unsigned int count, size_t size) +{ + struct ft_buffer* buf; + size_t total = (size + 1) * count; + char* mem; + + buf = kmalloc(sizeof(*buf), GFP_KERNEL); + if (!buf) + return NULL; + + + mem = vmalloc(total); + + if (!mem) { + kfree(buf); + return NULL; + } + + if (!init_ft_buffer(buf, count, size, + mem + (count * size), /* markers at the end */ + mem)) { /* buffer objects */ + vfree(mem); + kfree(buf); + return NULL; + } + return buf; +} + +void free_ft_buffer(struct ft_buffer* buf) +{ + if (buf) { + vfree(buf->buffer_mem); + kfree(buf); + } +} + +struct ftdev_event { + int id; + struct ftdev_event* next; +}; + +static int activate(struct ftdev_event** chain, int id) +{ + struct ftdev_event* ev = kmalloc(sizeof(*ev), GFP_KERNEL); + if (ev) { + printk(KERN_INFO + "Enabling feather-trace event %d.\n", (int) id); + ft_enable_event(id); + ev->id = id; + ev->next = *chain; + *chain = ev; + } + return ev ? 0 : -ENOMEM; +} + +static void deactivate(struct ftdev_event** chain, int id) +{ + struct ftdev_event **cur = chain; + struct ftdev_event *nxt; + while (*cur) { + if ((*cur)->id == id) { + nxt = (*cur)->next; + kfree(*cur); + *cur = nxt; + printk(KERN_INFO + "Disabling feather-trace event %d.\n", (int) id); + ft_disable_event(id); + break; + } + cur = &(*cur)->next; + } +} + +static int ftdev_open(struct inode *in, struct file *filp) +{ + struct ftdev* ftdev; + struct ftdev_minor* ftdm; + unsigned int buf_idx = iminor(in); + int err = 0; + + ftdev = container_of(in->i_cdev, struct ftdev, cdev); + + if (buf_idx >= ftdev->minor_cnt) { + err = -ENODEV; + goto out; + } + if (ftdev->can_open && (err = ftdev->can_open(ftdev, buf_idx))) + goto out; + + ftdm = ftdev->minor + buf_idx; + ftdm->ftdev = ftdev; + filp->private_data = ftdm; + + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; + } + + if (!ftdm->readers && ftdev->alloc) + err = ftdev->alloc(ftdev, buf_idx); + if (0 == err) + ftdm->readers++; + + mutex_unlock(&ftdm->lock); +out: + return err; +} + +static int ftdev_release(struct inode *in, struct file *filp) +{ + struct ftdev* ftdev; + struct ftdev_minor* ftdm; + unsigned int buf_idx = iminor(in); + int err = 0; + + ftdev = container_of(in->i_cdev, struct ftdev, cdev); + + if (buf_idx >= ftdev->minor_cnt) { + err = -ENODEV; + goto out; + } + ftdm = ftdev->minor + buf_idx; + + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; + } + + if (ftdm->readers == 1) { + while (ftdm->events) + deactivate(&ftdm->events, ftdm->events->id); + + /* wait for any pending events to complete */ + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(HZ); + + printk(KERN_ALERT "Failed trace writes: %u\n", + ftdm->buf->failed_writes); + + if (ftdev->free) + ftdev->free(ftdev, buf_idx); + } + + ftdm->readers--; + mutex_unlock(&ftdm->lock); +out: + return err; +} + +/* based on ft_buffer_read + * @returns < 0 : page fault + * = 0 : no data available + * = 1 : one slot copied + */ +static int ft_buffer_copy_to_user(struct ft_buffer* buf, char __user *dest) +{ + unsigned int idx; + int err = 0; + if (buf->free_count != buf->slot_count) { + /* data available */ + idx = buf->read_idx % buf->slot_count; + if (buf->slots[idx] == SLOT_READY) { + err = copy_to_user(dest, ((char*) buf->buffer_mem) + + idx * buf->slot_size, + buf->slot_size); + if (err == 0) { + /* copy ok */ + buf->slots[idx] = SLOT_FREE; + buf->read_idx++; + fetch_and_inc(&buf->free_count); + err = 1; + } + } + } + return err; +} + +static ssize_t ftdev_read(struct file *filp, + char __user *to, size_t len, loff_t *f_pos) +{ + /* we ignore f_pos, this is strictly sequential */ + + ssize_t err = 0; + size_t chunk; + int copied; + struct ftdev_minor* ftdm = filp->private_data; + + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; + } + + + chunk = ftdm->buf->slot_size; + while (len >= chunk) { + copied = ft_buffer_copy_to_user(ftdm->buf, to); + if (copied == 1) { + len -= chunk; + to += chunk; + err += chunk; + } else if (err == 0 && copied == 0 && ftdm->events) { + /* Only wait if there are any events enabled and only + * if we haven't copied some data yet. We cannot wait + * here with copied data because that data would get + * lost if the task is interrupted (e.g., killed). + */ + mutex_unlock(&ftdm->lock); + set_current_state(TASK_INTERRUPTIBLE); + + schedule_timeout(50); + + if (signal_pending(current)) { + if (err == 0) + /* nothing read yet, signal problem */ + err = -ERESTARTSYS; + goto out; + } + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; + } + } else if (copied < 0) { + /* page fault */ + err = copied; + break; + } else + /* nothing left to get, return to user space */ + break; + } + mutex_unlock(&ftdm->lock); +out: + return err; +} + +static long ftdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + long err = -ENOIOCTLCMD; + struct ftdev_minor* ftdm = filp->private_data; + + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; + } + + /* FIXME: check id against list of acceptable events */ + + switch (cmd) { + case FTDEV_ENABLE_CMD: + if (activate(&ftdm->events, arg)) + err = -ENOMEM; + else + err = 0; + break; + + case FTDEV_DISABLE_CMD: + deactivate(&ftdm->events, arg); + err = 0; + break; + + case FTDEV_CALIBRATE: + if (ftdm->ftdev->calibrate) { + err = ftdm->ftdev->calibrate(ftdm->ftdev, iminor(filp->f_dentry->d_inode), arg); + } + break; + + default: + printk(KERN_DEBUG "ftdev: strange ioctl (%u, %lu)\n", cmd, arg); + }; + + mutex_unlock(&ftdm->lock); +out: + return err; +} + +static ssize_t ftdev_write(struct file *filp, const char __user *from, + size_t len, loff_t *f_pos) +{ + struct ftdev_minor* ftdm = filp->private_data; + ssize_t err = -EINVAL; + struct ftdev* ftdev = ftdm->ftdev; + + /* dispatch write to buffer-specific code, if available */ + if (ftdev->write) + err = ftdev->write(ftdm->buf, len, from); + + return err; +} + +struct file_operations ftdev_fops = { + .owner = THIS_MODULE, + .open = ftdev_open, + .release = ftdev_release, + .write = ftdev_write, + .read = ftdev_read, + .unlocked_ioctl = ftdev_ioctl, +}; + +int ftdev_init( struct ftdev* ftdev, struct module* owner, + const int minor_cnt, const char* name) +{ + int i, err; + + BUG_ON(minor_cnt < 1); + + cdev_init(&ftdev->cdev, &ftdev_fops); + ftdev->name = name; + ftdev->minor_cnt = minor_cnt; + ftdev->cdev.owner = owner; + ftdev->cdev.ops = &ftdev_fops; + ftdev->alloc = NULL; + ftdev->free = NULL; + ftdev->can_open = NULL; + ftdev->write = NULL; + ftdev->calibrate = NULL; + + ftdev->minor = kcalloc(ftdev->minor_cnt, sizeof(*ftdev->minor), + GFP_KERNEL); + if (!ftdev->minor) { + printk(KERN_WARNING "ftdev(%s): Could not allocate memory\n", + ftdev->name); + err = -ENOMEM; + goto err_out; + } + + for (i = 0; i < ftdev->minor_cnt; i++) { + mutex_init(&ftdev->minor[i].lock); + ftdev->minor[i].readers = 0; + ftdev->minor[i].buf = NULL; + ftdev->minor[i].events = NULL; + } + + ftdev->class = class_create(owner, ftdev->name); + if (IS_ERR(ftdev->class)) { + err = PTR_ERR(ftdev->class); + printk(KERN_WARNING "ftdev(%s): " + "Could not create device class.\n", ftdev->name); + goto err_dealloc; + } + + return 0; + +err_dealloc: + kfree(ftdev->minor); +err_out: + return err; +} + +/* + * Destroy minor devices up to, but not including, up_to. + */ +static void ftdev_device_destroy(struct ftdev* ftdev, unsigned int up_to) +{ + dev_t minor_cntr; + + if (up_to < 1) + up_to = (ftdev->minor_cnt < 1) ? 0 : ftdev->minor_cnt; + + for (minor_cntr = 0; minor_cntr < up_to; ++minor_cntr) + device_destroy(ftdev->class, MKDEV(ftdev->major, minor_cntr)); +} + +void ftdev_exit(struct ftdev* ftdev) +{ + printk("ftdev(%s): Exiting\n", ftdev->name); + ftdev_device_destroy(ftdev, -1); + cdev_del(&ftdev->cdev); + unregister_chrdev_region(MKDEV(ftdev->major, 0), ftdev->minor_cnt); + class_destroy(ftdev->class); + kfree(ftdev->minor); +} + +int register_ftdev(struct ftdev* ftdev) +{ + struct device **device; + dev_t trace_dev_tmp, minor_cntr; + int err; + + err = alloc_chrdev_region(&trace_dev_tmp, 0, ftdev->minor_cnt, + ftdev->name); + if (err) { + printk(KERN_WARNING "ftdev(%s): " + "Could not allocate char. device region (%d minors)\n", + ftdev->name, ftdev->minor_cnt); + goto err_out; + } + + ftdev->major = MAJOR(trace_dev_tmp); + + err = cdev_add(&ftdev->cdev, trace_dev_tmp, ftdev->minor_cnt); + if (err) { + printk(KERN_WARNING "ftdev(%s): " + "Could not add cdev for major %u with %u minor(s).\n", + ftdev->name, ftdev->major, ftdev->minor_cnt); + goto err_unregister; + } + + /* create the minor device(s) */ + for (minor_cntr = 0; minor_cntr < ftdev->minor_cnt; ++minor_cntr) + { + trace_dev_tmp = MKDEV(ftdev->major, minor_cntr); + device = &ftdev->minor[minor_cntr].device; + + *device = device_create(ftdev->class, NULL, trace_dev_tmp, NULL, + "litmus/%s%d", ftdev->name, minor_cntr); + if (IS_ERR(*device)) { + err = PTR_ERR(*device); + printk(KERN_WARNING "ftdev(%s): " + "Could not create device major/minor number " + "%u/%u\n", ftdev->name, ftdev->major, + minor_cntr); + printk(KERN_WARNING "ftdev(%s): " + "will attempt deletion of allocated devices.\n", + ftdev->name); + goto err_minors; + } + } + + return 0; + +err_minors: + ftdev_device_destroy(ftdev, minor_cntr); + cdev_del(&ftdev->cdev); +err_unregister: + unregister_chrdev_region(MKDEV(ftdev->major, 0), ftdev->minor_cnt); +err_out: + return err; +} diff --git a/litmus/trace.c b/litmus/trace.c new file mode 100644 index 0000000..378b657 --- /dev/null +++ b/litmus/trace.c @@ -0,0 +1,582 @@ +#include +#include +#include + +#include +#include + +/******************************************************************************/ +/* Allocation */ +/******************************************************************************/ + +static struct ftdev overhead_dev; +static struct ftdev cpu_overhead_dev; + +#define trace_ts_buf overhead_dev.minor[0].buf + +#define cpu_trace_ts_buf(cpu) cpu_overhead_dev.minor[(cpu)].buf + +static unsigned int ts_seq_no = 0; + +DEFINE_PER_CPU(atomic_t, irq_fired_count;) +DEFINE_PER_CPU(atomic_t, cpu_irq_fired_count); + +static DEFINE_PER_CPU(unsigned int, cpu_ts_seq_no); + +static int64_t cycle_offset[NR_CPUS][NR_CPUS]; + +void ft_irq_fired(void) +{ + /* Only called with preemptions disabled. */ + atomic_inc(&__get_cpu_var(irq_fired_count)); + atomic_inc(&__get_cpu_var(cpu_irq_fired_count)); +} + +static inline void clear_irq_fired(void) +{ + atomic_set(&__raw_get_cpu_var(irq_fired_count), 0); +} + +static inline unsigned int get_and_clear_irq_fired(void) +{ + /* This is potentially not atomic since we might migrate if + * preemptions are not disabled. As a tradeoff between + * accuracy and tracing overheads, this seems acceptable. + * If it proves to be a problem, then one could add a callback + * from the migration code to invalidate irq_fired_count. + */ + return atomic_xchg(&__raw_get_cpu_var(irq_fired_count), 0); +} + +static inline unsigned int get_and_clear_irq_fired_for_cpu(int cpu) +{ + return atomic_xchg(&per_cpu(irq_fired_count, cpu), 0); +} + +static inline void cpu_clear_irq_fired(void) +{ + atomic_set(&__raw_get_cpu_var(cpu_irq_fired_count), 0); +} + +static inline unsigned int cpu_get_and_clear_irq_fired(void) +{ + return atomic_xchg(&__raw_get_cpu_var(cpu_irq_fired_count), 0); +} + +static inline void save_irq_flags(struct timestamp *ts, unsigned int irq_count) +{ + /* Store how many interrupts occurred. */ + ts->irq_count = irq_count; + /* Extra flag because ts->irq_count overflows quickly. */ + ts->irq_flag = irq_count > 0; +} + +#define NO_IRQ_COUNT 0 +#define LOCAL_IRQ_COUNT 1 +#define REMOTE_IRQ_COUNT 2 + +#define DO_NOT_RECORD_TIMESTAMP 0 +#define RECORD_LOCAL_TIMESTAMP 1 +#define RECORD_OFFSET_TIMESTAMP 2 + +static inline void write_timestamp(uint8_t event, + uint8_t type, + uint8_t cpu, + uint16_t pid_fragment, + unsigned int irq_count, + int record_irq, + int hide_irq, + uint64_t timestamp, + int record_timestamp) +{ + unsigned long flags; + unsigned int seq_no; + struct timestamp *ts; + + /* Avoid preemptions while recording the timestamp. This reduces the + * number of "out of order" timestamps in the stream and makes + * post-processing easier. */ + + local_irq_save(flags); + + seq_no = fetch_and_inc((int *) &ts_seq_no); + if (ft_buffer_start_write(trace_ts_buf, (void**) &ts)) { + ts->event = event; + ts->seq_no = seq_no; + + ts->task_type = type; + ts->pid = pid_fragment; + + ts->cpu = cpu; + + if (likely(record_irq == LOCAL_IRQ_COUNT)) + irq_count = get_and_clear_irq_fired(); + else if (record_irq == REMOTE_IRQ_COUNT) + irq_count = get_and_clear_irq_fired_for_cpu(cpu); + + save_irq_flags(ts, irq_count - hide_irq); + + if (record_timestamp) + timestamp = ft_timestamp(); + if (record_timestamp == RECORD_OFFSET_TIMESTAMP) + timestamp += cycle_offset[smp_processor_id()][cpu]; + + ts->timestamp = timestamp; + ft_buffer_finish_write(trace_ts_buf, ts); + } + + local_irq_restore(flags); +} + +static inline void write_cpu_timestamp( + uint8_t event, + uint8_t type, + uint16_t pid_fragment, + unsigned int irq_count, + int record_irq, + int hide_irq, + uint64_t timestamp, + int record_timestamp) +{ + unsigned long flags; + unsigned int seq_no; + struct timestamp *ts; + int cpu; + struct ft_buffer* buf; + + local_irq_save(flags); + cpu = smp_processor_id(); + + seq_no = __get_cpu_var(cpu_ts_seq_no)++; + + buf = cpu_trace_ts_buf(cpu); + /* If buf is non-NULL here, then the buffer cannot be deallocated until + * we turn interrupts on again. This is because free_timestamp_buffer() + * indirectly causes TLB invalidations due to modifications of the + * kernel address space, namely via vfree() in free_ft_buffer(), which + * cannot be processed until we turn on interrupts again. + */ + + if (buf && ft_buffer_start_single_write(buf, (void**) &ts)) { + ts->event = event; + ts->seq_no = seq_no; + + ts->task_type = type; + ts->pid = pid_fragment; + + ts->cpu = cpu; + + if (record_irq) + irq_count = cpu_get_and_clear_irq_fired(); + + save_irq_flags(ts, irq_count - hide_irq); + + if (record_timestamp) + timestamp = ft_timestamp(); + + ts->timestamp = timestamp; + ft_buffer_finish_write(buf, ts); + } + + local_irq_restore(flags); +} + +static void __add_timestamp_user(struct timestamp *pre_recorded) +{ + unsigned long flags; + unsigned int seq_no; + struct timestamp *ts; + + + local_irq_save(flags); + + seq_no = fetch_and_inc((int *) &ts_seq_no); + if (ft_buffer_start_write(trace_ts_buf, (void**) &ts)) { + *ts = *pre_recorded; + ts->seq_no = seq_no; + ts->cpu = raw_smp_processor_id(); + save_irq_flags(ts, get_and_clear_irq_fired()); + ft_buffer_finish_write(trace_ts_buf, ts); + } + + local_irq_restore(flags); +} + +feather_callback void save_timestamp(unsigned long event) +{ + write_timestamp(event, TSK_UNKNOWN, + raw_smp_processor_id(), + current->pid, + 0, 1, 0, + 0, 1); +} + + +feather_callback void save_cpu_timestamp(unsigned long event) +{ + write_cpu_timestamp(event, TSK_UNKNOWN, + current->pid, + 0, 1, 0, + 0, 1); +} + +feather_callback void save_timestamp_def(unsigned long event, + unsigned long type) +{ + write_timestamp(event, type, + raw_smp_processor_id(), + current->pid, + 0, 1, 0, + 0, 1); +} + +feather_callback void save_cpu_timestamp_def(unsigned long event, + unsigned long type) +{ + write_cpu_timestamp(event, type, + current->pid, + 0, 1, 0, + 0, 1); +} + +feather_callback void save_timestamp_task(unsigned long event, + unsigned long t_ptr) +{ + struct task_struct *t = (struct task_struct *) t_ptr; + int rt = is_realtime(t); + + write_timestamp(event, rt ? TSK_RT : TSK_BE, + raw_smp_processor_id(), + t->pid, + 0, 1, 0, + 0, 1); +} + +feather_callback void save_cpu_timestamp_task(unsigned long event, + unsigned long t_ptr) +{ + struct task_struct *t = (struct task_struct *) t_ptr; + int rt = is_realtime(t); + + write_cpu_timestamp(event, rt ? TSK_RT : TSK_BE, + t->pid, + 0, 1, 0, + 0, 1); +} + +feather_callback void save_timestamp_cpu(unsigned long event, + unsigned long cpu) +{ + write_timestamp(event, TSK_UNKNOWN, cpu, current->pid, + 0, REMOTE_IRQ_COUNT, 0, + 0, RECORD_OFFSET_TIMESTAMP); +} + +feather_callback void save_cpu_task_latency(unsigned long event, + unsigned long when_ptr) +{ + lt_t now = litmus_clock(); + lt_t *when = (lt_t*) when_ptr; + + write_cpu_timestamp(event, TSK_RT, + 0, + 0, 1, 0, + now - *when, 0); +} + +/* fake timestamp to user-reported time */ +feather_callback void save_timestamp_time(unsigned long event, + unsigned long ptr) +{ + uint64_t* time = (uint64_t*) ptr; + + write_timestamp(event, is_realtime(current) ? TSK_RT : TSK_BE, + raw_smp_processor_id(), current->pid, + 0, 1, 0, + *time, 0); +} + +/* Record user-reported IRQ count */ +feather_callback void save_timestamp_irq(unsigned long event, + unsigned long irq_counter_ptr) +{ + uint64_t* irqs = (uint64_t*) irq_counter_ptr; + + write_timestamp(event, is_realtime(current) ? TSK_RT : TSK_BE, + raw_smp_processor_id(), current->pid, + *irqs, 0, 0, + 0, 1); +} + +/* Suppress one IRQ from the irq count. Used by TS_SEND_RESCHED_END, which is + * called from within an interrupt that is expected. */ +feather_callback void save_timestamp_hide_irq(unsigned long event) +{ + write_timestamp(event, is_realtime(current) ? TSK_RT : TSK_BE, + raw_smp_processor_id(), current->pid, + 0, LOCAL_IRQ_COUNT, 1, + 0, 1); +} + +/* Suppress one IRQ from the irq count. */ +feather_callback void save_timestamp_hide_irq_task(unsigned long event, + unsigned long t_ptr) +{ + struct task_struct *t = (struct task_struct *) t_ptr; + int rt = is_realtime(t); + + write_timestamp(event, rt ? TSK_RT : TSK_BE, + raw_smp_processor_id(), t->pid, + 0, 1, 1, + 0, 1); + +} + +/* Suppress one IRQ from the irq count. */ +feather_callback void save_timestamp_hide_irq_cpu(unsigned long event, + unsigned long cpu) +{ + write_timestamp(event, TSK_UNKNOWN, cpu, current->pid, + 0, 1, 1, + 0, 1); +} + +/******************************************************************************/ +/* DEVICE FILE DRIVER */ +/******************************************************************************/ + +struct calibrate_info { + atomic_t ready; + + uint64_t cycle_count; +}; + +static void calibrate_helper(void *_info) +{ + struct calibrate_info *info = _info; + /* check in with master */ + atomic_inc(&info->ready); + + /* wait for master to signal start */ + while (atomic_read(&info->ready)) + cpu_relax(); + + /* report time stamp */ + info->cycle_count = ft_timestamp(); + + /* tell master that we are done */ + atomic_inc(&info->ready); +} + + +static int64_t calibrate_cpu(int cpu) +{ + uint64_t cycles; + struct calibrate_info info; + unsigned long flags; + int64_t delta; + + atomic_set(&info.ready, 0); + info.cycle_count = 0; + smp_wmb(); + + smp_call_function_single(cpu, calibrate_helper, &info, 0); + + /* wait for helper to become active */ + while (!atomic_read(&info.ready)) + cpu_relax(); + + /* avoid interrupt interference */ + local_irq_save(flags); + + /* take measurement */ + atomic_set(&info.ready, 0); + smp_wmb(); + cycles = ft_timestamp(); + + /* wait for helper reading */ + while (!atomic_read(&info.ready)) + cpu_relax(); + + /* positive offset: the other guy is ahead of us */ + delta = (int64_t) info.cycle_count; + delta -= (int64_t) cycles; + + local_irq_restore(flags); + + return delta; +} + +#define NUM_SAMPLES 10 + +static long calibrate_tsc_offsets(struct ftdev* ftdev, unsigned int idx, + unsigned long uarg) +{ + int cpu, self, i; + int64_t delta, sample; + + preempt_disable(); + self = smp_processor_id(); + + if (uarg) + printk(KERN_INFO "Feather-Trace: determining TSC offsets for P%d\n", self); + + for_each_online_cpu(cpu) + if (cpu != self) { + delta = calibrate_cpu(cpu); + for (i = 1; i < NUM_SAMPLES; i++) { + sample = calibrate_cpu(cpu); + delta = sample < delta ? sample : delta; + } + + cycle_offset[self][cpu] = delta; + + if (uarg) + printk(KERN_INFO "Feather-Trace: TSC offset for P%d->P%d is %lld cycles.\n", + self, cpu, cycle_offset[self][cpu]); + } + + preempt_enable(); + return 0; +} + +#define NO_TIMESTAMPS (2 << CONFIG_SCHED_OVERHEAD_TRACE_SHIFT) + +static int alloc_timestamp_buffer(struct ftdev* ftdev, unsigned int idx) +{ + unsigned int count = NO_TIMESTAMPS; + + /* An overhead-tracing timestamp should be exactly 16 bytes long. */ + BUILD_BUG_ON(sizeof(struct timestamp) != 16); + + while (count && !ftdev->minor[idx].buf) { + printk("time stamp buffer: trying to allocate %u time stamps for minor=%u.\n", count, idx); + ftdev->minor[idx].buf = alloc_ft_buffer(count, sizeof(struct timestamp)); + count /= 2; + } + return ftdev->minor[idx].buf ? 0 : -ENOMEM; +} + +static void free_timestamp_buffer(struct ftdev* ftdev, unsigned int idx) +{ + ftdev->minor[idx].buf = NULL; + /* Make sure all cores have actually seen buf == NULL before + * yanking out the mappings from underneath them. */ + smp_wmb(); + free_ft_buffer(ftdev->minor[idx].buf); +} + +static ssize_t write_timestamp_from_user(struct ft_buffer* buf, size_t len, + const char __user *from) +{ + ssize_t consumed = 0; + struct timestamp ts; + + /* don't give us partial timestamps */ + if (len % sizeof(ts)) + return -EINVAL; + + while (len >= sizeof(ts)) { + if (copy_from_user(&ts, from, sizeof(ts))) { + consumed = -EFAULT; + goto out; + } + len -= sizeof(ts); + from += sizeof(ts); + consumed += sizeof(ts); + + __add_timestamp_user(&ts); + } + +out: + return consumed; +} + + +static int __init init_global_ft_overhead_trace(void) +{ + int err; + + printk("Initializing Feather-Trace overhead tracing device.\n"); + err = ftdev_init(&overhead_dev, THIS_MODULE, 1, "ft_trace"); + if (err) + goto err_out; + + overhead_dev.alloc = alloc_timestamp_buffer; + overhead_dev.free = free_timestamp_buffer; + overhead_dev.write = write_timestamp_from_user; + overhead_dev.calibrate = calibrate_tsc_offsets; + + err = register_ftdev(&overhead_dev); + if (err) + goto err_dealloc; + + return 0; + +err_dealloc: + ftdev_exit(&overhead_dev); +err_out: + printk(KERN_WARNING "Could not register ft_trace module.\n"); + return err; +} + +static int __init init_cpu_ft_overhead_trace(void) +{ + int err, cpu; + + printk("Initializing Feather-Trace per-cpu overhead tracing device.\n"); + err = ftdev_init(&cpu_overhead_dev, THIS_MODULE, + num_online_cpus(), "ft_cpu_trace"); + if (err) + goto err_out; + + cpu_overhead_dev.alloc = alloc_timestamp_buffer; + cpu_overhead_dev.free = free_timestamp_buffer; + + err = register_ftdev(&cpu_overhead_dev); + if (err) + goto err_dealloc; + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + per_cpu(cpu_ts_seq_no, cpu) = 0; + } + + return 0; + +err_dealloc: + ftdev_exit(&cpu_overhead_dev); +err_out: + printk(KERN_WARNING "Could not register per-cpu ft_trace module.\n"); + return err; +} + + +static int __init init_ft_overhead_trace(void) +{ + int err, i, j; + + for (i = 0; i < NR_CPUS; i++) + for (j = 0; j < NR_CPUS; j++) + cycle_offset[i][j] = 0; + + err = init_global_ft_overhead_trace(); + if (err) + return err; + + err = init_cpu_ft_overhead_trace(); + if (err) { + ftdev_exit(&overhead_dev); + return err; + } + + return 0; +} + +static void __exit exit_ft_overhead_trace(void) +{ + ftdev_exit(&overhead_dev); + ftdev_exit(&cpu_overhead_dev); +} + +module_init(init_ft_overhead_trace); +module_exit(exit_ft_overhead_trace);