From 04f79cb08aad1373e16b7d5442f14d7276e27ec9 Mon Sep 17 00:00:00 2001 From: Glenn Elliott Date: Sun, 19 May 2013 23:21:29 -0400 Subject: [PATCH] squash commit of GPUSync on Linux 3.0 Squashed commit of the following: commit 44326648c2ea81b9a32619644fe9c665ed0d9e0b Author: Glenn Elliott Date: Mon May 14 16:51:05 2012 -0400 Final GPUSync implementation. commit af6eeb156c7da47ff5df03a3da04432c8ac4460c Author: Glenn Elliott Date: Fri Apr 27 19:52:34 2012 -0400 fix minor bugs. there is still a bug in GEDF PAI. commit 52056e94a94517e250f7f4e36e7470a4b002404e Author: Glenn Elliott Date: Thu Apr 26 15:20:22 2012 -0400 No-op useless unlock calls. commit 040301747953ae9a2017def70e6004affe1e9aeb Author: Glenn Elliott Date: Wed Apr 25 19:30:48 2012 -0400 Fix/test C-EDF for GPU RTSS12. commit 58f04ff13ac7128609ee468eb317c71817474a84 Author: Glenn Elliott Date: Wed Apr 25 17:31:08 2012 -0400 Port rtss12 features to C-EDF (untested) commit 4aabc4a7f68aae11c79b39fa65a9c54d3f3451e7 Author: Glenn Elliott Date: Mon Apr 23 19:22:58 2012 -0400 Match replica before spinlocks in ikglp_unlock() commit 3025aea8d0ed6ee4ab68281e5cbcc76ec4dab1e2 Author: Glenn Elliott Date: Mon Apr 23 19:18:19 2012 -0400 Fix line-endings. :P commit 6f436b63bc551bbd9f9ddcf4d8a960d7d847948e Author: Glenn Elliott Date: Mon Apr 23 19:11:32 2012 -0400 Tested/Fixed IKGLP heurs. OS WORK FINALLY DONE! commit fa43d7a6bb9b0e748f23529424ac5eebd849d9d7 Author: Glenn Elliott Date: Mon Apr 23 12:12:58 2012 -0400 Donees cannot be amongst the top-m requests. commit 372db158e2a5c7e2b455262c0959eb13da4433b9 Author: Glenn Elliott Date: Fri Apr 20 22:48:32 2012 -0400 Donor dequeue heuristic for IKGLP. Untested. commit 273e902c50ef94966815a92c2af5ab8c5b2d77ce Author: Glenn Elliott Date: Fri Apr 20 21:20:21 2012 -0400 Untested donee selection heuristic for IKGLP. commit c6d04216a123f8e0b50eb78bbb1eaf646a1ca4e0 Author: Glenn Elliott Date: Wed Apr 18 23:18:32 2012 -0400 Added hooks for IKGLP affinity and a little logic. simple IKGLP is already done. it does: 1) auto gpu de/registration. 2) distruption amongst simultanous users across queues 3) calls default IKGLP routines when appropriate. Remaining work: 1) FQ advisement. 2) Donor stealing advisement. 3) Donee selection advisement. commit 149ef3b424a49e6b928c5e23fea83380ed95ea38 Author: Glenn Elliott Date: Wed Apr 18 21:33:21 2012 -0400 Zap line-endings commit f916cdb8e6a9ee2c917fddb7351e6bb39f6c953e Author: Glenn Elliott Date: Wed Apr 18 21:30:36 2012 -0400 Added support for simult-users in kfmlp commit 6ab36ca992441f7353840c70fc91d99a500a940e Author: Glenn Elliott Date: Wed Apr 18 16:24:56 2012 -0400 Fixed and tested aff-aware KFMLP. (finally!) commit 440aa2083245b81583980e3f4177f3b4cc805556 Author: Glenn Elliott Date: Mon Apr 16 20:18:07 2012 -0400 make gpu registration a little more robust commit 8675824ed85d6e83a24e77dabaf3a5c02c91ef6f Author: Glenn Elliott Date: Mon Apr 16 20:09:15 2012 -0400 Implement GPU-affinity-aware kfmlp (untested) commit 0b865246946a97dc03a81ccf55bf84acce923c4b Author: Glenn Elliott Date: Sun Apr 15 19:29:09 2012 -0400 Infrastructure for affinity-aware k-exclusion commit bb4922c968aa1a30fddd6ad9d0f750706c7b3b29 Author: Glenn Elliott Date: Sun Apr 15 18:09:59 2012 -0400 PAI::change_prio(): check work before locking commit f4aef3b7d845324eb79a226d87f232dcd8867f3b Author: Glenn Elliott Date: Sun Apr 15 18:06:04 2012 -0400 Update PAI to support multiGPUs (todo: klitirqd) commit 786d383a58108ad3437a38d0e2583859cb94a4ee Author: Glenn Elliott Date: Sun Apr 15 15:05:02 2012 -0400 remove fifo/rm header files left over commit 3f53a88be223f484db011f0f42e843aa57be8fca Author: Glenn Elliott Date: Sun Apr 15 15:04:15 2012 -0400 add kfmlp as separate file commit b3ae67412531cbc583d5697d2366fc58d6dd07e7 Merge: c0667dc adeff95 Author: Glenn Elliott Date: Sun Apr 15 15:03:33 2012 -0400 Merge branch 'wip-gpu-interrupts' into wip-gpu-rtss12 Conflicts: include/litmus/fdso.h include/litmus/rt_param.h include/litmus/sched_plugin.h include/litmus/unistd_32.h include/litmus/unistd_64.h litmus/Makefile litmus/edf_common.c litmus/litmus.c litmus/locking.c litmus/sched_gsn_edf.c litmus/sched_plugin.c commit c0667dc4894e913048cf8904f0ce9a79b481b556 Author: Glenn Elliott Date: Fri Apr 13 16:18:03 2012 -0400 Move RSM and IKGLP imp. to own .c files Also reformated code to be slightly more standard coding practice compliant. commit 8eb55f8fa1a2c3854f0f77b9b8663178c0129f6c Author: Glenn Elliott Date: Wed Apr 11 15:57:59 2012 -0400 Added support for Dynamic Group Locks (DGLs) Added support for Dynamic Group Locks. Locks are FIFO ordered (no timestamps), so a big DGL lock is needed to enqueue for resources atomically. Unfortunatly, this requires nested inheritance to use coarse-grain locking. Coarse-grain locking is used when DGLs are enabled. Fine-grain locking is used when DGLs are disabled. TODO: Clean up IKGLP implementatio. There is a lot of needless debug/TRACE work. commit 0c80d0acbbc2103a744f2b2b76cb66ddeb28ebbf Author: Glenn Elliott Date: Mon Apr 9 00:18:24 2012 -0400 Fix IKGLP bugs discovered in test. Apply fixes to the IKGLP. Also, break binheap.h into binheap.h/.c commit f5c9f29c1d17131870ec113cc357b40d2f087bc2 Author: Glenn Elliott Date: Wed Apr 4 23:11:47 2012 -0400 Cleanup use of binheap_entry(). The entry is already casted by the macro. commit 0ccecdaf12334b2241ee5185b04eda4f91f95fe2 Author: Glenn Elliott Date: Wed Apr 4 23:05:47 2012 -0400 Untested implementation of IKGLP. I don't like coding so much w/o testing, but it's sort of hard to do without both lock() and unlock(). commit d2f4875d7a183cc3c95c27c193af2c0cd1d1c555 Author: Glenn Elliott Date: Sat Mar 31 19:56:20 2012 -0400 Infrastructure of IKGLP. lock/unlock are stubs commit 62f2907f445b08f958acf1cc1a0c29736d4ba206 Author: Glenn Elliott Date: Fri Mar 30 16:43:52 2012 -0400 Nested inheritance with fine-grained locking. Minor hack to lockdep was required too allow the inheritance propagation locking logic to work. commit d0961e328a2a4c026c884c768b798cb882922708 Merge: fb0c271 4be8ef6 Author: Glenn Elliott Date: Fri Mar 23 11:16:13 2012 -0400 Merge branch 'wip-binary-heap' into wip-nested-locks commit fb0c271c1e8a4d4eac440d3e47d35f19235e07ac Author: Glenn Elliott Date: Fri Mar 23 11:16:09 2012 -0400 blah commit 8973214f010cf55fbf18cb88471d6c99ed6ff575 Author: Glenn Elliott Date: Thu Mar 22 14:45:39 2012 -0400 Introduction of basic nesting foundations. commit 4be8ef609123d4b4d281976f6bf5e65024e66b0b Author: Glenn Elliott Date: Wed Mar 21 19:25:02 2012 -0400 Make C-EDF work with simplified binheap_delete commit bf57086c9aa497c016efc208a0ceb66f262ab18b Author: Glenn Elliott Date: Wed Mar 21 19:24:13 2012 -0400 Make GSN-EDF work with simlified binheap_delete() commit 33f5fe82661086d27467821aaf418364774e360a Author: Glenn Elliott Date: Wed Mar 21 19:21:46 2012 -0400 Simplify binheap_delete and add binheap_decrease commit ee525fe7ba4edf4da2d293629ffdff2caa9ad02b Author: Glenn Elliott Date: Wed Mar 21 16:39:26 2012 -0400 C-EDF: Use binary heap instead of binomial heap. Use binary heap for ordering priority of CPUs. commit bdce67bc2babc2e5b3b2440964e9cf819ac814dc Author: Glenn Elliott Date: Wed Mar 21 16:26:27 2012 -0400 GSN-EDF: Use binary heap instead of binomial heap. Use binary heap to track CPU priorities. commit 5b73afc4eb1b0303cb92eb29a2ecc59c1db69537 Author: Glenn Elliott Date: Wed Mar 21 14:59:52 2012 -0400 Binary heap implementation Motivation: Linux's prio_heap.h is of fixed size. Litmus's binomial heap may be overkill (and perhaps not general enough) for some applications. Implemented in the style of linked lists. commit adeff95dcdcf88789e983f20b0657f29286de8d7 Author: Glenn Elliott Date: Mon Mar 5 14:59:07 2012 -0500 Remove option for threading of all softirqs. commit 3e41d4826b0aa175c3f194548fa6ab20cd1cc32d Author: Glenn Elliott Date: Sun Mar 4 21:20:45 2012 -0500 Clean up PAI. commit 12d312072e3f4caa6e4e500d5a23c85402494cd1 Merge: 6a00f20 3d1c6d4 Author: Glenn Elliott Date: Sun Mar 4 20:52:29 2012 -0500 Merge branch 'wip-pai' into wip-gpu-interrupts Conflicts: include/litmus/affinity.h kernel/sched.c kernel/softirq.c litmus/Kconfig litmus/affinity.c litmus/litmus.c litmus/preempt.c litmus/sched_cedf.c litmus/sched_gsn_edf.c commit 3d1c6d44d3f133909d1c594351c2b7c779b1d7d4 Author: Glenn Elliott Date: Sun Mar 4 16:09:04 2012 -0500 Some cleanup of PAI commit 6a00f206debf8a5c8899055726ad127dbeeed098 Author: Jonathan Herman Date: Thu Feb 16 19:13:16 2012 -0500 Typo in macro commit 83b11ea1c6ad113519c488853cf06e626c95a64d Author: Bjoern B. Brandenburg Date: Tue Jan 24 09:36:12 2012 +0100 Feather-Trace: keep track of interrupt-related interference. Increment a processor-local counter whenever an interrupt is handled. This allows Feather-Trace to include a (truncated) counter and a flag to report interference from interrupts. This could be used to filter samples that were disturbed by interrupts. commit f5264e2cb8213dad425cb2d2db564edbc443a51a Author: Glenn Elliott Date: Fri Jan 20 11:09:15 2012 -0500 Fix bugs in tracing and PAI handling commit 1a582a2c5e361e01a4c64f185bb1a23c3f70701a Author: Glenn Elliott Date: Sat Jan 14 16:56:47 2012 -0500 Port PAI interrupts to GSN-EDF, C-RM/RM-SRT/FIFO. commit 53a6dbb9f5337e77fce9c2672488c1c5e0621beb Author: Glenn Elliott Date: Sat Jan 14 14:20:07 2012 -0500 Completed PAI for C-EDF. commit 5d7dcfa10ea0dd283773a301e3ce610a7797d582 Author: Glenn Elliott Date: Wed Jan 11 14:37:13 2012 -0500 PAI implementation, C-RM, C-FIFO. commit 5bd89a34d89f252619d83fef3c9325e24311389e Author: Bjoern B. Brandenburg Date: Thu Jul 28 01:15:58 2011 -0400 Litmus core: simplify np-section protocol User a 32-bit word for all non-preemptive section flags. Set the "please yield soon" flag atomically when accessing it on remotely-scheduled tasks. commit 81b8eb2ae452c241df9b3a1fb2116fa4d5adcb75 Author: Bjoern B. Brandenburg Date: Tue Jul 26 22:03:18 2011 -0400 C-EDF: rename lock -> cluster_lock The macro lock conflicts with locking protocols... commit 71083a7604e93e44536edde032706348f3a752ca Author: Bjoern B. Brandenburg Date: Mon Jul 25 15:31:55 2011 -0400 locking: use correct timestamp commit e079932a0a1aab6adbc42fedefc6caa2d9a8af2b Author: Bjoern B. Brandenburg Date: Sat Jul 23 23:40:10 2011 -0400 Feather-trace: let userspace add overhead events This is useful for measuring locking-related overheads that are partially recorded in userspace. commit 12982f31a233250c7a62b17fb4bd13594cb78777 Author: Bjoern B. Brandenburg Date: Sat Jul 23 23:38:57 2011 -0400 ftdev: let bufffer-specific code handle writes from userspace This allows us to splice in information into logs from events that were recorded in userspace. commit 49e5b0c0d7c09bef5b9bfecaaac3f0ea2cf24e43 Author: Bjoern B. Brandenburg Date: Sat Jul 23 23:06:20 2011 -0400 ftdev: remove event activation hack Instead of doing the hackisch 'write commands to device' thing, let's just use a real ioctl() interface. commit 1dead199b4ae68ab98eacec4a661fd5ecb5a2704 Author: Bjoern B. Brandenburg Date: Sat Feb 5 23:15:09 2011 -0500 Feather-Trace: keep track of release latency commit 4490f9ecf94e28458069a02e8cfcf4f385390499 Author: Bjoern B. Brandenburg Date: Sat Feb 5 22:57:57 2011 -0500 Feather-Trace: trace locking-related suspensions commit b739b4033c0f55f9194be2793db9e6ace06047db Author: Bjoern B. Brandenburg Date: Sat Feb 5 20:11:30 2011 -0500 Feather-Trace: start with the largest permissible range MAX_ORDER is 11, but this is about number of records, not number of pages. commit fd6d753fc4e01f91427176ebfcced2c3d3f36c32 Author: Bjoern B. Brandenburg Date: Tue Feb 8 17:33:44 2011 -0500 bugfix: add processors in order of increasing indices to clusters Pfair expects to look at processors in order of increasing index. Without this patch, Pfair could deadlock in certain situations. commit 2fec12d43b366b7257c602af784b172466d8d4c5 Author: Bjoern B. Brandenburg Date: Thu Nov 24 13:59:33 2011 -0500 Pfair: improve robustness of suspensions This patch fixes two crash or hang bugs related to suspensions in Pfair. 1) When a job was not present at the end of its last subtask, then its linked_on field was not cleared. This confused the scheduler when it later resumed. Fix: clear the field. 2) Just testing for linked_on == NO_CPU is insufficient in the wake_up path to determine whether a task should be added to the ready queue. If the task remained linked and then was "preempted" at a later quantum boundary, then it already is in the ready queue and nothing is required. Fix: encode need to requeue in task_rt(t)->flags. commit d1d6e4c300d858c47b834be145f30973bc2921bf Author: Bjoern B. Brandenburg Date: Thu Nov 24 13:42:59 2011 -0500 Add option to turn off preemption state tracing Preemption state tracing is only useful when debugging preemption- and IPI-related races. Since it creates a lot of clutter in the logs, this patch turns it off unless explicitly requested. commit a7a7f71529d9a6aae02ab3cb64451e036ce9d028 Author: Glenn Elliott Date: Wed Nov 2 11:33:44 2011 -0400 Add unlikely() to rel master check (match pfair). commit 89174d049ea77b127fb3f8b3bbd8bc2996d0a535 Author: Bjoern B. Brandenburg Date: Sat Feb 12 16:40:43 2011 -0500 bugfix: release master CPU must signal task was picked commit ec77ede8baa013138fe03ff45dd57f7bac50e5d4 Author: Bjoern B. Brandenburg Date: Tue Feb 8 12:41:10 2011 -0500 Pfair: various fixes concerning release timers commit 0720416e5b1bcb825619ba4b212d9056017ffd62 Author: Bjoern B. Brandenburg Date: Sat Feb 5 21:50:36 2011 -0500 Pfair: add support for true sporadic releases This patch also converts Pfair to implement early releasing such that no timer wheel is required anymore. This removes the need for a maximum period restriction. commit 399455c0e529bb07760f17e8fe0fddc342b67bc2 Author: Bjoern B. Brandenburg Date: Sat Feb 5 22:49:52 2011 -0500 Pfair: add release master support. Merged in release master support for Pfair. Some merge conflicts had to be resolved. commit b4c52e27caa701a16e120b43a0e70ca6529a58a4 Author: Glenn Elliott Date: Wed Jun 22 01:30:25 2011 -0400 C-EDF: Make migration affinity work with Release Master Needed to update C-EDF to handle release master. Also updated get_nearest_available_cpu() to take NO_CPU instead of -1 to indicate that there is no release master. While NO_CPU is 0xffffffff (-1 in two's complement), we still translate this value to -1 in case NO_CPU changes. Signed-off-by: Andrea Bastoni commit b751e4e17e667f11404fc2f290416c0df050e964 Author: Bjoern B. Brandenburg Date: Thu Feb 10 18:41:38 2011 -0500 C-EDF: add release master support As with GSN-EDF, do not insert release master into CPU heap. commit 17e34f413750b26aa493f1f8307f111bc5d487de Author: Bjoern B. Brandenburg Date: Thu Feb 10 20:05:15 2011 -0500 PSN-EDF: add release master support We can give up a processor under partitioning, too. commit f5bee93f09b907a302e908c3cc3381ffbe826e2b Author: Glenn Elliott Date: Tue Jun 21 02:00:52 2011 -0400 COMMENT: Correct comment on precise budget enforcement Original comment said that this feature wasn't supported, though it has been since around October 2010. commit 592eaca1409e55407e980f71b2ec604ca3610ba5 Author: Glenn Elliott Date: Tue Jun 21 01:29:34 2011 -0400 Avoid needlessly costly migrations. CONFIG_SCHED_CPU_AFFINITY Given a choice between several available CPUs (unlinked) on which to schedule a task, let the scheduler select the CPU closest to where that task was previously scheduled. Hopefully, this will reduce cache migration penalties. Notes: SCHED_CPU_AFFINITY is dependent upon x86 (only x86 is supported at this time). Also PFair/PD^2 does not make use of this feature. Signed-off-by: Andrea Bastoni commit fb8d6602af1cbc09115544056b872b976c6349c3 Author: Andrea Bastoni Date: Wed Aug 24 17:32:21 2011 +0200 Prevent Linux to send IPI and queue tasks on remote CPUs. Whether to send IPIs and enqueue tasks on remote runqueues is plugin-specific. The recent ttwu_queue() mechanism (by calling ttwu_queue_remote()) interferes with Litmus plugin decisions. commit ea62a6fe914f7463f89422dcb1812eb071cbd495 Author: Andrea Bastoni Date: Wed Aug 24 12:06:42 2011 +0200 Update PULL_TIMERS_VECTOR number From 2.6.39 the "0xee" vector number that we used for pull_timers low-level management is is use by invalidate_tlb_X interrupts. Move the pull_timers vector below the max size of invalidate_tlb. commit 56c5c609615322bfbda5adff94ce011eb3d28fef Author: Andrea Bastoni Date: Sat Aug 27 16:10:06 2011 +0200 Fix prototype mismatching and synch syscall numbers * Update prototypes for switched_to(), prio_changed(), select_task_rq(). * Fix missing pid field in printk output. * Synchronize syscall numbers for arm and x86. commit 7b1bb388bc879ffcc6c69b567816d5c354afe42b Merge: 7d75459 02f8c6a Author: Andrea Bastoni Date: Sat Aug 27 15:43:54 2011 +0200 Merge 'Linux v3.0' into Litmus Some notes: * Litmus^RT scheduling class is the topmost scheduling class (above stop_sched_class). * scheduler_ipi() function (e.g., in smp_reschedule_interrupt()) may increase IPI latencies. * Added path into schedule() to quickly re-evaluate scheduling decision without becoming preemptive again. This used to be a standard path before the removal of BKL. Conflicts: Makefile arch/arm/kernel/calls.S arch/arm/kernel/smp.c arch/x86/include/asm/unistd_32.h arch/x86/kernel/smp.c arch/x86/kernel/syscall_table_32.S include/linux/hrtimer.h kernel/printk.c kernel/sched.c kernel/sched_fair.c commit 3d5537c160c1484e8d562b9828baf679cc53f67a Author: Glenn Elliott Date: Thu Jun 2 16:06:05 2011 -0400 Full patch for klitirqd with Nvidia GPU support. commit 7d754596756240fa918b94cd0c3011c77a638987 Author: Christopher Kenna Date: Sat Apr 16 20:12:00 2011 -0400 LITMUS Core: Check for valid class in RT-param syscall. commit 6d4cc883ec2470500be6c95fd2e7c6944e89c3e8 Author: Bjoern B. Brandenburg Date: Sat Feb 12 16:40:43 2011 -0500 bugfix: release master CPU must signal task was picked Otherwise, the release master CPU may try to reschedule in an infinite loop. commit 0f6a8e02773f8c23b5b6a3dbfa044e50c9d7d811 Author: Glenn Elliott Date: Thu Mar 31 10:47:01 2011 -0400 Improve FMLP queue management. The next owner of a FMLP-protected resource is dequeued from the FMLP FIFO queue by unlock() (when the resource is freed by the previous owner) instead of performing the dequeue by the next owner immediately after it has been woken up. This simplifies the code a little bit and also reduces potential spinlock contention. commit c05eaa8091d2cadc20363d44a85ee454262f4bc2 Author: Bjoern B. Brandenburg Date: Thu Jan 27 20:11:59 2011 -0500 Pfair: remove sporadic_release flag Instead of having an extra flag, Pfair should just infer sporadic release based on deadlines like other plugins, too. commit 71efbc5459ef95ed902a6980eae646197529364e Author: Bjoern B. Brandenburg Date: Fri Jan 7 17:37:01 2011 -0500 Pfair: support clustered scheduling Just like C-EDF is a global scheduler that is split across several clusters, Pfair can be applied on a per-cluster basis. This patch changes the Pfair implementation to enable clustering based on the recently added generic clustering support. commit 343d4ead3b12992f494134114cf50e4f37c656c5 Author: Bjoern B. Brandenburg Date: Thu Jan 27 16:23:46 2011 -0500 Litmus core: add generic clustering support Inspired by the existing C-EDF code, this generic version will build clusters of CPUs based on a given cache level. commit 4ce37704ec0bedb28b5708d32964fca471e793d0 Author: Bjoern B. Brandenburg Date: Wed Jan 26 20:42:49 2011 -0500 Litmus core: extract userspace interface from C-EDF Make the cluster size configuration in C-EDF generic so that it can be used by other clustered schedulers. commit 963fd846e36b48d5338ef2a134d3ee8d208abc07 Author: Bjoern B. Brandenburg Date: Sat Jan 29 14:45:49 2011 -0500 Feather-Trace: rename locking trace points Since we don't expect to trace more than one lock type at a time, having protocol-specific trace points is not required. commit 7f0bd4c213ff8dca0eb3bdd887f5c62c8d30fab5 Author: Bjoern B. Brandenburg Date: Sat Jan 29 15:50:52 2011 -0500 fdso: pass userpsace config argument to object constructor As Glenn pointed out, it is useful for some protocols (e.g., k-exclusion protocols) to know the userspace configuration at object creation time. This patch changes the fdso API to pass the parameter to the object constructor, which is then in turn passed to the lock allocater. The return code from the lock allocater is passed to userspace in return. This also fixes some null pointer dereferences in the FDSO code found by the test suite in liblitmus. commit fab768a4cdc49ad7886cac0d0361f8432965a817 Author: Bjoern B. Brandenburg Date: Sat Jan 29 13:38:24 2011 -0500 GSN-EDF: re-implement FMLP support This introduces the global FMLP based on the generic locking layer. commit e705aa52df711112d434ccc87ee5fb5838c205a2 Author: Bjoern B. Brandenburg Date: Fri Jan 28 19:06:11 2011 -0500 PSN-EDF: re-implement FMLP support Implement the partitioned FMLP with priority boosting based on the generic lock API. commit e593c9dbe858c82e284ff85e625837ae3ab32f1c Author: Bjoern B. Brandenburg Date: Fri Jan 28 19:04:08 2011 -0500 EDF: support priority boosting While we are at it, simplify edf_higher_prio() a bit. commit fc6482bb7a6a638474565c90159997bd59069297 Author: Bjoern B. Brandenburg Date: Fri Jan 28 17:30:14 2011 -0500 FMLP: remove old implementation commit e1b81e70c3af9d19d639bc8bdaa5a8fc13bf17a8 Author: Bjoern B. Brandenburg Date: Fri Jan 28 17:04:58 2011 -0500 SRP: port to new generic locking API This re-enables SRP support under PSN-EDF and demonstrates how the new locking API should be used. commit cc602187d4466374bca031039e145aa1b89aca96 Author: Bjoern B. Brandenburg Date: Fri Jan 28 16:41:16 2011 -0500 Litmus core: replace FMLP & SRP system calls with generic syscalls This renders the FMLP and SRP unfunctional until they are ported to the new locking API. commit a3db326495d4051bddc657d3b226ad4daa7997c4 Author: Bjoern B. Brandenburg Date: Fri Jan 28 13:26:15 2011 -0500 Litmus core: add generic locking API Provide a unified userspace interface for plugin-specific locking protocols. commit 2dea9d5e7727b8474981557cbf925687b8f33865 Author: Bjoern B. Brandenburg Date: Fri Jan 28 12:24:58 2011 -0500 Litmus core: change plugin locking interface to generic 'allocate_lock()' As the number of supported locking protocols is expected to rise, hard-coding things like priority inheritance in the plugin interface doesn't scale. Instead, use a new generic lock-ops approach. With this approach, each plugin can define its own protocol implementation (or use a generic one), and plugins can support multiple protocols without having to change the plugin interface for each protocol. commit fd8ae31c74975c8499983c9831bff2b136b98434 Author: Bjoern B. Brandenburg Date: Fri Jan 28 11:54:38 2011 -0500 fdso: supply object type to constructor and destructor methods Passing the object type explicitly will enable generic lock constructors. commit a0f243fd1d66c3499f88a690e485e94160ac1a8c Author: Jonathan Herman Date: Sun Jan 30 15:14:20 2011 -0500 Fixed is_hrt, is_srt, and is_be macros. commit 3cb35a8d90658bd8fb6f9b4f60eb7f97d0643313 Author: Jonathan Herman Date: Sun Jan 30 15:10:49 2011 -0500 Added task class to feather trace param record. commit 904531a6321964579ab0972a8833616e97dbf582 Author: Bjoern B. Brandenburg Date: Sat Jan 29 20:31:57 2011 -0500 bugfix: don't let children stay Litmus real-time tasks It has always been LITMUS^RT policy that children of real-time tasks may not skip the admissions test, etc. This used to be enforced, but was apparently dropped during some port. This commit re-introduces this policy. This fixes a kernel panic that occurred when "real-time children" exited without proper initilization. commit 3d8eb93db513bd9caa982f27fee8156405fac754 Author: Bjoern B. Brandenburg Date: Wed Jan 26 20:36:49 2011 -0500 Litmus core: add copy_and_chomp() helper We read in a line from userspace and remove the trailing newline in a number of places. This function extracts the common code to avoid future duplication. commit 7dbc4a842d3bcfa755ba82cae46171d0098d4c2c Author: Jonathan Herman Date: Wed Jan 26 17:47:49 2011 -0500 Added support for tracing arbitrary actions. commit d11808b5c6b032de4284281ed2ff77ae697a4ebd Author: Christopher Kenna Date: Sun Jan 9 19:33:49 2011 -0500 Feather-Trace: dynamic memory allocation and clean exit This patch changes Feather-Trace to allocate memory for the minor devices dynamically, which addresses a long-standing FIXME. It also provides clean module exit and error conditions for Feather-Trace. commit 37eb46be881dde4b405d3d8b48e76b4a8d62ae2c Author: Christopher Kenna Date: Fri Jan 7 20:46:25 2011 -0500 Feather-Trace: register devices with sysfs This patch implements support for Feather-Trace devices to use the sysfs file system and, consequently, udev support. This allows us to allocate major/minor numbers for Feather-Trace devices dynamically, which is desirable because our old static allocations tend to create conflicts on modern distributions and/or when there are many cores. commit 7648363e5636bd865aeac3236eb4675f0687eb4a Author: Bjoern B. Brandenburg Date: Mon Jan 3 07:29:48 2011 -0500 cleanup C-EDF cluster size configuration Refactor the code that determines the C-EDF cluster size. - Use an enum with symbolic constants instead of magic int values. - Complain and fail to switch if an unsupported cluster size is requested. - Default to ALL as suggested by Glenn and Andrea. commit 73da50b48b6e7c60add2fcf0b683318b76ecb340 Author: Andrea Bastoni Date: Tue Dec 21 18:19:27 2010 -0500 bugfix: clear scheduled field of the correct CPU upon task_exit in C-EDF Do not use the "scheduled_on" field to address the cpus structure within a cluster. cpus may contain less items than num_online_cpus and we may cause an out-of-bound access. Instead, use "scheduled_on" to directly access the per-cpu cpu_entry_t structure. Reported-by: Jonathan Herman commit f07bb0a4549916107a7619d0bc4cb5dc09d5744a Author: Bjoern B. Brandenburg Date: Mon Nov 29 09:20:03 2010 -0500 bugfix: avoid underflow in budget_remaining() budget_remaining() reports incorrect values due to the operands being switched, which leads to an integer underflow. Reported-by: Chris Kenna commit 7b544c16beaa1f6ec70a72d53fe84cae95f70a41 Author: Andrea Bastoni Date: Thu Nov 25 13:29:31 2010 +0100 bugfix: fix out-of-bound array access in cedf_activate_plugin() Make sure to check for maximum index value when accessing cedf_domain_t array in cedf_activate_plugin(). Reported-by: Jeremy Erickson commit 2aad06b056054442964f46752bdb098030cdb866 Author: Bjoern B. Brandenburg Date: Mon Nov 22 01:25:19 2010 -0500 add optional [function@file:line] tag to TRACE() log Add information to each trace message that makes it easier to locate where it came from. It is disabled by default since this adds a lot of clutter. Example: 81281 P1 [gsnedf_schedule@litmus/sched_gsn_edf.c:406]: (rtspin/1483:1) blocks:0 out_of_time:0 np:0 sleep:1 preempt:0 state:0 sig:0 81282 P1 [job_completion@litmus/sched_gsn_edf.c:303]: (rtspin/1483:1) job_completion(). 81283 P1 [__add_release@litmus/rt_domain.c:344]: (rtspin/1483:2) add_release(), rel=41941764351 81284 P1 [gsnedf_schedule@litmus/sched_gsn_edf.c:453]: (rtspin/1483:2) scheduled_on = NO_CPU commit 7779685f05219ff6e713ee6591644c080f51a8bf Author: Bjoern B. Brandenburg Date: Mon Nov 22 00:39:45 2010 -0500 log job number in TRACE_TASK() and TRACE_CUR() For some problems it can be helpful to know which job of a task generated a log message. This patch changes TRACE_TASK to add : to the existing (/) tag. The result is a trace such as the following, in which the third job of rtspin/1511 completes and the fourth job is added to the release queue. 137615 P0: (rtspin/1511:3) job_completion(). 137616 P0: (rtspin/1511:4) add_release(), rel=262013223089 137617 P0: (rtspin/1511:4) scheduled_on = NO_CPU The job number for non-real-time tasks is always zero. commit d40413efabc0ab388f6ed83f48b28dc253d47238 Author: Andrea Bastoni Date: Fri Nov 19 12:52:08 2010 +0100 Bugfix: synchronize with all other CPUs before switching plugin The CPU triggering the plugin switch should wait until all other CPUs are in a proper state (synch_on_plugin_switch()) before performing the actual switch. Based on the original patch from Jeremy Erickson . This should solve (for most practical cases) the C-EDF-related plugin-switch problem reported on the ML. commit 1726017e944d0086f14f867befbf5ebf07adc7dd Author: Bjoern B. Brandenburg Date: Tue Nov 16 11:44:54 2010 -0500 Improve help message for TRACE() buffer It's not being allocated per cpu anymore. Further, provide a hint to the user where to find the data in userspace. commit d922f5eb1c375ab0445240110656c1d793eaad04 Author: Bjoern B. Brandenburg Date: Tue Nov 16 11:13:10 2010 -0500 Make TRACE() buffer size configurable Let the user choose an appropriate buffer size (instead of scaling with NR_CPUS). The kfifo api requires the buffer to be a power of two, so enforce this constraint in the configuration. This fixes a previously-existing compile-time error for values of NR_CPU that are not a power of two. Based on a patch by Mac Mollison . commit 6fbc3b495cccf2e4ab7d4ab674b5c576e9946bed Author: Bjoern B. Brandenburg Date: Thu Nov 11 16:54:20 2010 -0500 Workaround: do not set rq->skip_clock_update Disabling the clock update seems to be causing problems even in normal Linux, and causes major bugs under LITMUS^RT. As a workaround, just disable this "optimization" for now. Details: the idle load balancer causes tasks that suspsend to be marked with set_tsk_need_resched(). When such a task resumes, it may wrongly trigger the setting of skip_clock_update. However, a corresponding rescheduling event may not happen immediately, such that the currently-scheduled task is no longer charged for its execution time. commit 5a0df8d4e9a5da47c804d89426f06e08aa44426f Author: Bjoern B. Brandenburg Date: Thu Nov 11 02:54:40 2010 -0500 Remove LITMUS^RT TRACE_BUG_ON macro Linux now has a macro of the same name, which causes namespace collisions. Since our version is only being used in two places that haven't triggered in several years, let's just remove it. commit 7c1446ddceb89ee1ddbe5d7a90cfd4cb2bc8ad37 Author: Bjoern B. Brandenburg Date: Thu Nov 11 02:45:32 2010 -0500 Avoid warning on 64bit builds The specifier %u doesn't match sizeof() if sizeof() returns a 64bit quantity on x86_64. Always cast it to int to avoid the warning. commit 98ac0cd2bbe476d79ebf44139a6259cb8d0dc6be Author: Bjoern B. Brandenburg Date: Thu Nov 11 02:33:43 2010 -0500 Cleanup TRACE() implementation Since the intial rebased from .24 to .32, the TRACE() implementation was a hybrid between our old ringbuffer implementation and the new generic kfifo API. This was a) ugly and b) not save for TRACE() invoctations during early boot. This patch rips out the old parts and replaces the actual buffer with a static kfifo. This also increases TRACE() buffer size considerably. As we avoid a dynamic allocation, this a larger size is less problematic for debug builds. This helps a bit with holes in the debug log if the buffer-flushing task is starved. commit f599a587e1c7446a76d7d62ed7748f3c4435acd8 Author: Bjoern B. Brandenburg Date: Wed Nov 10 12:20:48 2010 -0500 Hook up LITMUS^RT remote preemption support on ARM Call into scheduler state machine in the IPI handler. commit 2c142d1028f276c6d5e58c553768ae32ed9bda68 Author: Bjoern B. Brandenburg Date: Wed Nov 10 12:25:43 2010 -0500 Hook up LITMUS^RT remote preemption support on x86 Call into scheduler state machine in the IPI handler. commit fb3df2ec261d8cd6bcb8206d9d985355214d7767 Author: Bjoern B. Brandenburg Date: Wed Nov 10 12:10:49 2010 -0500 Implement proper remote preemption support To date, Litmus has just hooked into the smp_send_reschedule() IPI handler and marked tasks as having to reschedule to implement remote preemptions. This was never particularly clean, but so far we got away with it. However, changes in the underlying Linux, and peculartities of the ARM code (interrupts enabled before context switch) break this naive approach. This patch introduces new state-machine based remote preemption support. By examining the local state before calling set_tsk_need_resched(), we avoid confusing the underlying Linux scheduler. Further, this patch avoids sending unncessary IPIs. commit 516b6601bb5f71035e8859735a25dea0da4a0211 Author: Bjoern B. Brandenburg Date: Mon Nov 8 20:21:35 2010 -0500 hook litmus tick function into hrtimer-driven ticks Litmus plugins should also be activated if ticks are triggered by hrtimer. commit 34310fd7dbc3ad98d8e7cafa4f872ba71ca00860 Author: Bjoern B. Brandenburg Date: Mon Nov 8 15:02:09 2010 -0500 Split out TRACE() from litmus.h and cleanup some includes The TRACE() functionality doesn't need all of litmus.h. Currently, it's impossible to use TRACE() in sched.h due to a circular dependency. This patch moves TRACE() and friends to litmus/sched_debug.h, which can be included in sched.h. While at it, also fix some minor include ugliness that was revealed by this change. commit c6182ba4a548baf0d1238d0df54e7d38ed299c3e Author: Bjoern B. Brandenburg Date: Mon Nov 1 19:40:02 2010 -0400 sched_trace: make buffer size configurable Large sched_trace buffers cause boot problems on the ARM box. Allow the user to specify smaller buffers. commit 8e10e1803e695a08f1fb59e90dac4ba0d8744f89 Author: Bjoern B. Brandenburg Date: Mon May 31 13:06:50 2010 -0400 ARM: hookup LITMUS^RT system calls Includes the LITMUS^RT-specifc unistd.h extension and modifies the actual syscall table. commit 9907691855fa49ec8ed317fc54a626fcd137c73b Author: Bjoern B. Brandenburg Date: Sun May 30 18:52:30 2010 -0400 ARM: Include LITMUS^RT KConfig Make the ARM built aware of the LITMUS^RT-specific options. commit dd9d29e1f6ec74af4ff7df1bbe4d05829887475f Author: Bjoern B. Brandenburg Date: Mon May 31 15:19:02 2010 -0400 ARM: provide get_cycles() for RealView PB11{MP,76} and Cortex-A8 Use the CCNT register to override the default get_cycles() implementation in arch/arm/asm/timex.h. This is useful for overhead measurements and debugging. commit b39ae3793ab590efbdb8aab63a598071782d32b8 Author: Bjoern B. Brandenburg Date: Mon May 31 15:12:58 2010 -0400 ARM: allow mach/timex.h to define get_cycles() Some platforms have access to a cycle coutner (CCNT) register in the CP15 coprocessor. This trivial change will allow such platforms to provide specialized implementations. commit 52d5524f64b4f118672f5d80235221fe1c622c18 Author: Bjoern B. Brandenburg Date: Fri Oct 22 22:47:09 2010 -0400 C-EDF: move /proc/litmus/cluster_cache to /proc/litmus/plugins/C-EDF/cluster Make use of the new per-plugin proc file infrastructure to avoid littering the global namespace. While at it, also move all the relevant bits to sched_cedf.c. In the future, each plugin's parameters should be handled in the respective plugin file. commit e06e8374b5c04aeaddf14e9686842011f80f5664 Author: Christopher Kenna Date: Fri Oct 22 21:04:34 2010 -0400 Litmus core: refactor the implementation of /proc commit 98f56816fcb5c97e0afd21a6e242bb72d5b7a551 Author: Christopher Kenna Date: Fri Oct 22 17:26:38 2010 -0400 Litmus core: per-plugin proc directories Change the Litmus proc layout so that loaded plugins are visible in /proc/litmus/plugins/loaded and add Litmus functions make_plugin_proc_dir() and remove_plugin_proc_dir() to add per-plugin proc directories. commit 3dd41424090a0ca3a660218d06afe6ff4441bad3 Merge: 5c54564 f6f94e2 Author: Andrea Bastoni Date: Sat Oct 23 01:01:49 2010 -0400 Merge commit 'v2.6.36' into wip-merge-2.6.36 Conflicts: Makefile arch/x86/include/asm/unistd_32.h arch/x86/kernel/syscall_table_32.S kernel/sched.c kernel/time/tick-sched.c Relevant API and functions changes (solved in this commit): - (API) .enqueue_task() (enqueue_task_litmus), dequeue_task() (dequeue_task_litmus), [litmus/sched_litmus.c] - (API) .select_task_rq() (select_task_rq_litmus) [litmus/sched_litmus.c] - (API) sysrq_dump_trace_buffer() and sysrq_handle_kill_rt_tasks() [litmus/sched_trace.c] - struct kfifo internal buffer name changed (buffer -> buf) [litmus/sched_trace.c] - add_wait_queue_exclusive_locked -> __add_wait_queue_tail_exclusive [litmus/fmlp.c] - syscall numbers for both x86_32 and x86_64 commit 5c5456402d467969b217d7fdd6670f8c8600f5a8 Author: Bjoern B. Brandenburg Date: Wed Sep 22 15:52:03 2010 -0400 Litmus core: allow PRECISE_ENFORCEMENT Allow all kinds of budget enforcement settings now that we have the supporting infrastructure. commit 7caae3d71eae4f5307cae98131390e9d10627c01 Author: Bjoern B. Brandenburg Date: Mon Oct 18 16:55:37 2010 -0400 Litmus core: enable precise budget enforcement Update the budget enforcement timer after each scheduling decision. commit 576b1ad144f81d3fd3bd37d18dab86cd1e8660b0 Author: Bjoern B. Brandenburg Date: Mon Oct 18 16:01:10 2010 -0400 Litmus core: add plugin-independent precise budget enforcement infrastructure Simple logic: if a task requires precise enforcement, then program a hr-timer to fire when the task must be descheduled. When the timer fires, simply activate the scheduler. When we switch to a different task, either reprogram the timer or cancel it. commit 9b718afbc5db5a808804a336c17ba896a9f048a1 Author: Bjoern B. Brandenburg Date: Wed Sep 22 17:56:59 2010 -0400 Litmus core: add macro to test for PRECISE_ENFORCEMENT Required for EDF-WM. We should implement precise enforcement in the core distribution soon anyway (once we know how it works in EDF-WM). commit bf34c69c682443b5bf2f9009b1a0039fd60e654f Author: Bjoern B. Brandenburg Date: Mon Oct 18 15:58:27 2010 -0400 Litmus core: add budget_remaining() helper Quick way to figure out how much budget a LITMUS^RT job has left. commit bd6d5f1dd586a27c2082ad4d95ee58913b471f5c Author: Andrea Bastoni Date: Tue Sep 28 11:08:17 2010 -0400 hrtimer: add init function to properly set hrtimer_start_on_info params This helper function is also useful to remind us that if we use hrtimer_pull outside the scope of triggering remote releases, we need to take care of properly set the "state" field of hrtimer_start_on_info structure. commit c8f95e3e04ffc1d96b7b615f8be9b7ac941ead15 Author: Bjoern B. Brandenburg Date: Wed Sep 22 23:13:03 2010 -0400 Litmus core: set state to TASK_RUNNING before calling wake_up() Having tasks that are !is_running() in shared structures is very confusing during development and debugging, and can likely mask bugs and/or create races. It seems like a strange choice that Linux changes a task's state only _after_ activating it. For LITMUS^RT tasks, we change this order. commit 8ad8bfcab56a140389df2ed323b56d849e6cf5fb Author: Bjoern B. Brandenburg Date: Wed Sep 22 18:17:37 2010 -0400 rt_domain_t: disable timer TRACE() spam by default These messages are highly useful when debugging races, but they quickly litter the log when looking for something else. We keep them around, but by default they shouldn't show up. commit 8cc60b37588e130bed9d418bcfbe4d64c3a91935 Author: Bjoern B. Brandenburg Date: Tue Sep 21 22:49:37 2010 -0400 rt_domain_t: add add_release_on() This API addition allows the calling code to override the release master for a given rt_domain_t object. This is particularly useful if a job is supposed to migrate to a particular CPU. This need arises for example in semi- partitioned schedulers. commit 2ed4499a959f8fc30e430b6644ec83ceb7d49ef6 Author: Bjoern B. Brandenburg Date: Tue Sep 21 12:16:00 2010 -0400 PSN-EDF: remove outdated comment ...and replace it with a more useful one. We don't directly modify Linux run queues anymore since (at least) LITMUS^RT 2008. commit 136a08dbe8c28e751b01e932420f715edb229f6b Author: Bjoern B. Brandenburg Date: Fri Jul 16 10:30:06 2010 -0400 Bugfix: avoid link error in Feather-Trace on x86 If no events are defined but Feater-Trace support is enabled, then the current implementation generates a link error because the __event_table sections is absent. > arch/x86/built-in.o: In function `ft_disable_all_events': > (.text+0x242af): undefined reference to `__start___event_table' As a simple work around, we force zero-element array to always be "allocated" in the __event_table section. This ensures that we end up with a zero-byte section if no events are enabled, and does not affect the layout of the section if events are present. > bbb@ludwig:~/dev/litmus2010$ nm vmlinux | grep event_table > ffffffff81950cdc D __event_table_dummy > ffffffff81950cdc A __start___event_table > ffffffff81950cdc A __stop___event_table commit cbc5d49e4973400737aab50b60dc5d86e71f5420 Author: Andrea Bastoni Date: Sat Jun 19 13:45:36 2010 -0400 Bugfix: avoid conditional compilation dependent error If RELEASE_MASTER is not selected the "info" hrtimer_start_on_info structure in release_heap structure is not visible and trying to access "info" from reinit_release_heap() causes the following error: error: 'struct release_heap' has no member named 'info' info should not be referenced if RELEASE_MASTER is not used. The problem was first reported by Glenn commit d1aa1956eb23202e4d614574f686e53b8785212c Author: Andrea Bastoni Date: Sat Jun 12 20:22:16 2010 -0400 Bugfix: change __ARCH_HAS_SEND_PULL_TIMERS in CONFIG_ARCH_HAS_SEND_PULL_TIMERS Commit "0c527966 Make release master support optional" uses __ARCH_HAS_SEND_PULL_TIMERS instead of CONFIG_ARCH_HAS_SEND_PULL_TIMERS (introduced in commit 0fb33c99) to conditionally compile a pull timer related code in rt_domain.c. This code is disabled and pull-timer's state is no longer properly reset. Therefore, a pulled timer cannot be armed anymore. commit 9840983a4f30145bcf0b82b6e2bc8518e7212fb5 Author: Andrea Bastoni Date: Wed Jun 2 18:27:47 2010 -0400 Make litmus_sched_class static litmus_sched_class wasn't declared static, but it's not used outside sched.c, so change it's signature to static. commit 753fb14dfb0662e1d38758ffc6876c0ab1c7bd9e Author: Bjoern B. Brandenburg Date: Mon May 31 12:52:35 2010 -0400 Make platform-specific Feather-Trace depend on !CONFIG_DEBUG_RODATA Feather-Trace rewrites instructions in the kernel's .text segment. This segment may be write-protected if CONFIG_DEBUG_RODATA is selected. In this case, fall back to the default flag-based Feather-Trace implementation. In the future, we could either adopt the ftrace method of rewriting .text addresses using non-.text mappings or we could consider replacing Feather-Trace with ftrace altogether. For now, this patch avoids unexpected runtime errors. commit 62c186fde48926a30f4e61332a805430dc1325cd Author: Bjoern B. Brandenburg Date: Mon May 31 12:15:51 2010 -0400 Make PFAIR optional to prevent build and runtime failures. The PFAIR plugin always implicitly assumed !NO_HZ (the schedule is wrong if NO_HZ is enabled) and does not built if hrtimers are absent: > litmus/built-in.o: In function `pfair_activate_plugin': > sched_pfair.c:(.text+0x7f07): undefined reference to `cpu_stagger_offset' > litmus/built-in.o: In function `init_pfair': > sched_pfair.c:(.init.text+0x487): undefined reference to `cpu_stagger_offset' cpu_stagger_offset() is only available if hrtimers are enabled. This patch makes these dependencies explicit. commit 4382e90cf851fc1d209a466bab92e256aeb7acf1 Author: Andrea Bastoni Date: Mon May 31 00:54:07 2010 -0400 Make C-EDF depend on x86 and SYSFS C-EDF depends on intel_cacheinfo.c (for get_shared_cpu_map()) which is only available on x86 architectures. Furthermore, get_shared_cpu_map() is only available if SYSFS filesystem is present. commit 8bf9de45b663e4b9ce889eb24929ce773f306339 Author: Bjoern B. Brandenburg Date: Sun May 30 19:50:52 2010 -0400 Make smp_send_pull_timers() optional. There is currently no need to implement this in ARM. So let's make it optional instead. commit cedc8df1cf1ff935af5455a9d565dac05192a47f Author: Bjoern B. Brandenburg Date: Sun May 30 19:46:21 2010 -0400 Make release master support optional Introduces CONFIG_RELEASE_MASTER and makes release master support dependent on the new symbol. This is useful because dedicated interrupt handling only applies to "large" multicore platforms. This will allow us to not implement smp_send_pull_timers() for all platforms. commit 5b54b24c13b7c5dbaa06eae5e1a0075da354289c Author: Bjoern B. Brandenburg Date: Sun May 30 18:59:30 2010 -0400 Make compilation of C-EDF optional. C-EDF only makes sense on multicore platforms that have shared caches. Make it possible to disable it on other platforms, in particular, on those that do not export get_shared_cpu_map(). commit 152968b15afb74a6adba6d512c5eebf0280c8f00 Author: Bjoern B. Brandenburg Date: Sun May 30 18:41:28 2010 -0400 Make __ARCH_HAS_FEATHER_TRACE a proper CONFIG_ variable. The idea of the Feather-Trace default implementation is that LITMUS^RT should work without a specialized Feather-Trace implementation present. This was actually broken. Changes litmus/feather_trace.h to only include asm/feather_trace.h if actually promised by the architecture. commit a7205820bae197a89fc746f9f3c07e389d7068ba Author: Andrea Bastoni Date: Fri May 28 15:45:23 2010 -0400 Bugfix: re-insert missing TS_PLUGIN_TICK_END tracing point Insert PLUGIN_TICK_END tracing point in litmus_tick(). It was lost during the porting of 2008.3 to 2010.1. commit de2d5dfa2dce8ec40555b3bb6dfe21627e472c52 Author: Andrea Bastoni Date: Thu May 20 16:14:00 2010 -0400 Add support for one single cluster (all cpus) on C-EDF - With the "ALL" cluster size option the behavior of C-EDF is equivalent to G-EDF (one single cluster) commit 6f89d4f31485546674187cf3b4d472f230b263d0 Author: Glenn Elliott Date: Thu May 20 14:33:27 2010 -0400 Added support for choices in budget policy enforcement. NO_ENFORCEMENT - A job may execute beyond its declared execution time. Jobs notify the kernel that they are complete via liblitmus's sleep_next_period() QUANTUM_ENFORCEMENT - The kernel terminates a job if its actual execution time exceeds the declared execution time. PRECISE_ENFORCEMENT - Hook declared, but not yet implemented. Plan to support this policy through hrtimers. Error thrown if specified. commit 521422c4ef2c64731f709030915a7b301709f4b4 Author: Andrea Bastoni Date: Sat May 29 23:53:40 2010 -0400 Update kfifo and spinlock_t in sched_trace.c - kfifo needs to be defined and used differently (see include/linux/kfifo.h) - spinlock -> raw_spinlock - include slab.h when using kmalloc and friends This commit compiles and is the logical end of the merge of Litmus and 2.6.34. commit 8e9830a5bdb081fd3f4387db3a3838a687dfdad2 Author: Andrea Bastoni Date: Sat May 29 23:50:17 2010 -0400 Update sched_class and spinlock_t in litmus.c - get_rr_interval() changed signature - load_balance() and move_one_tak() are no longer needed - spinlock_t -> raw_spinlock_t This commit does not compile. commit a66246f9e973a68fb9955a2fa7663a2e02afbd30 Author: Andrea Bastoni Date: Sat May 29 23:45:13 2010 -0400 Change most LitmusRT spinlock_t in raw_spinlock_t Adapt to new schema for spinlock: (tglx 20091217) spinlock - the weakest one, which might sleep in RT raw_spinlock - spinlock which always spins even on RT arch_spinlock - the hardware level architecture dependent implementation ---- Most probably, all the spinlocks changed by this commit will be true spinning lock (raw_spinlock) in PreemptRT (so hopefully we'll need few changes when porting Litmmus to PreemptRT). There are a couple of spinlock that the kernel still defines as spinlock_t (therefore no changes reported in this commit) that might cause us troubles: - wait_queue_t lock is defined as spinlock_t; it is used in: * fmlp.c -- sem->wait.lock * sync.c -- ts_release.wait.lock - rwlock_t used in fifo implementation in sched_trace.c * this need probably to be changed to something always spinning in RT at the expense of increased locking time. ---- This commit also fixes warnings and errors due to the need to include slab.h when using kmalloc() and friends. ---- This commit does not compile. commit 6ffc1fee98c4b995eb3a0285f4f8fb467cb0306e Merge: e40152e 7c1ff4c Author: Andrea Bastoni Date: Sat May 29 23:35:01 2010 -0400 Merge branch 'master' into wip-merge-2.6.34 Simple merge between master and 2.6.34 with conflicts resolved. This commit does not compile, the following main problems are still unresolved: - spinlock -> raw_spinlock API changes - kfifo API changes - sched_class API changes Conflicts: Makefile arch/x86/include/asm/hw_irq.h arch/x86/include/asm/unistd_32.h arch/x86/kernel/syscall_table_32.S include/linux/hrtimer.h kernel/sched.c kernel/sched_fair.c commit 7c1ff4c544dd650cceff3cd69a04bcba60856678 Author: Andrea Bastoni Date: Fri May 28 10:51:01 2010 -0400 Add C-EDF Plugin Improved C-EDF plugin. C-EDF now supports different cluster sizes (based on L2 and L3 cache sharing) and supports dynamic changes of cluster size (this requires reloading the plugin). commit 425a6b5043bcc2142804107c853f978ac2fe3040 Author: Andrea Bastoni Date: Fri May 28 10:49:09 2010 -0400 Export shared_cpu_map The cpumap of CPUs that share the same cache level is not normally available outside intel_cacheinfo.c. This commit allows to export such map. commit f85625ccf28d1bffd4dac916babb76b910ebef31 Author: Andrea Bastoni Date: Tue Apr 27 11:00:19 2010 -0400 Synchronize plugin switching Make sure the plugin is not used by any CPUs while switching. The CPU performing the switch sends an IPI to all other CPUs forcing them to synchronize on an atomic variable. commit 8fe2fb8bb1c1cd0194608bc783d0ce7029e8d869 Author: Andrea Bastoni Date: Mon Apr 26 13:42:00 2010 -0400 Measure timer re-arming in the proper location hrtimers are properly rearmed during arm_release_timer() and no longer after rescheduling (with the norqlock mechanism of 2008.3). This commit accordingly updates the locations where measures are taken. commit 5da9b3e7aab0755f6ca19738d33e218e02b19a41 Author: Andrea Bastoni Date: Fri Mar 12 12:23:13 2010 -0500 Bugfix: PSN-EDF should log job_completion events Log task completions in job_completion() for PSN-EDF. This fixes the problem of missing job-completion events for PSN-EDF. commit 7a4affe47db86075eb36519049d047f6facab378 Author: Bjoern B. Brandenburg Date: Tue Mar 2 11:51:07 2010 -0500 Bugfix: PSN-EDF should only requeue tasks that are not scheduled Requeue a task that is already scheduled will cause it to be effectively in the runqueue twice since scheduled tasks are conceptually the head of the queue. If a task is still scheduled, then schedule() will do the right thing and do the requeuing if necessary. This fixes crashes reported by Glenn and Andrea. commit 0c1a489cb92c996d50adfb84fee5edd7205e0c1b Author: Bjoern B. Brandenburg Date: Thu Feb 4 19:47:29 2010 -0500 Used miscdevice API for sched_trace This patch changes sched_trace.c to use the miscdevice API instead of doing all the cdev management ourselves. This remove a chunk of code and we get sysfs / udev integration for free. On systems with default udev rules, this will result in a /dev/litmus/log device being created automatically. commit 8815090d72fe0fe8f5f67e3bcc8fbe7a5ad1704d Author: Bjoern B. Brandenburg Date: Thu Feb 25 19:33:22 2010 -0500 Bugfix: make fdso syscalls 64bit clean This fixes a bug found by liblitmus's regression test suite. Before: > ** LITMUS^RT test suite. > ** Running tests for LINUX. > ** Testing: don't open FMLP semaphores if FMLP is not supported... > !! TEST FAILURE open_fmlp_sem(fd, 0) -> -16, Success (expected: EBUSY) > at tests/fdso.c:21 (test_fmlp_not_active) > ** Testing: reject invalid object descriptors... ok. > ** Testing: reject invalid object types... > !! TEST FAILURE od_open(0, -1, 0) -> -22, Bad file descriptor (expected: EINVAL) > at tests/fdso.c:51 (test_invalid_obj_type) > ** Testing: reject invalid rt_task pointers... ok. > ** Result: 2 ok, 2 failed. After: > ** LITMUS^RT test suite. > ** Running tests for LINUX. > ** Testing: don't open FMLP semaphores if FMLP is not supported... ok. > ** Testing: reject invalid object descriptors... ok. > ** Testing: reject invalid object types... ok. > ** Testing: reject invalid rt_task pointers... ok. > ** Result: 4 ok, 0 failed. commit 8ad3e04b815e44d084b855cfa3dcda260cdf56ae Author: Bjoern B. Brandenburg Date: Fri Feb 19 13:42:35 2010 -0500 Bugfix: don't inherit od_table across forks The od_table is strictly per-thread and should not be inherited across a fork/clone. This caused memory corruption when a task exited, which ultimately could lead to oopses in unrelated code. Bug and testcase initially reported by Glenn. commit 944f051fda9551483399bed556870b0895df1efa Author: Glenn Elliott Date: Fri May 28 10:39:56 2010 -0400 Bugfix: 1) incorrect FMLP high prio task tracking and 2) race in print statement 1) High priority task tied to FMLP semaphore in P-EDF scheduling is incorrectly tracked for tasks acquiring the lock without contention. (HP is always set to CPU 0 instead of proper CPU.) 2) Race in a print statement from P-EDF's pi_block() causes NULL pointer dereference. commit 9039e5f731ca5f9a0c69f8523ccfee044111d2e3 Author: Bjoern B. Brandenburg Date: Wed Feb 3 19:56:21 2010 -0500 Use generic preemption function in GSN- and PSN-EDF. This patch updates non-preemptive section support in GSN- and PSN-EDF. commit f3a6cb9af5cdb01f29ad32b01aa56a14f0da144e Author: Bjoern B. Brandenburg Date: Wed Feb 3 19:42:02 2010 -0500 Introduce generic NP-section aware preemption function Dealing with preemptions across CPUs in the presence of non-preemptive sections can be tricky and should not be replicated across (event-driven) plugins. This patch introduces a generic preemption function that handles non-preemptive sections (hopefully) correctly. commit fb95c290fe461de794c984bc4130741f04f9142d Author: Bjoern B. Brandenburg Date: Wed Feb 3 19:40:01 2010 -0500 Re-implement non-preemptive section support. Re-introduce NP sections in the configuration and in litmus.h. Remove the old np_flag from rt_param. If CONFIG_NP_SECTION is disabled, then all non-preemptive section checks are constant expressions which should get removed by the dead code elimination during optimization. Instead of re-implementing sys_exit_np(), we simply repurposed sched_yield() for calling into the scheduler to trigger delayed preemptions. commit b973c95c86e6710c913c01a67013605f68a3c2c3 Author: Bjoern B. Brandenburg Date: Wed Feb 3 19:35:20 2010 -0500 Add virtual LITMUS^RT control device. This device only supports mmap()'ing a single page. This page is shared RW between the kernel and userspace. It is inteded to allow near-zero-overhead communication between the kernel and userspace. It's first use will be a proper implementation of user-signaled non-preemptable section support. commit 5e987d486c0f89d615d134512938fc1198b3ca67 Author: Bjoern B. Brandenburg Date: Fri Jan 29 19:25:26 2010 -0500 Bugfix: clear LITMUS^RT state on fork completely When a real-time task forks, then its LITMUS^RT-specific fields should be cleared, because we don't want real-time tasks to spawn new real-time tasks that bypass the plugin's admission control (if any). This was broken in three ways: 1) kernel/fork.c did not erase all of tsk->rt_param, only the first few bytes due to a wrong size argument to memset(). 2) It should have been calling litmus_fork() instead anyway. 3) litmus_fork() was _also_ not clearing all of tsk->rt_param, due to another size argument bug. Interestingly, 1) and 2) can be traced back to the 2007->2008 port, whereas 3) was added by Mitchell much later on (to dead code, no less). I'm really surprised that this never blew up before. commit 37b840336a1663a5ce62d663a702d9afefd56d23 Author: Andrea Bastoni Date: Mon Feb 1 23:07:54 2010 -0500 Add Feather-Trace x86_64 architecture dependent code commit d1a840d7194fdd09c1bd9977e30fd391ef2a7526 Author: Andrea Bastoni Date: Tue Jan 19 19:38:14 2010 -0500 [ported from 2008.3] Add Feather-Trace x86_32 architecture dependent code - [ported from 2008.3] Add x86_32 architecture dependent code. - Add the infrastructure for x86_32 - x86_64 integration. commit 07ae7efcb81f95eb8e870cad21c7ba72573af7e8 Author: Andrea Bastoni Date: Thu Dec 17 21:48:38 2009 -0500 Add support for x86_64 architecture - Add syscall on x86_64 - Refactor __NR_sleep_next_period -> __NR_complete_job for both x86_32 and x86_64 commit 5306b9834e9660e370fb8430ff22d4a47b4bbdf5 Author: Andrea Bastoni Date: Thu Dec 17 21:47:51 2009 -0500 Add pull_timers_interrupt() to x86_64 Add apic interrupt vector for pull_timers() in x86_64 arch. commit b30bc467e88c9f1e6335ac7d442d0190bf6f6a2e Author: Andrea Bastoni Date: Thu Jan 28 19:03:17 2010 -0500 [ported from 2008.3] Add PSN-EDF Plugin commit c2f4c165b208062d90f65a1c1a0c815261c6a81e Author: Andrea Bastoni Date: Wed Jan 27 19:57:09 2010 -0500 [ported from 2008.3] Add PFAIR plugin commit cddade083e5ea74cba6f0e4b2fa10c6bbec1336c Author: Andrea Bastoni Date: Sat Jan 16 19:39:40 2010 -0500 Add optional dynamic assignment of tracing devices major nr Setting FT_TASK_TRACE_MAJOR, LOG_MAJOR, FT_TRACE_MAJOR to 0 allows to have them automatically assigned by the kernel commit a084c01569bcfe13fd880a0b1e3a9026629a89da Author: Bjoern B. Brandenburg Date: Fri May 28 10:30:29 2010 -0400 Better explanation of jump-to-CFS optimization removal GSN-EDF and friends rely on being called even if there is currently no runnable real-time task on the runqueue for (at least) two reasons: 1) To initiate migrations. LITMUS^RT pull tasks for migrations; this requires plugins to be called even if no task is currently present. 2) To maintain invariants when jobs block. commit e68debebdc2983600063cd6b04c6a51c4b7ddcc1 Author: Andrea Bastoni Date: Fri May 28 10:25:34 2010 -0400 Integrate litmus_tick() in task_tick_litmus() - remove the call to litmus_tick() from scheduler_tick() just after having performed the class task_tick() and integrate litmus_tick() in task_tick_litmus() - task_tick_litmus() is the handler for the litmus class task_tick() method. It is called in non-queued mode from scheduler_tick() commit 9ac80419f88f192cdf586da3df585c224ef27773 Author: Bjoern B. Brandenburg Date: Wed Feb 3 13:59:40 2010 -0500 Turn off GSN-EDF TRACE() spam by default. Having GSN-EDF log so many things each tick is useful when tracking down race conditions, but it also makes it really hard to find anything else. Thus, turn it off by default but leave it in for future debugging fun. commit ee09f78d8faa0b988088d93142e6f5f8a6e75394 Author: Andrea Bastoni Date: Mon Dec 21 12:23:57 2009 -0500 Refactor binomial heap names: heap -> bheap - Binomial heap "heap" names conflicted with priority heap of cgroup in kernel - This patch change binomial heap "heap" names in "bheap" commit 0b28a3122d6917784701377e15a863489aee1c6c Author: Andrea Bastoni Date: Thu Dec 17 21:47:19 2009 -0500 [ported from 2008.3] Add release-master support commit c15be843778236e9f2fdbc207ab36ba996b2bb1b Author: Andrea Bastoni Date: Thu Dec 17 21:45:38 2009 -0500 [ported from 2008.3] Add hrtimer_start_on() API commit b085cafc43bc395e255626204169e20a587f28ba Author: Andrea Bastoni Date: Thu Dec 17 21:44:47 2009 -0500 [ported from 2008.3] Add send_pull_timers() support for x86_32 arch commit 50ca05ff9cc85176c3ee18bf1363d3d7c34aa355 Author: Andrea Bastoni Date: Thu Dec 17 21:39:14 2009 -0500 [ported from 2008.3] Add GSN-EDF plugin - insert arm_release_timer() in add_relese() path - arm_release_timer() uses __hrtimer_start_range_ns() instead of hrtimer_start() to avoid deadlock on rq->lock. commit 2a94c7bf9869a13e32de7a1fe94596de7b4789a8 Author: Andrea Bastoni Date: Fri May 28 10:03:24 2010 -0400 [ported from 2008.3] Add LITRMUS^RT syscalls to x86_32 commit 269cf3c49cef2b23605e98ad4a8133357bebaac0 Author: Andrea Bastoni Date: Thu Dec 17 21:36:40 2009 -0500 [ported from 2008.3] Add FMLP support commit 5442a8adfce93c1cd556e04bfc0a118adc3b683e Author: Andrea Bastoni Date: Thu Dec 17 21:34:09 2009 -0500 [ported from 2008.3] Add Stack Resource Policy (SRP) support commit fa3c94fc9cd1619fe0dd6081a1a980c09ef3e119 Author: Andrea Bastoni Date: Thu Dec 17 21:33:26 2009 -0500 [ported from 2008.3] Add File Descriptor Attached Shared Objects (FDSO) infrastructure commit f5936ecf0cff0b94419b6768efba3e15622beeb6 Author: Andrea Bastoni Date: Thu Dec 17 21:32:31 2009 -0500 [ported from 2008.3] Add common EDF functions commit 53696c1fe6a6ada66f2a47c078d62aee40ad8ebe Author: Andrea Bastoni Date: Thu Dec 17 21:31:46 2009 -0500 [ported from 2008.3] Add rt_domain_t support Still to be merged: - arm_release_timer() with no rq locking commit 4e593e7105dec02e62ea7a1812dccb35a0d56d01 Author: Andrea Bastoni Date: Thu Dec 17 21:30:47 2009 -0500 [ported from 2008.3] Add support for quantum alignment commit 1d823f50678d7cc3bf72bf89ec0bddc7338e23d5 Author: Andrea Bastoni Date: Thu Dec 17 21:30:11 2009 -0500 [ported from 2008.3] Add synchronous task release API commit 59d8d4c53f1e9f6408b87fc22e319e78f664276f Author: Andrea Bastoni Date: Thu Dec 17 21:29:31 2009 -0500 [ported from 2008.3] Add complete_n() call commit 2079f38466395c64ef40ef3429ee52fd92cdbd99 Author: Andrea Bastoni Date: Sat Jan 16 19:36:50 2010 -0500 Move sched_trace ring buffer to kfifo implementation Use kfifo [kernel/kfifo.c] to implement the ring buffer used for sched_trace (TRACE() and TRACE_TASK() macros) This patch also includes some reorganization of sched_trace.c code and some fixes: - 1c39c59b3 Fix GFP_KERNEL in rb_alloc_buf with interrupt disabled. - 193ad2688 Let TRACE() log buffer size and comment converge. - 6195e2ae8 re-enable capturing of printk() messages in TRACE() logs. commit 96979188007a0671d3f067d7edf144742d7433ee Author: Andrea Bastoni Date: Thu Dec 17 21:26:50 2009 -0500 [ported from 2008.3] Add tracing support and hook up Litmus KConfig for x86 - fix requesting more than 2^11 pages (MAX_ORDER) to system allocator Still to be merged: - feather-trace generic implementation commit cf3f4bd8db320f3f487d66bdec924e926f004787 Author: Andrea Bastoni Date: Thu Dec 17 21:24:47 2009 -0500 [ported from 2008.3] Add Feather-Trace device file support commit 4b38febbd59fd33542a343991262119eb9860f5e Author: Andrea Bastoni Date: Thu Dec 17 21:23:36 2009 -0500 [ported from 2008.3] Core LITMUS^RT infrastructure Port 2008.3 Core LITMUS^RT infrastructure to Linux 2.6.32 litmus_sched_class implements 4 new methods: - prio_changed: void - switched_to: void - get_rr_interval: return infinity (i.e., 0) - select_task_rq: return current cpu --- Makefile | 4 +- arch/arm/Kconfig | 8 + arch/arm/include/asm/timex.h | 2 + arch/arm/include/asm/unistd.h | 3 + arch/arm/kernel/calls.S | 12 + arch/arm/kernel/smp.c | 4 + arch/arm/mach-realview/include/mach/timex.h | 27 + arch/x86/Kconfig | 8 + arch/x86/include/asm/entry_arch.h | 1 + arch/x86/include/asm/feather_trace.h | 17 + arch/x86/include/asm/feather_trace_32.h | 79 + arch/x86/include/asm/feather_trace_64.h | 67 + arch/x86/include/asm/hw_irq.h | 3 + arch/x86/include/asm/irq_vectors.h | 7 + arch/x86/include/asm/processor.h | 4 + arch/x86/include/asm/unistd_32.h | 6 +- arch/x86/include/asm/unistd_64.h | 4 + arch/x86/kernel/Makefile | 2 + arch/x86/kernel/cpu/intel_cacheinfo.c | 17 + arch/x86/kernel/entry_64.S | 2 + arch/x86/kernel/ft_event.c | 118 ++ arch/x86/kernel/irq.c | 4 + arch/x86/kernel/irqinit.c | 3 + arch/x86/kernel/smp.c | 31 + arch/x86/kernel/syscall_table_32.S | 13 + fs/exec.c | 13 +- fs/inode.c | 2 + include/linux/completion.h | 2 + include/linux/fs.h | 21 +- include/linux/hardirq.h | 4 + include/linux/hrtimer.h | 32 + include/linux/interrupt.h | 12 +- include/linux/mutex.h | 10 + include/linux/sched.h | 19 +- include/linux/semaphore.h | 9 + include/linux/smp.h | 5 + include/linux/tick.h | 5 + include/linux/workqueue.h | 18 + include/litmus/affinity.h | 80 + include/litmus/bheap.h | 77 + include/litmus/binheap.h | 207 ++ include/litmus/budget.h | 8 + include/litmus/clustered.h | 44 + include/litmus/debug_trace.h | 37 + include/litmus/edf_common.h | 37 + include/litmus/fdso.h | 83 + include/litmus/feather_buffer.h | 94 + include/litmus/feather_trace.h | 65 + include/litmus/fpmath.h | 145 ++ include/litmus/ftdev.h | 55 + include/litmus/gpu_affinity.h | 49 + include/litmus/ikglp_lock.h | 160 ++ include/litmus/jobs.h | 9 + include/litmus/kexclu_affinity.h | 35 + include/litmus/kfmlp_lock.h | 97 + include/litmus/litmus.h | 282 +++ include/litmus/litmus_proc.h | 25 + include/litmus/litmus_softirq.h | 199 ++ include/litmus/locking.h | 160 ++ include/litmus/nvidia_info.h | 46 + include/litmus/preempt.h | 164 ++ include/litmus/rsm_lock.h | 54 + include/litmus/rt_domain.h | 182 ++ include/litmus/rt_param.h | 307 +++ include/litmus/sched_plugin.h | 183 ++ include/litmus/sched_trace.h | 380 ++++ include/litmus/sched_trace_external.h | 78 + include/litmus/srp.h | 28 + include/litmus/trace.h | 148 ++ include/litmus/trace_irq.h | 21 + include/litmus/unistd_32.h | 24 + include/litmus/unistd_64.h | 40 + kernel/exit.c | 4 + kernel/fork.c | 7 + kernel/hrtimer.c | 95 + kernel/lockdep.c | 7 +- kernel/mutex.c | 125 ++ kernel/printk.c | 14 +- kernel/sched.c | 164 +- kernel/sched_fair.c | 3 + kernel/sched_rt.c | 2 +- kernel/semaphore.c | 13 +- kernel/softirq.c | 322 ++- kernel/time/tick-sched.c | 47 + kernel/workqueue.c | 71 +- litmus/Kconfig | 364 ++++ litmus/Makefile | 38 + litmus/affinity.c | 42 + litmus/bheap.c | 314 +++ litmus/binheap.c | 443 +++++ litmus/budget.c | 111 ++ litmus/clustered.c | 111 ++ litmus/ctrldev.c | 150 ++ litmus/edf_common.c | 211 ++ litmus/fdso.c | 306 +++ litmus/ft_event.c | 43 + litmus/ftdev.c | 439 +++++ litmus/gpu_affinity.c | 113 ++ litmus/ikglp_lock.c | 2838 +++++++++++++++++++++++++++ litmus/jobs.c | 56 + litmus/kexclu_affinity.c | 92 + litmus/kfmlp_lock.c | 1002 ++++++++++ litmus/litmus.c | 684 +++++++ litmus/litmus_pai_softirq.c | 64 + litmus/litmus_proc.c | 364 ++++ litmus/litmus_softirq.c | 1582 +++++++++++++++ litmus/locking.c | 524 +++++ litmus/nvidia_info.c | 597 ++++++ litmus/preempt.c | 138 ++ litmus/rsm_lock.c | 796 ++++++++ litmus/rt_domain.c | 357 ++++ litmus/sched_cedf.c | 1849 +++++++++++++++++ litmus/sched_gsn_edf.c | 1862 ++++++++++++++++++ litmus/sched_litmus.c | 327 +++ litmus/sched_pfair.c | 1067 ++++++++++ litmus/sched_plugin.c | 360 ++++ litmus/sched_psn_edf.c | 645 ++++++ litmus/sched_task_trace.c | 509 +++++ litmus/sched_trace.c | 252 +++ litmus/sched_trace_external.c | 64 + litmus/srp.c | 295 +++ litmus/sync.c | 104 + litmus/trace.c | 225 +++ 123 files changed, 24316 insertions(+), 97 deletions(-) create mode 100644 arch/x86/include/asm/feather_trace.h create mode 100644 arch/x86/include/asm/feather_trace_32.h create mode 100644 arch/x86/include/asm/feather_trace_64.h create mode 100644 arch/x86/kernel/ft_event.c create mode 100644 include/litmus/affinity.h create mode 100644 include/litmus/bheap.h create mode 100644 include/litmus/binheap.h create mode 100644 include/litmus/budget.h create mode 100644 include/litmus/clustered.h create mode 100644 include/litmus/debug_trace.h create mode 100644 include/litmus/edf_common.h create mode 100644 include/litmus/fdso.h create mode 100644 include/litmus/feather_buffer.h create mode 100644 include/litmus/feather_trace.h create mode 100644 include/litmus/fpmath.h create mode 100644 include/litmus/ftdev.h create mode 100644 include/litmus/gpu_affinity.h create mode 100644 include/litmus/ikglp_lock.h create mode 100644 include/litmus/jobs.h create mode 100644 include/litmus/kexclu_affinity.h create mode 100644 include/litmus/kfmlp_lock.h create mode 100644 include/litmus/litmus.h create mode 100644 include/litmus/litmus_proc.h create mode 100644 include/litmus/litmus_softirq.h create mode 100644 include/litmus/locking.h create mode 100644 include/litmus/nvidia_info.h create mode 100644 include/litmus/preempt.h create mode 100644 include/litmus/rsm_lock.h create mode 100644 include/litmus/rt_domain.h create mode 100644 include/litmus/rt_param.h create mode 100644 include/litmus/sched_plugin.h create mode 100644 include/litmus/sched_trace.h create mode 100644 include/litmus/sched_trace_external.h create mode 100644 include/litmus/srp.h create mode 100644 include/litmus/trace.h create mode 100644 include/litmus/trace_irq.h create mode 100644 include/litmus/unistd_32.h create mode 100644 include/litmus/unistd_64.h create mode 100644 litmus/Kconfig create mode 100644 litmus/Makefile create mode 100644 litmus/affinity.c create mode 100644 litmus/bheap.c create mode 100644 litmus/binheap.c create mode 100644 litmus/budget.c create mode 100644 litmus/clustered.c create mode 100644 litmus/ctrldev.c create mode 100644 litmus/edf_common.c create mode 100644 litmus/fdso.c create mode 100644 litmus/ft_event.c create mode 100644 litmus/ftdev.c create mode 100644 litmus/gpu_affinity.c create mode 100644 litmus/ikglp_lock.c create mode 100644 litmus/jobs.c create mode 100644 litmus/kexclu_affinity.c create mode 100644 litmus/kfmlp_lock.c create mode 100644 litmus/litmus.c create mode 100644 litmus/litmus_pai_softirq.c create mode 100644 litmus/litmus_proc.c create mode 100644 litmus/litmus_softirq.c create mode 100644 litmus/locking.c create mode 100644 litmus/nvidia_info.c create mode 100644 litmus/preempt.c create mode 100644 litmus/rsm_lock.c create mode 100644 litmus/rt_domain.c create mode 100644 litmus/sched_cedf.c create mode 100644 litmus/sched_gsn_edf.c create mode 100644 litmus/sched_litmus.c create mode 100644 litmus/sched_pfair.c create mode 100644 litmus/sched_plugin.c create mode 100644 litmus/sched_psn_edf.c create mode 100644 litmus/sched_task_trace.c create mode 100644 litmus/sched_trace.c create mode 100644 litmus/sched_trace_external.c create mode 100644 litmus/srp.c create mode 100644 litmus/sync.c create mode 100644 litmus/trace.c diff --git a/Makefile b/Makefile index 6a5bdad..a327725 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ VERSION = 3 PATCHLEVEL = 0 SUBLEVEL = 0 -EXTRAVERSION = +EXTRAVERSION =-litmus NAME = Sneaky Weasel # *DOCUMENTATION* @@ -708,7 +708,7 @@ export mod_strip_cmd ifeq ($(KBUILD_EXTMOD),) -core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ +core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ litmus/ vmlinux-dirs := $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \ $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 9adc278..fb228ea 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -2040,3 +2040,11 @@ source "security/Kconfig" source "crypto/Kconfig" source "lib/Kconfig" + +config ARCH_HAS_SEND_PULL_TIMERS + def_bool n + +config ARCH_HAS_FEATHER_TRACE + def_bool n + +source "litmus/Kconfig" diff --git a/arch/arm/include/asm/timex.h b/arch/arm/include/asm/timex.h index 3be8de3..8a102a3 100644 --- a/arch/arm/include/asm/timex.h +++ b/arch/arm/include/asm/timex.h @@ -16,9 +16,11 @@ typedef unsigned long cycles_t; +#ifndef get_cycles static inline cycles_t get_cycles (void) { return 0; } +#endif #endif diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h index 2c04ed5..0196edf 100644 --- a/arch/arm/include/asm/unistd.h +++ b/arch/arm/include/asm/unistd.h @@ -403,6 +403,9 @@ #define __NR_sendmmsg (__NR_SYSCALL_BASE+374) #define __NR_setns (__NR_SYSCALL_BASE+375) +#define __NR_LITMUS (__NR_SYSCALL_BASE+376) +#include + /* * The following SWIs are ARM private. */ diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S index 80f7896..ed2ae93 100644 --- a/arch/arm/kernel/calls.S +++ b/arch/arm/kernel/calls.S @@ -385,6 +385,18 @@ CALL(sys_syncfs) CALL(sys_sendmmsg) /* 375 */ CALL(sys_setns) + CALL(sys_set_rt_task_param) + CALL(sys_get_rt_task_param) + CALL(sys_complete_job) + CALL(sys_od_open) +/* 380 */ CALL(sys_od_close) + CALL(sys_litmus_lock) + CALL(sys_litmus_unlock) + CALL(sys_query_job_no) + CALL(sys_wait_for_job_release) +/* 385 */ CALL(sys_wait_for_ts_release) + CALL(sys_release_ts) + CALL(sys_null_call) #ifndef syscalls_counted .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls #define syscalls_counted diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index e7f92a4..5a57429 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -40,6 +40,8 @@ #include #include +#include + /* * as from 2.5, kernels no longer have an init_tasks structure * so we need some other way of telling a new secondary core @@ -572,6 +574,8 @@ asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs) break; case IPI_RESCHEDULE: + /* LITMUS^RT: take action based on scheduler state */ + sched_state_ipi(); scheduler_ipi(); break; diff --git a/arch/arm/mach-realview/include/mach/timex.h b/arch/arm/mach-realview/include/mach/timex.h index 4eeb069..e8bcc40 100644 --- a/arch/arm/mach-realview/include/mach/timex.h +++ b/arch/arm/mach-realview/include/mach/timex.h @@ -21,3 +21,30 @@ */ #define CLOCK_TICK_RATE (50000000 / 16) + +#if defined(CONFIG_MACH_REALVIEW_PB11MP) || defined(CONFIG_MACH_REALVIEW_PB1176) + +static inline unsigned long realview_get_arm11_cp15_ccnt(void) +{ + unsigned long cycles; + /* Read CP15 CCNT register. */ + asm volatile ("mrc p15, 0, %0, c15, c12, 1" : "=r" (cycles)); + return cycles; +} + +#define get_cycles realview_get_arm11_cp15_ccnt + +#elif defined(CONFIG_MACH_REALVIEW_PBA8) + + +static inline unsigned long realview_get_a8_cp15_ccnt(void) +{ + unsigned long cycles; + /* Read CP15 CCNT register. */ + asm volatile ("mrc p15, 0, %0, c9, c13, 0" : "=r" (cycles)); + return cycles; +} + +#define get_cycles realview_get_a8_cp15_ccnt + +#endif diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 37357a5..9f5e143 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2166,3 +2166,11 @@ source "crypto/Kconfig" source "arch/x86/kvm/Kconfig" source "lib/Kconfig" + +config ARCH_HAS_FEATHER_TRACE + def_bool y + +config ARCH_HAS_SEND_PULL_TIMERS + def_bool y + +source "litmus/Kconfig" diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h index 1cd6d26..3b0d7ef 100644 --- a/arch/x86/include/asm/entry_arch.h +++ b/arch/x86/include/asm/entry_arch.h @@ -13,6 +13,7 @@ BUILD_INTERRUPT(reschedule_interrupt,RESCHEDULE_VECTOR) BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR) BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR) +BUILD_INTERRUPT(pull_timers_interrupt,PULL_TIMERS_VECTOR) BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR) BUILD_INTERRUPT(reboot_interrupt,REBOOT_VECTOR) diff --git a/arch/x86/include/asm/feather_trace.h b/arch/x86/include/asm/feather_trace.h new file mode 100644 index 0000000..4fd3163 --- /dev/null +++ b/arch/x86/include/asm/feather_trace.h @@ -0,0 +1,17 @@ +#ifndef _ARCH_FEATHER_TRACE_H +#define _ARCH_FEATHER_TRACE_H + +#include + +static inline unsigned long long ft_timestamp(void) +{ + return __native_read_tsc(); +} + +#ifdef CONFIG_X86_32 +#include "feather_trace_32.h" +#else +#include "feather_trace_64.h" +#endif + +#endif diff --git a/arch/x86/include/asm/feather_trace_32.h b/arch/x86/include/asm/feather_trace_32.h new file mode 100644 index 0000000..70202f9 --- /dev/null +++ b/arch/x86/include/asm/feather_trace_32.h @@ -0,0 +1,79 @@ +/* Do not directly include this file. Include feather_trace.h instead */ + +#define feather_callback __attribute__((regparm(0))) + +/* + * make the compiler reload any register that is not saved in + * a cdecl function call + */ +#define CLOBBER_LIST "memory", "cc", "eax", "ecx", "edx" + +#define ft_event(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " call " #callback " \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : : CLOBBER_LIST) + +#define ft_event0(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " subl $4, %%esp \n\t" \ + " movl $" #id ", (%%esp) \n\t" \ + " call " #callback " \n\t" \ + " addl $4, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : : CLOBBER_LIST) + +#define ft_event1(id, callback, param) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " subl $8, %%esp \n\t" \ + " movl %0, 4(%%esp) \n\t" \ + " movl $" #id ", (%%esp) \n\t" \ + " call " #callback " \n\t" \ + " addl $8, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : "r" (param) : CLOBBER_LIST) + +#define ft_event2(id, callback, param, param2) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " subl $12, %%esp \n\t" \ + " movl %1, 8(%%esp) \n\t" \ + " movl %0, 4(%%esp) \n\t" \ + " movl $" #id ", (%%esp) \n\t" \ + " call " #callback " \n\t" \ + " addl $12, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : "r" (param), "r" (param2) : CLOBBER_LIST) + + +#define ft_event3(id, callback, p, p2, p3) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " subl $16, %%esp \n\t" \ + " movl %2, 12(%%esp) \n\t" \ + " movl %1, 8(%%esp) \n\t" \ + " movl %0, 4(%%esp) \n\t" \ + " movl $" #id ", (%%esp) \n\t" \ + " call " #callback " \n\t" \ + " addl $16, %%esp \n\t" \ + ".section __event_table, \"aw\" \n\t" \ + ".long " #id ", 0, 1b, 2f \n\t" \ + ".previous \n\t" \ + "2: \n\t" \ + : : "r" (p), "r" (p2), "r" (p3) : CLOBBER_LIST) + diff --git a/arch/x86/include/asm/feather_trace_64.h b/arch/x86/include/asm/feather_trace_64.h new file mode 100644 index 0000000..54ac2ae --- /dev/null +++ b/arch/x86/include/asm/feather_trace_64.h @@ -0,0 +1,67 @@ +/* Do not directly include this file. Include feather_trace.h instead */ + +/* regparm is the default on x86_64 */ +#define feather_callback + +# define _EVENT_TABLE(id,from,to) \ + ".section __event_table, \"aw\"\n\t" \ + ".balign 8\n\t" \ + ".quad " #id ", 0, " #from ", " #to " \n\t" \ + ".previous \n\t" + +/* + * x86_64 callee only owns rbp, rbx, r12 -> r15 + * the called can freely modify the others + */ +#define CLOBBER_LIST "memory", "cc", "rdi", "rsi", "rdx", "rcx", \ + "r8", "r9", "r10", "r11", "rax" + +#define ft_event(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " call " #callback " \n\t" \ + _EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : : : CLOBBER_LIST) + +#define ft_event0(id, callback) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movq $" #id ", %%rdi \n\t" \ + " call " #callback " \n\t" \ + _EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : : : CLOBBER_LIST) + +#define ft_event1(id, callback, param) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movq %0, %%rsi \n\t" \ + " movq $" #id ", %%rdi \n\t" \ + " call " #callback " \n\t" \ + _EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : : "r" (param) : CLOBBER_LIST) + +#define ft_event2(id, callback, param, param2) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movq %1, %%rdx \n\t" \ + " movq %0, %%rsi \n\t" \ + " movq $" #id ", %%rdi \n\t" \ + " call " #callback " \n\t" \ + _EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : : "r" (param), "r" (param2) : CLOBBER_LIST) + +#define ft_event3(id, callback, p, p2, p3) \ + __asm__ __volatile__( \ + "1: jmp 2f \n\t" \ + " movq %2, %%rcx \n\t" \ + " movq %1, %%rdx \n\t" \ + " movq %0, %%rsi \n\t" \ + " movq $" #id ", %%rdi \n\t" \ + " call " #callback " \n\t" \ + _EVENT_TABLE(id,1b,2f) \ + "2: \n\t" \ + : : "r" (p), "r" (p2), "r" (p3) : CLOBBER_LIST) diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index bb9efe8..c490d89 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -77,6 +77,8 @@ extern void threshold_interrupt(void); extern void call_function_interrupt(void); extern void call_function_single_interrupt(void); +extern void pull_timers_interrupt(void); + /* IOAPIC */ #define IO_APIC_IRQ(x) (((x) >= NR_IRQS_LEGACY) || ((1<<(x)) & io_apic_irqs)) extern unsigned long io_apic_irqs; @@ -155,6 +157,7 @@ extern asmlinkage void smp_irq_move_cleanup_interrupt(void); extern void smp_reschedule_interrupt(struct pt_regs *); extern void smp_call_function_interrupt(struct pt_regs *); extern void smp_call_function_single_interrupt(struct pt_regs *); +extern void smp_pull_timers_interrupt(struct pt_regs *); #ifdef CONFIG_X86_32 extern void smp_invalidate_interrupt(struct pt_regs *); #else diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h index 6e976ee..99a44cf 100644 --- a/arch/x86/include/asm/irq_vectors.h +++ b/arch/x86/include/asm/irq_vectors.h @@ -135,6 +135,13 @@ #define INVALIDATE_TLB_VECTOR_START \ (INVALIDATE_TLB_VECTOR_END-NUM_INVALIDATE_TLB_VECTORS+1) +/* + * LITMUS^RT pull timers IRQ vector + * Make sure it's below the above max 32 vectors. + */ +#define PULL_TIMERS_VECTOR 0xce + + #define NR_VECTORS 256 #define FPU_IRQ 13 diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 2193715..b844edc 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -166,6 +166,10 @@ extern void print_cpu_info(struct cpuinfo_x86 *); extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c); extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c); extern unsigned short num_cache_leaves; +#ifdef CONFIG_SYSFS +extern int get_shared_cpu_map(cpumask_var_t mask, + unsigned int cpu, int index); +#endif extern void detect_extended_topology(struct cpuinfo_x86 *c); extern void detect_ht(struct cpuinfo_x86 *c); diff --git a/arch/x86/include/asm/unistd_32.h b/arch/x86/include/asm/unistd_32.h index 593485b..2f6e127 100644 --- a/arch/x86/include/asm/unistd_32.h +++ b/arch/x86/include/asm/unistd_32.h @@ -353,9 +353,13 @@ #define __NR_sendmmsg 345 #define __NR_setns 346 +#define __NR_LITMUS 347 + +#include "litmus/unistd_32.h" + #ifdef __KERNEL__ -#define NR_syscalls 347 +#define NR_syscalls 347 + NR_litmus_syscalls #define __ARCH_WANT_IPC_PARSE_VERSION #define __ARCH_WANT_OLD_READDIR diff --git a/arch/x86/include/asm/unistd_64.h b/arch/x86/include/asm/unistd_64.h index 705bf13..e347f07 100644 --- a/arch/x86/include/asm/unistd_64.h +++ b/arch/x86/include/asm/unistd_64.h @@ -682,6 +682,10 @@ __SYSCALL(__NR_sendmmsg, sys_sendmmsg) #define __NR_setns 308 __SYSCALL(__NR_setns, sys_setns) +#define __NR_LITMUS 309 + +#include "litmus/unistd_64.h" + #ifndef __NO_STUBS #define __ARCH_WANT_OLD_READDIR #define __ARCH_WANT_OLD_STAT diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 90b06d4..d727f8f 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -116,6 +116,8 @@ obj-$(CONFIG_X86_CHECK_BIOS_CORRUPTION) += check.o obj-$(CONFIG_SWIOTLB) += pci-swiotlb.o obj-$(CONFIG_OF) += devicetree.o +obj-$(CONFIG_FEATHER_TRACE) += ft_event.o + ### # 64 bit specific files ifeq ($(CONFIG_X86_64),y) diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c index c105c53..0bf1264 100644 --- a/arch/x86/kernel/cpu/intel_cacheinfo.c +++ b/arch/x86/kernel/cpu/intel_cacheinfo.c @@ -747,6 +747,23 @@ unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c) static DEFINE_PER_CPU(struct _cpuid4_info *, ici_cpuid4_info); #define CPUID4_INFO_IDX(x, y) (&((per_cpu(ici_cpuid4_info, x))[y])) +/* returns CPUs that share the index cache with cpu */ +int get_shared_cpu_map(cpumask_var_t mask, unsigned int cpu, int index) +{ + int ret = 0; + struct _cpuid4_info *this_leaf; + + if (index >= num_cache_leaves) { + index = num_cache_leaves - 1; + ret = index; + } + + this_leaf = CPUID4_INFO_IDX(cpu,index); + cpumask_copy(mask, to_cpumask(this_leaf->shared_cpu_map)); + + return ret; +} + #ifdef CONFIG_SMP static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) { diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 8a445a0..47a4bcd 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -1003,6 +1003,8 @@ apicinterrupt CALL_FUNCTION_VECTOR \ call_function_interrupt smp_call_function_interrupt apicinterrupt RESCHEDULE_VECTOR \ reschedule_interrupt smp_reschedule_interrupt +apicinterrupt PULL_TIMERS_VECTOR \ + pull_timers_interrupt smp_pull_timers_interrupt #endif apicinterrupt ERROR_APIC_VECTOR \ diff --git a/arch/x86/kernel/ft_event.c b/arch/x86/kernel/ft_event.c new file mode 100644 index 0000000..37cc332 --- /dev/null +++ b/arch/x86/kernel/ft_event.c @@ -0,0 +1,118 @@ +#include + +#include + +/* the feather trace management functions assume + * exclusive access to the event table + */ + +#ifndef CONFIG_DEBUG_RODATA + +#define BYTE_JUMP 0xeb +#define BYTE_JUMP_LEN 0x02 + +/* for each event, there is an entry in the event table */ +struct trace_event { + long id; + long count; + long start_addr; + long end_addr; +}; + +extern struct trace_event __start___event_table[]; +extern struct trace_event __stop___event_table[]; + +/* Workaround: if no events are defined, then the event_table section does not + * exist and the above references cause linker errors. This could probably be + * fixed by adjusting the linker script, but it is easier to maintain for us if + * we simply create a dummy symbol in the event table section. + */ +int __event_table_dummy[0] __attribute__ ((section("__event_table"))); + +int ft_enable_event(unsigned long id) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->id == id && ++te->count == 1) { + instr = (unsigned char*) te->start_addr; + /* make sure we don't clobber something wrong */ + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + 1); + *delta = 0; + } + } + if (te->id == id) + count++; + te++; + } + + printk(KERN_DEBUG "ft_enable_event: enabled %d events\n", count); + return count; +} + +int ft_disable_event(unsigned long id) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->id == id && --te->count == 0) { + instr = (unsigned char*) te->start_addr; + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + 1); + *delta = te->end_addr - te->start_addr - + BYTE_JUMP_LEN; + } + } + if (te->id == id) + count++; + te++; + } + + printk(KERN_DEBUG "ft_disable_event: disabled %d events\n", count); + return count; +} + +int ft_disable_all_events(void) +{ + struct trace_event* te = __start___event_table; + int count = 0; + char* delta; + unsigned char* instr; + + while (te < __stop___event_table) { + if (te->count) { + instr = (unsigned char*) te->start_addr; + if (*instr == BYTE_JUMP) { + delta = (((unsigned char*) te->start_addr) + + 1); + *delta = te->end_addr - te->start_addr - + BYTE_JUMP_LEN; + te->count = 0; + count++; + } + } + te++; + } + return count; +} + +int ft_is_event_enabled(unsigned long id) +{ + struct trace_event* te = __start___event_table; + + while (te < __stop___event_table) { + if (te->id == id) + return te->count; + te++; + } + return 0; +} + +#endif diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 6c0802e..680a5cb 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -10,6 +10,10 @@ #include #include +#ifdef CONFIG_LITMUS_NVIDIA +#include +#endif + #include #include #include diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c index f470e4e..48acf71 100644 --- a/arch/x86/kernel/irqinit.c +++ b/arch/x86/kernel/irqinit.c @@ -252,6 +252,9 @@ static void __init smp_intr_init(void) alloc_intr_gate(CALL_FUNCTION_SINGLE_VECTOR, call_function_single_interrupt); + /* IPI for hrtimer pulling on remote cpus */ + alloc_intr_gate(PULL_TIMERS_VECTOR, pull_timers_interrupt); + /* Low priority IPI to cleanup after moving an irq */ set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt); set_bit(IRQ_MOVE_CLEANUP_VECTOR, used_vectors); diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index 013e7eb..ed4c4f5 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -23,6 +23,10 @@ #include #include +#include +#include +#include + #include #include #include @@ -118,6 +122,7 @@ static void native_smp_send_reschedule(int cpu) WARN_ON(1); return; } + TS_SEND_RESCHED_START(cpu); apic->send_IPI_mask(cpumask_of(cpu), RESCHEDULE_VECTOR); } @@ -147,6 +152,16 @@ void native_send_call_func_ipi(const struct cpumask *mask) free_cpumask_var(allbutself); } +/* trigger timers on remote cpu */ +void smp_send_pull_timers(int cpu) +{ + if (unlikely(cpu_is_offline(cpu))) { + WARN_ON(1); + return; + } + apic->send_IPI_mask(cpumask_of(cpu), PULL_TIMERS_VECTOR); +} + /* * this function calls the 'stop' function on all other CPUs in the system. */ @@ -199,8 +214,15 @@ static void native_stop_other_cpus(int wait) void smp_reschedule_interrupt(struct pt_regs *regs) { ack_APIC_irq(); + /* LITMUS^RT: this IPI might need to trigger the sched state machine. */ + sched_state_ipi(); inc_irq_stat(irq_resched_count); + /* + * LITMUS^RT: starting from 3.0 schedule_ipi() actually does something. + * This may increase IPI latencies compared with previous versions. + */ scheduler_ipi(); + TS_SEND_RESCHED_END; /* * KVM uses this interrupt to force a cpu out of guest mode */ @@ -224,6 +246,15 @@ void smp_call_function_single_interrupt(struct pt_regs *regs) irq_exit(); } +extern void hrtimer_pull(void); + +void smp_pull_timers_interrupt(struct pt_regs *regs) +{ + ack_APIC_irq(); + TRACE("pull timer interrupt\n"); + hrtimer_pull(); +} + struct smp_ops smp_ops = { .smp_prepare_boot_cpu = native_smp_prepare_boot_cpu, .smp_prepare_cpus = native_smp_prepare_cpus, diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S index fbb0a04..0cb4373 100644 --- a/arch/x86/kernel/syscall_table_32.S +++ b/arch/x86/kernel/syscall_table_32.S @@ -346,3 +346,16 @@ ENTRY(sys_call_table) .long sys_syncfs .long sys_sendmmsg /* 345 */ .long sys_setns + .long sys_set_rt_task_param /* LITMUS^RT 347 */ + .long sys_get_rt_task_param + .long sys_complete_job + .long sys_od_open + .long sys_od_close + .long sys_litmus_lock /* +5 */ + .long sys_litmus_unlock + .long sys_query_job_no + .long sys_wait_for_job_release + .long sys_wait_for_ts_release + .long sys_release_ts /* +10 */ + .long sys_null_call + .long sys_register_nv_device diff --git a/fs/exec.c b/fs/exec.c index 6075a1e..9984562 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -19,7 +19,7 @@ * current->executable is only used by the procfs. This allows a dispatch * table to check for several different types of binary formats. We keep * trying until we recognize the file or we run out of supported binary - * formats. + * formats. */ #include @@ -56,6 +56,8 @@ #include #include +#include + #include #include #include @@ -85,7 +87,7 @@ int __register_binfmt(struct linux_binfmt * fmt, int insert) insert ? list_add(&fmt->lh, &formats) : list_add_tail(&fmt->lh, &formats); write_unlock(&binfmt_lock); - return 0; + return 0; } EXPORT_SYMBOL(__register_binfmt); @@ -1160,7 +1162,7 @@ void setup_new_exec(struct linux_binprm * bprm) group */ current->self_exec_id++; - + flush_signal_handlers(current, 0); flush_old_files(current->files); } @@ -1250,8 +1252,8 @@ int check_unsafe_exec(struct linux_binprm *bprm) return res; } -/* - * Fill the binprm structure from the inode. +/* + * Fill the binprm structure from the inode. * Check permissions, then read the first 128 (BINPRM_BUF_SIZE) bytes * * This may be called multiple times for binary chains (scripts for example). @@ -1459,6 +1461,7 @@ static int do_execve_common(const char *filename, goto out_unmark; sched_exec(); + litmus_exec(); bprm->file = file; bprm->filename = filename; diff --git a/fs/inode.c b/fs/inode.c index 43566d1..dbf0e76 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -308,6 +308,8 @@ void inode_init_once(struct inode *inode) #ifdef CONFIG_FSNOTIFY INIT_HLIST_HEAD(&inode->i_fsnotify_marks); #endif + INIT_LIST_HEAD(&inode->i_obj_list); + mutex_init(&inode->i_obj_mutex); } EXPORT_SYMBOL(inode_init_once); diff --git a/include/linux/completion.h b/include/linux/completion.h index 51494e6..cff405c 100644 --- a/include/linux/completion.h +++ b/include/linux/completion.h @@ -76,6 +76,7 @@ static inline void init_completion(struct completion *x) init_waitqueue_head(&x->wait); } +extern void __wait_for_completion_locked(struct completion *); extern void wait_for_completion(struct completion *); extern int wait_for_completion_interruptible(struct completion *x); extern int wait_for_completion_killable(struct completion *x); @@ -90,6 +91,7 @@ extern bool completion_done(struct completion *x); extern void complete(struct completion *); extern void complete_all(struct completion *); +extern void complete_n(struct completion *, int n); /** * INIT_COMPLETION - reinitialize a completion structure diff --git a/include/linux/fs.h b/include/linux/fs.h index b5b9792..8d5834b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -17,8 +17,8 @@ * nr_file rlimit, so it's safe to set up a ridiculously high absolute * upper limit on files-per-process. * - * Some programs (notably those using select()) may have to be - * recompiled to take full advantage of the new limits.. + * Some programs (notably those using select()) may have to be + * recompiled to take full advantage of the new limits.. */ /* Fixed constants first: */ @@ -172,7 +172,7 @@ struct inodes_stat_t { #define SEL_EX 4 /* public flags for file_system_type */ -#define FS_REQUIRES_DEV 1 +#define FS_REQUIRES_DEV 1 #define FS_BINARY_MOUNTDATA 2 #define FS_HAS_SUBTYPE 4 #define FS_REVAL_DOT 16384 /* Check the paths ".", ".." for staleness */ @@ -480,7 +480,7 @@ struct iattr { */ #include -/** +/** * enum positive_aop_returns - aop return codes with specific semantics * * @AOP_WRITEPAGE_ACTIVATE: Informs the caller that page writeback has @@ -490,7 +490,7 @@ struct iattr { * be a candidate for writeback again in the near * future. Other callers must be careful to unlock * the page if they get this return. Returned by - * writepage(); + * writepage(); * * @AOP_TRUNCATED_PAGE: The AOP method that was handed a locked page has * unlocked it and the page might have been truncated. @@ -734,6 +734,7 @@ static inline int mapping_writably_mapped(struct address_space *mapping) struct posix_acl; #define ACL_NOT_CACHED ((void *)(-1)) +struct inode_obj_id_table; struct inode { /* RCU path lookup touches following: */ @@ -807,6 +808,8 @@ struct inode { struct posix_acl *i_acl; struct posix_acl *i_default_acl; #endif + struct list_head i_obj_list; + struct mutex i_obj_mutex; void *i_private; /* fs or device private pointer */ }; @@ -1032,10 +1035,10 @@ static inline int file_check_writeable(struct file *filp) #define MAX_NON_LFS ((1UL<<31) - 1) -/* Page cache limit. The filesystems should put that into their s_maxbytes - limits, otherwise bad things can happen in VM. */ +/* Page cache limit. The filesystems should put that into their s_maxbytes + limits, otherwise bad things can happen in VM. */ #if BITS_PER_LONG==32 -#define MAX_LFS_FILESIZE (((u64)PAGE_CACHE_SIZE << (BITS_PER_LONG-1))-1) +#define MAX_LFS_FILESIZE (((u64)PAGE_CACHE_SIZE << (BITS_PER_LONG-1))-1) #elif BITS_PER_LONG==64 #define MAX_LFS_FILESIZE 0x7fffffffffffffffUL #endif @@ -2234,7 +2237,7 @@ extern void free_write_pipe(struct file *); extern int kernel_read(struct file *, loff_t, char *, unsigned long); extern struct file * open_exec(const char *); - + /* fs/dcache.c -- generic fs support functions */ extern int is_subdir(struct dentry *, struct dentry *); extern int path_is_under(struct path *, struct path *); diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index ba36217..e6dd5a4 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -6,6 +6,8 @@ #include #include +#include + /* * We put the hardirq and softirq counter into the preemption * counter. The bitmask has the following meaning: @@ -186,6 +188,7 @@ extern void rcu_nmi_exit(void); account_system_vtime(current); \ add_preempt_count(HARDIRQ_OFFSET); \ trace_hardirq_enter(); \ + ft_irq_fired(); \ } while (0) /* @@ -216,6 +219,7 @@ extern void irq_exit(void); lockdep_off(); \ rcu_nmi_enter(); \ trace_hardirq_enter(); \ + ft_irq_fired(); \ } while (0) #define nmi_exit() \ diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index fd0dc30..d91bba5 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -174,6 +174,7 @@ enum hrtimer_base_type { * @nr_hangs: Total number of hrtimer interrupt hangs * @max_hang_time: Maximum time spent in hrtimer_interrupt * @clock_base: array of clock bases for this cpu + * @to_pull: LITMUS^RT list of timers to be pulled on this cpu */ struct hrtimer_cpu_base { raw_spinlock_t lock; @@ -188,8 +189,32 @@ struct hrtimer_cpu_base { ktime_t max_hang_time; #endif struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; + struct list_head to_pull; }; +#ifdef CONFIG_ARCH_HAS_SEND_PULL_TIMERS + +#define HRTIMER_START_ON_INACTIVE 0 +#define HRTIMER_START_ON_QUEUED 1 + +/* + * struct hrtimer_start_on_info - save timer info on remote cpu + * @list: list of hrtimer_start_on_info on remote cpu (to_pull) + * @timer: timer to be triggered on remote cpu + * @time: time event + * @mode: timer mode + * @state: activity flag + */ +struct hrtimer_start_on_info { + struct list_head list; + struct hrtimer *timer; + ktime_t time; + enum hrtimer_mode mode; + atomic_t state; +}; + +#endif + static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time) { timer->node.expires = time; @@ -355,6 +380,13 @@ __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns, const enum hrtimer_mode mode, int wakeup); +#ifdef CONFIG_ARCH_HAS_SEND_PULL_TIMERS +extern void hrtimer_start_on_info_init(struct hrtimer_start_on_info *info); +extern int hrtimer_start_on(int cpu, struct hrtimer_start_on_info *info, + struct hrtimer *timer, ktime_t time, + const enum hrtimer_mode mode); +#endif + extern int hrtimer_cancel(struct hrtimer *timer); extern int hrtimer_try_to_cancel(struct hrtimer *timer); diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index f6efed0..8fb3dad 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -445,6 +445,7 @@ static inline void __raise_softirq_irqoff(unsigned int nr) extern void raise_softirq_irqoff(unsigned int nr); extern void raise_softirq(unsigned int nr); +extern void wakeup_softirqd(void); /* This is the worklist that queues up per-cpu softirq work. * @@ -500,6 +501,10 @@ struct tasklet_struct atomic_t count; void (*func)(unsigned long); unsigned long data; + +#if defined(CONFIG_LITMUS_SOFTIRQD) || defined(CONFIG_LITMUS_PAI_SOFTIRQD) + struct task_struct *owner; +#endif }; #define DECLARE_TASKLET(name, func, data) \ @@ -523,7 +528,7 @@ static inline int tasklet_trylock(struct tasklet_struct *t) static inline void tasklet_unlock(struct tasklet_struct *t) { - smp_mb__before_clear_bit(); + smp_mb__before_clear_bit(); clear_bit(TASKLET_STATE_RUN, &(t)->state); } @@ -537,6 +542,7 @@ static inline void tasklet_unlock_wait(struct tasklet_struct *t) #define tasklet_unlock(t) do { } while (0) #endif +extern void ___tasklet_schedule(struct tasklet_struct *t); extern void __tasklet_schedule(struct tasklet_struct *t); static inline void tasklet_schedule(struct tasklet_struct *t) @@ -545,6 +551,7 @@ static inline void tasklet_schedule(struct tasklet_struct *t) __tasklet_schedule(t); } +extern void ___tasklet_hi_schedule(struct tasklet_struct *t); extern void __tasklet_hi_schedule(struct tasklet_struct *t); static inline void tasklet_hi_schedule(struct tasklet_struct *t) @@ -553,6 +560,7 @@ static inline void tasklet_hi_schedule(struct tasklet_struct *t) __tasklet_hi_schedule(t); } +extern void ___tasklet_hi_schedule_first(struct tasklet_struct *t); extern void __tasklet_hi_schedule_first(struct tasklet_struct *t); /* @@ -651,7 +659,7 @@ void tasklet_hrtimer_cancel(struct tasklet_hrtimer *ttimer) * if more than one irq occurred. */ -#if defined(CONFIG_GENERIC_HARDIRQS) && !defined(CONFIG_GENERIC_IRQ_PROBE) +#if defined(CONFIG_GENERIC_HARDIRQS) && !defined(CONFIG_GENERIC_IRQ_PROBE) static inline unsigned long probe_irq_on(void) { return 0; diff --git a/include/linux/mutex.h b/include/linux/mutex.h index a940fe4..cb47deb 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -126,6 +126,15 @@ static inline int mutex_is_locked(struct mutex *lock) return atomic_read(&lock->count) != 1; } +/* return non-zero to abort. only pre-side-effects may abort */ +typedef int (*side_effect_t)(unsigned long); +extern void mutex_lock_sfx(struct mutex *lock, + side_effect_t pre, unsigned long pre_arg, + side_effect_t post, unsigned long post_arg); +extern void mutex_unlock_sfx(struct mutex *lock, + side_effect_t pre, unsigned long pre_arg, + side_effect_t post, unsigned long post_arg); + /* * See kernel/mutex.c for detailed documentation of these APIs. * Also see Documentation/mutex-design.txt. @@ -153,6 +162,7 @@ extern void mutex_lock(struct mutex *lock); extern int __must_check mutex_lock_interruptible(struct mutex *lock); extern int __must_check mutex_lock_killable(struct mutex *lock); + # define mutex_lock_nested(lock, subclass) mutex_lock(lock) # define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock) # define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock) diff --git a/include/linux/sched.h b/include/linux/sched.h index 14a6c7b..9c990d1 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -39,6 +39,7 @@ #define SCHED_BATCH 3 /* SCHED_ISO: reserved but not implemented yet */ #define SCHED_IDLE 5 +#define SCHED_LITMUS 6 /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */ #define SCHED_RESET_ON_FORK 0x40000000 @@ -93,6 +94,9 @@ struct sched_param { #include +#include +#include + struct exec_domain; struct futex_pi_state; struct robust_list_head; @@ -1209,6 +1213,7 @@ struct sched_rt_entity { }; struct rcu_node; +struct od_table_entry; enum perf_event_task_context { perf_invalid_context = -1, @@ -1313,9 +1318,9 @@ struct task_struct { unsigned long stack_canary; #endif - /* + /* * pointers to (original) parent process, youngest child, younger sibling, - * older sibling, respectively. (p->father can be replaced with + * older sibling, respectively. (p->father can be replaced with * p->real_parent->pid) */ struct task_struct *real_parent; /* real parent process */ @@ -1526,6 +1531,13 @@ struct task_struct { int make_it_fail; #endif struct prop_local_single dirties; + + /* LITMUS RT parameters and state */ + struct rt_param rt_param; + + /* references to PI semaphores, etc. */ + struct od_table_entry *od_table; + #ifdef CONFIG_LATENCYTOP int latency_record_count; struct latency_record latency_record[LT_SAVECOUNT]; @@ -2136,7 +2148,7 @@ static inline int dequeue_signal_lock(struct task_struct *tsk, sigset_t *mask, s spin_unlock_irqrestore(&tsk->sighand->siglock, flags); return ret; -} +} extern void block_all_signals(int (*notifier)(void *priv), void *priv, sigset_t *mask); @@ -2446,6 +2458,7 @@ static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) static inline void set_tsk_need_resched(struct task_struct *tsk) { set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); + sched_state_will_schedule(tsk); } static inline void clear_tsk_need_resched(struct task_struct *tsk) diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h index 39fa049..c83fc2b 100644 --- a/include/linux/semaphore.h +++ b/include/linux/semaphore.h @@ -43,4 +43,13 @@ extern int __must_check down_trylock(struct semaphore *sem); extern int __must_check down_timeout(struct semaphore *sem, long jiffies); extern void up(struct semaphore *sem); +extern void __down(struct semaphore *sem); +extern void __up(struct semaphore *sem); + +struct semaphore_waiter { + struct list_head list; + struct task_struct *task; + int up; +}; + #endif /* __LINUX_SEMAPHORE_H */ diff --git a/include/linux/smp.h b/include/linux/smp.h index 8cc38d3..53b1bee 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -82,6 +82,11 @@ int smp_call_function_any(const struct cpumask *mask, smp_call_func_t func, void *info, int wait); /* + * sends a 'pull timer' event to a remote CPU + */ +extern void smp_send_pull_timers(int cpu); + +/* * Generic and arch helpers */ #ifdef CONFIG_USE_GENERIC_SMP_HELPERS diff --git a/include/linux/tick.h b/include/linux/tick.h index b232ccc..1e29bd5 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -74,6 +74,11 @@ extern int tick_is_oneshot_available(void); extern struct tick_device *tick_get_device(int cpu); # ifdef CONFIG_HIGH_RES_TIMERS +/* LITMUS^RT tick alignment */ +#define LINUX_DEFAULT_TICKS 0 +#define LITMUS_ALIGNED_TICKS 1 +#define LITMUS_STAGGERED_TICKS 2 + extern int tick_init_highres(void); extern int tick_program_event(ktime_t expires, int force); extern void tick_setup_sched_timer(void); diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index f584aba..1ec2ec7 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -83,6 +83,9 @@ struct work_struct { #ifdef CONFIG_LOCKDEP struct lockdep_map lockdep_map; #endif +#ifdef CONFIG_LITMUS_SOFTIRQD + struct task_struct *owner; +#endif }; #define WORK_DATA_INIT() ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU) @@ -115,11 +118,25 @@ struct execute_work { #define __WORK_INIT_LOCKDEP_MAP(n, k) #endif +#ifdef CONFIG_LITMUS_SOFTIRQD +#define __WORK_INIT_OWNER() \ + .owner = NULL, + +#define PREPARE_OWNER(_work, _owner) \ + do { \ + (_work)->owner = (_owner); \ + } while(0) +#else +#define __WORK_INIT_OWNER() +#define PREPARE_OWNER(_work, _owner) +#endif + #define __WORK_INITIALIZER(n, f) { \ .data = WORK_DATA_STATIC_INIT(), \ .entry = { &(n).entry, &(n).entry }, \ .func = (f), \ __WORK_INIT_LOCKDEP_MAP(#n, &(n)) \ + __WORK_INIT_OWNER() \ } #define __DELAYED_WORK_INITIALIZER(n, f) { \ @@ -357,6 +374,7 @@ extern int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, extern void flush_workqueue(struct workqueue_struct *wq); extern void flush_scheduled_work(void); +extern int __schedule_work(struct work_struct *work); extern int schedule_work(struct work_struct *work); extern int schedule_work_on(int cpu, struct work_struct *work); extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay); diff --git a/include/litmus/affinity.h b/include/litmus/affinity.h new file mode 100644 index 0000000..ca2e442 --- /dev/null +++ b/include/litmus/affinity.h @@ -0,0 +1,80 @@ +#ifndef __LITMUS_AFFINITY_H +#define __LITMUS_AFFINITY_H + +#include + +/* + L1 (instr) = depth 0 + L1 (data) = depth 1 + L2 = depth 2 + L3 = depth 3 + */ +#define NUM_CACHE_LEVELS 4 + +struct neighborhood +{ + unsigned int size[NUM_CACHE_LEVELS]; + cpumask_var_t neighbors[NUM_CACHE_LEVELS]; +}; + +/* topology info is stored redundently in a big array for fast lookups */ +extern struct neighborhood neigh_info[NR_CPUS]; + +void init_topology(void); /* called by Litmus module's _init_litmus() */ + +/* Works like: +void get_nearest_available_cpu( + cpu_entry_t **nearest, + cpu_entry_t *start, + cpu_entry_t *entries, + int release_master) + +Set release_master = NO_CPU for no Release Master. + +We use a macro here to exploit the fact that C-EDF and G-EDF +have similar structures for their cpu_entry_t structs, even though +they do not share a common base-struct. The macro allows us to +avoid code duplication. + +TODO: Factor out the job-to-processor linking from C/G-EDF into +a reusable "processor mapping". (See B.B.'s RTSS'09 paper & +dissertation.) + */ +#define get_nearest_available_cpu(nearest, start, entries, release_master) \ +{ \ + (nearest) = NULL; \ + if (!(start)->linked) { \ + (nearest) = (start); \ + } else { \ + int __level; \ + int __cpu; \ + int __release_master = ((release_master) == NO_CPU) ? -1 : (release_master); \ + struct neighborhood *__neighbors = &neigh_info[(start)->cpu]; \ + \ + for (__level = 0; (__level < NUM_CACHE_LEVELS) && !(nearest); ++__level) { \ + if (__neighbors->size[__level] > 1) { \ + for_each_cpu(__cpu, __neighbors->neighbors[__level]) { \ + if (__cpu != __release_master) { \ + cpu_entry_t *__entry = &per_cpu((entries), __cpu); \ + if (!__entry->linked) { \ + (nearest) = __entry; \ + break; \ + } \ + } \ + } \ + } else if (__neighbors->size[__level] == 0) { \ + break; \ + } \ + } \ + } \ + \ + if ((nearest)) { \ + TRACE("P%d is closest available CPU to P%d\n", \ + (nearest)->cpu, (start)->cpu); \ + } else { \ + TRACE("Could not find an available CPU close to P%d\n", \ + (start)->cpu); \ + } \ +} + +#endif diff --git a/include/litmus/bheap.h b/include/litmus/bheap.h new file mode 100644 index 0000000..cf4864a --- /dev/null +++ b/include/litmus/bheap.h @@ -0,0 +1,77 @@ +/* bheaps.h -- Binomial Heaps + * + * (c) 2008, 2009 Bjoern Brandenburg + */ + +#ifndef BHEAP_H +#define BHEAP_H + +#define NOT_IN_HEAP UINT_MAX + +struct bheap_node { + struct bheap_node* parent; + struct bheap_node* next; + struct bheap_node* child; + + unsigned int degree; + void* value; + struct bheap_node** ref; +}; + +struct bheap { + struct bheap_node* head; + /* We cache the minimum of the heap. + * This speeds up repeated peek operations. + */ + struct bheap_node* min; +}; + +typedef int (*bheap_prio_t)(struct bheap_node* a, struct bheap_node* b); + +void bheap_init(struct bheap* heap); +void bheap_node_init(struct bheap_node** ref_to_bheap_node_ptr, void* value); + +static inline int bheap_node_in_heap(struct bheap_node* h) +{ + return h->degree != NOT_IN_HEAP; +} + +static inline int bheap_empty(struct bheap* heap) +{ + return heap->head == NULL && heap->min == NULL; +} + +/* insert (and reinitialize) a node into the heap */ +void bheap_insert(bheap_prio_t higher_prio, + struct bheap* heap, + struct bheap_node* node); + +/* merge addition into target */ +void bheap_union(bheap_prio_t higher_prio, + struct bheap* target, + struct bheap* addition); + +struct bheap_node* bheap_peek(bheap_prio_t higher_prio, + struct bheap* heap); + +struct bheap_node* bheap_take(bheap_prio_t higher_prio, + struct bheap* heap); + +void bheap_uncache_min(bheap_prio_t higher_prio, struct bheap* heap); +int bheap_decrease(bheap_prio_t higher_prio, struct bheap_node* node); + +void bheap_delete(bheap_prio_t higher_prio, + struct bheap* heap, + struct bheap_node* node); + +/* allocate from memcache */ +struct bheap_node* bheap_node_alloc(int gfp_flags); +void bheap_node_free(struct bheap_node* hn); + +/* allocate a heap node for value and insert into the heap */ +int bheap_add(bheap_prio_t higher_prio, struct bheap* heap, + void* value, int gfp_flags); + +void* bheap_take_del(bheap_prio_t higher_prio, + struct bheap* heap); +#endif diff --git a/include/litmus/binheap.h b/include/litmus/binheap.h new file mode 100644 index 0000000..9e966e3 --- /dev/null +++ b/include/litmus/binheap.h @@ -0,0 +1,207 @@ +#ifndef LITMUS_BINARY_HEAP_H +#define LITMUS_BINARY_HEAP_H + +#include + +/** + * Simple binary heap with add, arbitrary delete, delete_root, and top + * operations. + * + * Style meant to conform with list.h. + * + * Motivation: Linux's prio_heap.h is of fixed size. Litmus's binomial + * heap may be overkill (and perhaps not general enough) for some applications. + * + * Note: In order to make node swaps fast, a node inserted with a data pointer + * may not always hold said data pointer. This is similar to the binomial heap + * implementation. This does make node deletion tricky since we have to + * (1) locate the node that holds the data pointer to delete, and (2) the + * node that was originally inserted with said data pointer. These have to be + * coalesced into a single node before removal (see usage of + * __binheap_safe_swap()). We have to track node references to accomplish this. + */ + +struct binheap_node { + void *data; + struct binheap_node *parent; + struct binheap_node *left; + struct binheap_node *right; + + /* pointer to binheap_node that holds *data for which this binheap_node + * was originally inserted. (*data "owns" this node) + */ + struct binheap_node *ref; + struct binheap_node **ref_ptr; +}; + +/** + * Signature of compator function. Assumed 'less-than' (min-heap). + * Pass in 'greater-than' for max-heap. + * + * TODO: Consider macro-based implementation that allows comparator to be + * inlined (similar to Linux red/black tree) for greater efficiency. + */ +typedef int (*binheap_order_t)(struct binheap_node *a, + struct binheap_node *b); + + +struct binheap_handle { + struct binheap_node *root; + + /* pointer to node to take next inserted child */ + struct binheap_node *next; + + /* pointer to last node in complete binary tree */ + struct binheap_node *last; + + /* comparator function pointer */ + binheap_order_t compare; +}; + + +#define BINHEAP_POISON ((void*)(0xdeadbeef)) + + +/** + * binheap_entry - get the struct for this heap node. + * Only valid when called upon heap nodes other than the root handle. + * @ptr: the heap node. + * @type: the type of struct pointed to by binheap_node::data. + * @member: unused. + */ +#define binheap_entry(ptr, type, member) \ +((type *)((ptr)->data)) + +/** + * binheap_node_container - get the struct that contains this node. + * Only valid when called upon heap nodes other than the root handle. + * @ptr: the heap node. + * @type: the type of struct the node is embedded in. + * @member: the name of the binheap_struct within the (type) struct. + */ +#define binheap_node_container(ptr, type, member) \ +container_of((ptr), type, member) + +/** + * binheap_top_entry - get the struct for the node at the top of the heap. + * Only valid when called upon the heap handle node. + * @ptr: the special heap-handle node. + * @type: the type of the struct the head is embedded in. + * @member: the name of the binheap_struct within the (type) struct. + */ +#define binheap_top_entry(ptr, type, member) \ +binheap_entry((ptr)->root, type, member) + +/** + * binheap_delete_root - remove the root element from the heap. + * @handle: handle to the heap. + * @type: the type of the struct the head is embedded in. + * @member: the name of the binheap_struct within the (type) struct. + */ +#define binheap_delete_root(handle, type, member) \ +__binheap_delete_root((handle), &((type *)((handle)->root->data))->member) + +/** + * binheap_delete - remove an arbitrary element from the heap. + * @to_delete: pointer to node to be removed. + * @handle: handle to the heap. + */ +#define binheap_delete(to_delete, handle) \ +__binheap_delete((to_delete), (handle)) + +/** + * binheap_add - insert an element to the heap + * new_node: node to add. + * @handle: handle to the heap. + * @type: the type of the struct the head is embedded in. + * @member: the name of the binheap_struct within the (type) struct. + */ +#define binheap_add(new_node, handle, type, member) \ +__binheap_add((new_node), (handle), container_of((new_node), type, member)) + +/** + * binheap_decrease - re-eval the position of a node (based upon its + * original data pointer). + * @handle: handle to the heap. + * @orig_node: node that was associated with the data pointer + * (whose value has changed) when said pointer was + * added to the heap. + */ +#define binheap_decrease(orig_node, handle) \ +__binheap_decrease((orig_node), (handle)) + +#define BINHEAP_NODE_INIT() { NULL, BINHEAP_POISON, NULL, NULL , NULL, NULL} + +#define BINHEAP_NODE(name) \ + struct binheap_node name = BINHEAP_NODE_INIT() + + +static inline void INIT_BINHEAP_NODE(struct binheap_node *n) +{ + n->data = NULL; + n->parent = BINHEAP_POISON; + n->left = NULL; + n->right = NULL; + n->ref = NULL; + n->ref_ptr = NULL; +} + +static inline void INIT_BINHEAP_HANDLE( + struct binheap_handle *handle, + binheap_order_t compare) +{ + handle->root = NULL; + handle->next = NULL; + handle->last = NULL; + handle->compare = compare; +} + +/* Returns true (1) if binheap is empty. */ +static inline int binheap_empty(struct binheap_handle *handle) +{ + return(handle->root == NULL); +} + +/* Returns true (1) if binheap node is in a heap. */ +static inline int binheap_is_in_heap(struct binheap_node *node) +{ + return (node->parent != BINHEAP_POISON); +} + + +int binheap_is_in_this_heap(struct binheap_node *node, struct binheap_handle* heap); + + + +void __binheap_add(struct binheap_node *new_node, + struct binheap_handle *handle, + void *data); + + +/** + * Removes the root node from the heap. The node is removed after coalescing + * the binheap_node with its original data pointer at the root of the tree. + * + * The 'last' node in the tree is then swapped up to the root and bubbled + * down. + */ +void __binheap_delete_root(struct binheap_handle *handle, + struct binheap_node *container); + +/** + * Delete an arbitrary node. Bubble node to delete up to the root, + * and then delete to root. + */ +void __binheap_delete( + struct binheap_node *node_to_delete, + struct binheap_handle *handle); + +/** + * Bubble up a node whose pointer has decreased in value. + */ +void __binheap_decrease(struct binheap_node *orig_node, + struct binheap_handle *handle); + + +#endif + diff --git a/include/litmus/budget.h b/include/litmus/budget.h new file mode 100644 index 0000000..732530e --- /dev/null +++ b/include/litmus/budget.h @@ -0,0 +1,8 @@ +#ifndef _LITMUS_BUDGET_H_ +#define _LITMUS_BUDGET_H_ + +/* Update the per-processor enforcement timer (arm/reproram/cancel) for + * the next task. */ +void update_enforcement_timer(struct task_struct* t); + +#endif diff --git a/include/litmus/clustered.h b/include/litmus/clustered.h new file mode 100644 index 0000000..0c18dcb --- /dev/null +++ b/include/litmus/clustered.h @@ -0,0 +1,44 @@ +#ifndef CLUSTERED_H +#define CLUSTERED_H + +/* Which cache level should be used to group CPUs into clusters? + * GLOBAL_CLUSTER means that all CPUs form a single cluster (just like under + * global scheduling). + */ +enum cache_level { + GLOBAL_CLUSTER = 0, + L1_CLUSTER = 1, + L2_CLUSTER = 2, + L3_CLUSTER = 3 +}; + +int parse_cache_level(const char *str, enum cache_level *level); +const char* cache_level_name(enum cache_level level); + +/* expose a cache level in a /proc dir */ +struct proc_dir_entry* create_cluster_file(struct proc_dir_entry* parent, + enum cache_level* level); + + + +struct scheduling_cluster { + unsigned int id; + /* list of CPUs that are part of this cluster */ + struct list_head cpus; +}; + +struct cluster_cpu { + unsigned int id; /* which CPU is this? */ + struct list_head cluster_list; /* List of the CPUs in this cluster. */ + struct scheduling_cluster* cluster; /* The cluster that this CPU belongs to. */ +}; + +int get_cluster_size(enum cache_level level); + +int assign_cpus_to_clusters(enum cache_level level, + struct scheduling_cluster* clusters[], + unsigned int num_clusters, + struct cluster_cpu* cpus[], + unsigned int num_cpus); + +#endif diff --git a/include/litmus/debug_trace.h b/include/litmus/debug_trace.h new file mode 100644 index 0000000..48d086d --- /dev/null +++ b/include/litmus/debug_trace.h @@ -0,0 +1,37 @@ +#ifndef LITMUS_DEBUG_TRACE_H +#define LITMUS_DEBUG_TRACE_H + +#ifdef CONFIG_SCHED_DEBUG_TRACE +void sched_trace_log_message(const char* fmt, ...); +void dump_trace_buffer(int max); +#else + +#define sched_trace_log_message(fmt, ...) + +#endif + +extern atomic_t __log_seq_no; + +#ifdef CONFIG_SCHED_DEBUG_TRACE_CALLER +#define TRACE_PREFIX "%d P%d [%s@%s:%d]: " +#define TRACE_ARGS atomic_add_return(1, &__log_seq_no), \ + raw_smp_processor_id(), \ + __FUNCTION__, __FILE__, __LINE__ +#else +#define TRACE_PREFIX "%d P%d: " +#define TRACE_ARGS atomic_add_return(1, &__log_seq_no), \ + raw_smp_processor_id() +#endif + +#define TRACE(fmt, args...) \ + sched_trace_log_message(TRACE_PREFIX fmt, \ + TRACE_ARGS, ## args) + +#define TRACE_TASK(t, fmt, args...) \ + TRACE("(%s/%d:%d) " fmt, (t)->comm, (t)->pid, \ + (t)->rt_param.job_params.job_no, ##args) + +#define TRACE_CUR(fmt, args...) \ + TRACE_TASK(current, fmt, ## args) + +#endif diff --git a/include/litmus/edf_common.h b/include/litmus/edf_common.h new file mode 100644 index 0000000..63dff7e --- /dev/null +++ b/include/litmus/edf_common.h @@ -0,0 +1,37 @@ +/* + * EDF common data structures and utility functions shared by all EDF + * based scheduler plugins + */ + +/* CLEANUP: Add comments and make it less messy. + * + */ + +#ifndef __UNC_EDF_COMMON_H__ +#define __UNC_EDF_COMMON_H__ + +#include + +void edf_domain_init(rt_domain_t* rt, check_resched_needed_t resched, + release_jobs_t release); + +int edf_higher_prio(struct task_struct* first, + struct task_struct* second); + +int edf_ready_order(struct bheap_node* a, struct bheap_node* b); + +#ifdef CONFIG_LITMUS_NESTED_LOCKING +/* binheap_nodes must be embedded within 'struct litmus_lock' */ +int edf_max_heap_order(struct binheap_node *a, struct binheap_node *b); +int edf_min_heap_order(struct binheap_node *a, struct binheap_node *b); +int edf_max_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b); +int edf_min_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b); + +int __edf_higher_prio(struct task_struct* first, comparison_mode_t first_mode, + struct task_struct* second, comparison_mode_t second_mode); + +#endif + +int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t); + +#endif diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h new file mode 100644 index 0000000..1f5d3bd --- /dev/null +++ b/include/litmus/fdso.h @@ -0,0 +1,83 @@ +/* fdso.h - file descriptor attached shared objects + * + * (c) 2007 B. Brandenburg, LITMUS^RT project + */ + +#ifndef _LINUX_FDSO_H_ +#define _LINUX_FDSO_H_ + +#include +#include + +#include +#include + +#define MAX_OBJECT_DESCRIPTORS 32 + +typedef enum { + MIN_OBJ_TYPE = 0, + + FMLP_SEM = 0, + SRP_SEM = 1, + + RSM_MUTEX = 2, + IKGLP_SEM = 3, + KFMLP_SEM = 4, + + IKGLP_SIMPLE_GPU_AFF_OBS = 5, + IKGLP_GPU_AFF_OBS = 6, + KFMLP_SIMPLE_GPU_AFF_OBS = 7, + KFMLP_GPU_AFF_OBS = 8, + + MAX_OBJ_TYPE = 8 +} obj_type_t; + +struct inode_obj_id { + struct list_head list; + atomic_t count; + struct inode* inode; + + obj_type_t type; + void* obj; + unsigned int id; +}; + +struct fdso_ops; + +struct od_table_entry { + unsigned int used; + + struct inode_obj_id* obj; + const struct fdso_ops* class; +}; + +struct fdso_ops { + int (*create)(void** obj_ref, obj_type_t type, void* __user); + void (*destroy)(obj_type_t type, void*); + int (*open) (struct od_table_entry*, void* __user); + int (*close) (struct od_table_entry*); +}; + +/* translate a userspace supplied od into the raw table entry + * returns NULL if od is invalid + */ +struct od_table_entry* get_entry_for_od(int od); + +/* translate a userspace supplied od into the associated object + * returns NULL if od is invalid + */ +static inline void* od_lookup(int od, obj_type_t type) +{ + struct od_table_entry* e = get_entry_for_od(od); + return e && e->obj->type == type ? e->obj->obj : NULL; +} + +#define lookup_fmlp_sem(od)((struct pi_semaphore*) od_lookup(od, FMLP_SEM)) +#define lookup_kfmlp_sem(od)((struct pi_semaphore*) od_lookup(od, KFMLP_SEM)) +#define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM)) +#define lookup_ics(od) ((struct ics*) od_lookup(od, ICS_ID)) + +#define lookup_rsm_mutex(od)((struct litmus_lock*) od_lookup(od, FMLP_SEM)) + + +#endif diff --git a/include/litmus/feather_buffer.h b/include/litmus/feather_buffer.h new file mode 100644 index 0000000..6c18277 --- /dev/null +++ b/include/litmus/feather_buffer.h @@ -0,0 +1,94 @@ +#ifndef _FEATHER_BUFFER_H_ +#define _FEATHER_BUFFER_H_ + +/* requires UINT_MAX and memcpy */ + +#define SLOT_FREE 0 +#define SLOT_BUSY 1 +#define SLOT_READY 2 + +struct ft_buffer { + unsigned int slot_count; + unsigned int slot_size; + + int free_count; + unsigned int write_idx; + unsigned int read_idx; + + char* slots; + void* buffer_mem; + unsigned int failed_writes; +}; + +static inline int init_ft_buffer(struct ft_buffer* buf, + unsigned int slot_count, + unsigned int slot_size, + char* slots, + void* buffer_mem) +{ + int i = 0; + if (!slot_count || UINT_MAX % slot_count != slot_count - 1) { + /* The slot count must divide UNIT_MAX + 1 so that when it + * wraps around the index correctly points to 0. + */ + return 0; + } else { + buf->slot_count = slot_count; + buf->slot_size = slot_size; + buf->slots = slots; + buf->buffer_mem = buffer_mem; + buf->free_count = slot_count; + buf->write_idx = 0; + buf->read_idx = 0; + buf->failed_writes = 0; + for (i = 0; i < slot_count; i++) + buf->slots[i] = SLOT_FREE; + return 1; + } +} + +static inline int ft_buffer_start_write(struct ft_buffer* buf, void **ptr) +{ + int free = fetch_and_dec(&buf->free_count); + unsigned int idx; + if (free <= 0) { + fetch_and_inc(&buf->free_count); + *ptr = 0; + fetch_and_inc(&buf->failed_writes); + return 0; + } else { + idx = fetch_and_inc((int*) &buf->write_idx) % buf->slot_count; + buf->slots[idx] = SLOT_BUSY; + *ptr = ((char*) buf->buffer_mem) + idx * buf->slot_size; + return 1; + } +} + +static inline void ft_buffer_finish_write(struct ft_buffer* buf, void *ptr) +{ + unsigned int idx = ((char*) ptr - (char*) buf->buffer_mem) / buf->slot_size; + buf->slots[idx] = SLOT_READY; +} + + +/* exclusive reader access is assumed */ +static inline int ft_buffer_read(struct ft_buffer* buf, void* dest) +{ + unsigned int idx; + if (buf->free_count == buf->slot_count) + /* nothing available */ + return 0; + idx = buf->read_idx % buf->slot_count; + if (buf->slots[idx] == SLOT_READY) { + memcpy(dest, ((char*) buf->buffer_mem) + idx * buf->slot_size, + buf->slot_size); + buf->slots[idx] = SLOT_FREE; + buf->read_idx++; + fetch_and_inc(&buf->free_count); + return 1; + } else + return 0; +} + + +#endif diff --git a/include/litmus/feather_trace.h b/include/litmus/feather_trace.h new file mode 100644 index 0000000..028dfb2 --- /dev/null +++ b/include/litmus/feather_trace.h @@ -0,0 +1,65 @@ +#ifndef _FEATHER_TRACE_H_ +#define _FEATHER_TRACE_H_ + +#include + +int ft_enable_event(unsigned long id); +int ft_disable_event(unsigned long id); +int ft_is_event_enabled(unsigned long id); +int ft_disable_all_events(void); + +/* atomic_* funcitons are inline anyway */ +static inline int fetch_and_inc(int *val) +{ + return atomic_add_return(1, (atomic_t*) val) - 1; +} + +static inline int fetch_and_dec(int *val) +{ + return atomic_sub_return(1, (atomic_t*) val) + 1; +} + +/* Don't use rewriting implementation if kernel text pages are read-only. + * Ftrace gets around this by using the identity mapping, but that's more + * effort that is warrented right now for Feather-Trace. + * Eventually, it may make sense to replace Feather-Trace with ftrace. + */ +#if defined(CONFIG_ARCH_HAS_FEATHER_TRACE) && !defined(CONFIG_DEBUG_RODATA) + +#include + +#else /* !__ARCH_HAS_FEATHER_TRACE */ + +/* provide default implementation */ + +#include /* for get_cycles() */ + +static inline unsigned long long ft_timestamp(void) +{ + return get_cycles(); +} + +#define feather_callback + +#define MAX_EVENTS 1024 + +extern int ft_events[MAX_EVENTS]; + +#define ft_event(id, callback) \ + if (ft_events[id]) callback(); + +#define ft_event0(id, callback) \ + if (ft_events[id]) callback(id); + +#define ft_event1(id, callback, param) \ + if (ft_events[id]) callback(id, param); + +#define ft_event2(id, callback, param, param2) \ + if (ft_events[id]) callback(id, param, param2); + +#define ft_event3(id, callback, p, p2, p3) \ + if (ft_events[id]) callback(id, p, p2, p3); + +#endif /* __ARCH_HAS_FEATHER_TRACE */ + +#endif diff --git a/include/litmus/fpmath.h b/include/litmus/fpmath.h new file mode 100644 index 0000000..04d4bca --- /dev/null +++ b/include/litmus/fpmath.h @@ -0,0 +1,145 @@ +#ifndef __FP_MATH_H__ +#define __FP_MATH_H__ + +#ifndef __KERNEL__ +#include +#define abs(x) (((x) < 0) ? -(x) : x) +#endif + +// Use 64-bit because we want to track things at the nanosecond scale. +// This can lead to very large numbers. +typedef int64_t fpbuf_t; +typedef struct +{ + fpbuf_t val; +} fp_t; + +#define FP_SHIFT 10 +#define ROUND_BIT (FP_SHIFT - 1) + +#define _fp(x) ((fp_t) {x}) + +#ifdef __KERNEL__ +static const fp_t LITMUS_FP_ZERO = {.val = 0}; +static const fp_t LITMUS_FP_ONE = {.val = (1 << FP_SHIFT)}; +#endif + +static inline fp_t FP(fpbuf_t x) +{ + return _fp(((fpbuf_t) x) << FP_SHIFT); +} + +/* divide two integers to obtain a fixed point value */ +static inline fp_t _frac(fpbuf_t a, fpbuf_t b) +{ + return _fp(FP(a).val / (b)); +} + +static inline fpbuf_t _point(fp_t x) +{ + return (x.val % (1 << FP_SHIFT)); + +} + +#define fp2str(x) x.val +/*(x.val >> FP_SHIFT), (x.val % (1 << FP_SHIFT)) */ +#define _FP_ "%ld/1024" + +static inline fpbuf_t _floor(fp_t x) +{ + return x.val >> FP_SHIFT; +} + +/* FIXME: negative rounding */ +static inline fpbuf_t _round(fp_t x) +{ + return _floor(x) + ((x.val >> ROUND_BIT) & 1); +} + +/* multiply two fixed point values */ +static inline fp_t _mul(fp_t a, fp_t b) +{ + return _fp((a.val * b.val) >> FP_SHIFT); +} + +static inline fp_t _div(fp_t a, fp_t b) +{ +#if !defined(__KERNEL__) && !defined(unlikely) +#define unlikely(x) (x) +#define DO_UNDEF_UNLIKELY +#endif + /* try not to overflow */ + if (unlikely( a.val > (2l << ((sizeof(fpbuf_t)*8) - FP_SHIFT)) )) + return _fp((a.val / b.val) << FP_SHIFT); + else + return _fp((a.val << FP_SHIFT) / b.val); +#ifdef DO_UNDEF_UNLIKELY +#undef unlikely +#undef DO_UNDEF_UNLIKELY +#endif +} + +static inline fp_t _add(fp_t a, fp_t b) +{ + return _fp(a.val + b.val); +} + +static inline fp_t _sub(fp_t a, fp_t b) +{ + return _fp(a.val - b.val); +} + +static inline fp_t _neg(fp_t x) +{ + return _fp(-x.val); +} + +static inline fp_t _abs(fp_t x) +{ + return _fp(abs(x.val)); +} + +/* works the same as casting float/double to integer */ +static inline fpbuf_t _fp_to_integer(fp_t x) +{ + return _floor(_abs(x)) * ((x.val > 0) ? 1 : -1); +} + +static inline fp_t _integer_to_fp(fpbuf_t x) +{ + return _frac(x,1); +} + +static inline int _leq(fp_t a, fp_t b) +{ + return a.val <= b.val; +} + +static inline int _geq(fp_t a, fp_t b) +{ + return a.val >= b.val; +} + +static inline int _lt(fp_t a, fp_t b) +{ + return a.val < b.val; +} + +static inline int _gt(fp_t a, fp_t b) +{ + return a.val > b.val; +} + +static inline int _eq(fp_t a, fp_t b) +{ + return a.val == b.val; +} + +static inline fp_t _max(fp_t a, fp_t b) +{ + if (a.val < b.val) + return b; + else + return a; +} +#endif diff --git a/include/litmus/ftdev.h b/include/litmus/ftdev.h new file mode 100644 index 0000000..0b95987 --- /dev/null +++ b/include/litmus/ftdev.h @@ -0,0 +1,55 @@ +#ifndef _LITMUS_FTDEV_H_ +#define _LITMUS_FTDEV_H_ + +#include +#include +#include +#include + +#define FTDEV_ENABLE_CMD 0 +#define FTDEV_DISABLE_CMD 1 + +struct ftdev; + +/* return 0 if buffer can be opened, otherwise -$REASON */ +typedef int (*ftdev_can_open_t)(struct ftdev* dev, unsigned int buf_no); +/* return 0 on success, otherwise -$REASON */ +typedef int (*ftdev_alloc_t)(struct ftdev* dev, unsigned int buf_no); +typedef void (*ftdev_free_t)(struct ftdev* dev, unsigned int buf_no); +/* Let devices handle writes from userspace. No synchronization provided. */ +typedef ssize_t (*ftdev_write_t)(struct ft_buffer* buf, size_t len, const char __user *from); + +struct ftdev_event; + +struct ftdev_minor { + struct ft_buffer* buf; + unsigned int readers; + struct mutex lock; + /* FIXME: filter for authorized events */ + struct ftdev_event* events; + struct device* device; + struct ftdev* ftdev; +}; + +struct ftdev { + dev_t major; + struct cdev cdev; + struct class* class; + const char* name; + struct ftdev_minor* minor; + unsigned int minor_cnt; + ftdev_alloc_t alloc; + ftdev_free_t free; + ftdev_can_open_t can_open; + ftdev_write_t write; +}; + +struct ft_buffer* alloc_ft_buffer(unsigned int count, size_t size); +void free_ft_buffer(struct ft_buffer* buf); + +int ftdev_init( struct ftdev* ftdev, struct module* owner, + const int minor_cnt, const char* name); +void ftdev_exit(struct ftdev* ftdev); +int register_ftdev(struct ftdev* ftdev); + +#endif diff --git a/include/litmus/gpu_affinity.h b/include/litmus/gpu_affinity.h new file mode 100644 index 0000000..6b3fb8b --- /dev/null +++ b/include/litmus/gpu_affinity.h @@ -0,0 +1,49 @@ +#ifndef LITMUS_GPU_AFFINITY_H +#define LITMUS_GPU_AFFINITY_H + +#include +#include +#include + +void update_gpu_estimate(struct task_struct* t, lt_t observed); +gpu_migration_dist_t gpu_migration_distance(int a, int b); + +static inline void reset_gpu_tracker(struct task_struct* t) +{ + t->rt_param.accum_gpu_time = 0; +} + +static inline void start_gpu_tracker(struct task_struct* t) +{ + t->rt_param.gpu_time_stamp = litmus_clock(); +} + +static inline void stop_gpu_tracker(struct task_struct* t) +{ + lt_t now = litmus_clock(); + t->rt_param.accum_gpu_time += (now - t->rt_param.gpu_time_stamp); +} + +static inline lt_t get_gpu_time(struct task_struct* t) +{ + return t->rt_param.accum_gpu_time; +} + +static inline lt_t get_gpu_estimate(struct task_struct* t, gpu_migration_dist_t dist) +{ + int i; + fpbuf_t temp = _fp_to_integer(t->rt_param.gpu_migration_est[dist].est); + lt_t val = (temp >= 0) ? temp : 0; // never allow negative estimates... + + WARN_ON(temp < 0); + + // lower-bound a distant migration to be at least equal to the level + // below it. + for(i = dist-1; (val == 0) && (i >= MIG_LOCAL); --i) { + val = _fp_to_integer(t->rt_param.gpu_migration_est[i].est); + } + + return ((val > 0) ? val : dist+1); +} + +#endif diff --git a/include/litmus/ikglp_lock.h b/include/litmus/ikglp_lock.h new file mode 100644 index 0000000..af6f151 --- /dev/null +++ b/include/litmus/ikglp_lock.h @@ -0,0 +1,160 @@ +#ifndef LITMUS_IKGLP_H +#define LITMUS_IKGLP_H + +#include +#include +#include + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING +#include + +struct ikglp_affinity; +#endif + +typedef struct ikglp_heap_node +{ + struct task_struct *task; + struct binheap_node node; +} ikglp_heap_node_t; + +struct fifo_queue; +struct ikglp_wait_state; + +typedef struct ikglp_donee_heap_node +{ + struct task_struct *task; + struct fifo_queue *fq; + struct ikglp_wait_state *donor_info; // cross-linked with ikglp_wait_state_t of donor + + struct binheap_node node; +} ikglp_donee_heap_node_t; + +// Maintains the state of a request as it goes through the IKGLP +typedef struct ikglp_wait_state { + struct task_struct *task; // pointer back to the requesting task + + // Data for while waiting in FIFO Queue + wait_queue_t fq_node; + ikglp_heap_node_t global_heap_node; + ikglp_donee_heap_node_t donee_heap_node; + + // Data for while waiting in PQ + ikglp_heap_node_t pq_node; + + // Data for while waiting as a donor + ikglp_donee_heap_node_t *donee_info; // cross-linked with donee's ikglp_donee_heap_node_t + struct nested_info prio_donation; + struct binheap_node node; +} ikglp_wait_state_t; + +/* struct for semaphore with priority inheritance */ +struct fifo_queue +{ + wait_queue_head_t wait; + struct task_struct* owner; + + // used for bookkeepping + ikglp_heap_node_t global_heap_node; + ikglp_donee_heap_node_t donee_heap_node; + + struct task_struct* hp_waiter; + int count; /* number of waiters + holder */ + + struct nested_info nest; +}; + +struct ikglp_semaphore +{ + struct litmus_lock litmus_lock; + + raw_spinlock_t lock; + raw_spinlock_t real_lock; + + int nr_replicas; // AKA k + int m; + + int max_fifo_len; // max len of a fifo queue + int nr_in_fifos; + + struct binheap_handle top_m; // min heap, base prio + int top_m_size; // number of nodes in top_m + + struct binheap_handle not_top_m; // max heap, base prio + + struct binheap_handle donees; // min-heap, base prio + struct fifo_queue *shortest_fifo_queue; // pointer to shortest fifo queue + + /* data structures for holding requests */ + struct fifo_queue *fifo_queues; // array nr_replicas in length + struct binheap_handle priority_queue; // max-heap, base prio + struct binheap_handle donors; // max-heap, base prio + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + struct ikglp_affinity *aff_obs; +#endif +}; + +static inline struct ikglp_semaphore* ikglp_from_lock(struct litmus_lock* lock) +{ + return container_of(lock, struct ikglp_semaphore, litmus_lock); +} + +int ikglp_lock(struct litmus_lock* l); +int ikglp_unlock(struct litmus_lock* l); +int ikglp_close(struct litmus_lock* l); +void ikglp_free(struct litmus_lock* l); +struct litmus_lock* ikglp_new(int m, struct litmus_lock_ops*, void* __user arg); + + + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) + +struct ikglp_queue_info +{ + struct fifo_queue* q; + lt_t estimated_len; + int *nr_cur_users; +}; + +struct ikglp_affinity_ops +{ + struct fifo_queue* (*advise_enqueue)(struct ikglp_affinity* aff, struct task_struct* t); // select FIFO + ikglp_wait_state_t* (*advise_steal)(struct ikglp_affinity* aff, struct fifo_queue* dst); // select steal from FIFO + ikglp_donee_heap_node_t* (*advise_donee_selection)(struct ikglp_affinity* aff, struct task_struct* t); // select a donee + ikglp_wait_state_t* (*advise_donor_to_fq)(struct ikglp_affinity* aff, struct fifo_queue* dst); // select a donor to move to PQ + + void (*notify_enqueue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // fifo enqueue + void (*notify_dequeue)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // fifo dequeue + void (*notify_acquired)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // replica acquired + void (*notify_freed)(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t); // replica freed + int (*replica_to_resource)(struct ikglp_affinity* aff, struct fifo_queue* fq); // convert a replica # to a GPU (includes offsets and simult user folding) +}; + +struct ikglp_affinity +{ + struct affinity_observer obs; + struct ikglp_affinity_ops *ops; + struct ikglp_queue_info *q_info; + int *nr_cur_users_on_rsrc; + int offset; + int nr_simult; + int nr_rsrc; + int relax_max_fifo_len; +}; + +static inline struct ikglp_affinity* ikglp_aff_obs_from_aff_obs(struct affinity_observer* aff_obs) +{ + return container_of(aff_obs, struct ikglp_affinity, obs); +} + +int ikglp_aff_obs_close(struct affinity_observer*); +void ikglp_aff_obs_free(struct affinity_observer*); +struct affinity_observer* ikglp_gpu_aff_obs_new(struct affinity_observer_ops*, + void* __user arg); +struct affinity_observer* ikglp_simple_gpu_aff_obs_new(struct affinity_observer_ops*, + void* __user arg); +#endif + + + +#endif diff --git a/include/litmus/jobs.h b/include/litmus/jobs.h new file mode 100644 index 0000000..9bd361e --- /dev/null +++ b/include/litmus/jobs.h @@ -0,0 +1,9 @@ +#ifndef __LITMUS_JOBS_H__ +#define __LITMUS_JOBS_H__ + +void prepare_for_next_period(struct task_struct *t); +void release_at(struct task_struct *t, lt_t start); +long complete_job(void); + +#endif + diff --git a/include/litmus/kexclu_affinity.h b/include/litmus/kexclu_affinity.h new file mode 100644 index 0000000..f6355de --- /dev/null +++ b/include/litmus/kexclu_affinity.h @@ -0,0 +1,35 @@ +#ifndef LITMUS_AFF_OBS_H +#define LITMUS_AFF_OBS_H + +#include + +struct affinity_observer_ops; + +struct affinity_observer +{ + struct affinity_observer_ops* ops; + int type; + int ident; + + struct litmus_lock* lock; // the lock under observation +}; + +typedef int (*aff_obs_open_t)(struct affinity_observer* aff_obs, + void* __user arg); +typedef int (*aff_obs_close_t)(struct affinity_observer* aff_obs); +typedef void (*aff_obs_free_t)(struct affinity_observer* aff_obs); + +struct affinity_observer_ops +{ + aff_obs_open_t open; + aff_obs_close_t close; + aff_obs_free_t deallocate; +}; + +struct litmus_lock* get_lock_from_od(int od); + +void affinity_observer_new(struct affinity_observer* aff, + struct affinity_observer_ops* ops, + struct affinity_observer_args* args); + +#endif diff --git a/include/litmus/kfmlp_lock.h b/include/litmus/kfmlp_lock.h new file mode 100644 index 0000000..5f0aae6 --- /dev/null +++ b/include/litmus/kfmlp_lock.h @@ -0,0 +1,97 @@ +#ifndef LITMUS_KFMLP_H +#define LITMUS_KFMLP_H + +#include +#include + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING +#include + +struct kfmlp_affinity; +#endif + +/* struct for semaphore with priority inheritance */ +struct kfmlp_queue +{ + wait_queue_head_t wait; + struct task_struct* owner; + struct task_struct* hp_waiter; + int count; /* number of waiters + holder */ +}; + +struct kfmlp_semaphore +{ + struct litmus_lock litmus_lock; + + spinlock_t lock; + + int num_resources; /* aka k */ + + struct kfmlp_queue *queues; /* array */ + struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */ + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + struct kfmlp_affinity *aff_obs; +#endif +}; + +static inline struct kfmlp_semaphore* kfmlp_from_lock(struct litmus_lock* lock) +{ + return container_of(lock, struct kfmlp_semaphore, litmus_lock); +} + +int kfmlp_lock(struct litmus_lock* l); +int kfmlp_unlock(struct litmus_lock* l); +int kfmlp_close(struct litmus_lock* l); +void kfmlp_free(struct litmus_lock* l); +struct litmus_lock* kfmlp_new(struct litmus_lock_ops*, void* __user arg); + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) + +struct kfmlp_queue_info +{ + struct kfmlp_queue* q; + lt_t estimated_len; + int *nr_cur_users; +}; + +struct kfmlp_affinity_ops +{ + struct kfmlp_queue* (*advise_enqueue)(struct kfmlp_affinity* aff, struct task_struct* t); + struct task_struct* (*advise_steal)(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from); + void (*notify_enqueue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); + void (*notify_dequeue)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); + void (*notify_acquired)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); + void (*notify_freed)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t); + int (*replica_to_resource)(struct kfmlp_affinity* aff, struct kfmlp_queue* fq); +}; + +struct kfmlp_affinity +{ + struct affinity_observer obs; + struct kfmlp_affinity_ops *ops; + struct kfmlp_queue_info *q_info; + int *nr_cur_users_on_rsrc; + int offset; + int nr_simult; + int nr_rsrc; +}; + +static inline struct kfmlp_affinity* kfmlp_aff_obs_from_aff_obs(struct affinity_observer* aff_obs) +{ + return container_of(aff_obs, struct kfmlp_affinity, obs); +} + +int kfmlp_aff_obs_close(struct affinity_observer*); +void kfmlp_aff_obs_free(struct affinity_observer*); +struct affinity_observer* kfmlp_gpu_aff_obs_new(struct affinity_observer_ops*, + void* __user arg); +struct affinity_observer* kfmlp_simple_gpu_aff_obs_new(struct affinity_observer_ops*, + void* __user arg); + + +#endif + +#endif + + diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h new file mode 100644 index 0000000..71df378 --- /dev/null +++ b/include/litmus/litmus.h @@ -0,0 +1,282 @@ +/* + * Constant definitions related to + * scheduling policy. + */ + +#ifndef _LINUX_LITMUS_H_ +#define _LINUX_LITMUS_H_ + +#include + +#ifdef CONFIG_RELEASE_MASTER +extern atomic_t release_master_cpu; +#endif + +/* in_list - is a given list_head queued on some list? + */ +static inline int in_list(struct list_head* list) +{ + return !( /* case 1: deleted */ + (list->next == LIST_POISON1 && + list->prev == LIST_POISON2) + || + /* case 2: initialized */ + (list->next == list && + list->prev == list) + ); +} + + +struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq); + +#define NO_CPU 0xffffffff + +void litmus_fork(struct task_struct *tsk); +void litmus_exec(void); +/* clean up real-time state of a task */ +void exit_litmus(struct task_struct *dead_tsk); + +long litmus_admit_task(struct task_struct *tsk); +void litmus_exit_task(struct task_struct *tsk); + +#define is_realtime(t) ((t)->policy == SCHED_LITMUS) +#define rt_transition_pending(t) \ + ((t)->rt_param.transition_pending) + +#define tsk_rt(t) (&(t)->rt_param) + +/* Realtime utility macros */ +#define get_rt_flags(t) (tsk_rt(t)->flags) +#define set_rt_flags(t,f) (tsk_rt(t)->flags=(f)) +#define get_exec_cost(t) (tsk_rt(t)->task_params.exec_cost) +#define get_exec_time(t) (tsk_rt(t)->job_params.exec_time) +#define get_rt_period(t) (tsk_rt(t)->task_params.period) +#define get_rt_phase(t) (tsk_rt(t)->task_params.phase) +#define get_partition(t) (tsk_rt(t)->task_params.cpu) +#define get_deadline(t) (tsk_rt(t)->job_params.deadline) +#define get_period(t) (tsk_rt(t)->task_params.period) +#define get_release(t) (tsk_rt(t)->job_params.release) +#define get_class(t) (tsk_rt(t)->task_params.cls) + +#define is_priority_boosted(t) (tsk_rt(t)->priority_boosted) +#define get_boost_start(t) (tsk_rt(t)->boost_start_time) + +#define effective_priority(t) ((!(tsk_rt(t)->inh_task)) ? t : tsk_rt(t)->inh_task) +#define base_priority(t) (t) + +inline static int budget_exhausted(struct task_struct* t) +{ + return get_exec_time(t) >= get_exec_cost(t); +} + +inline static lt_t budget_remaining(struct task_struct* t) +{ + if (!budget_exhausted(t)) + return get_exec_cost(t) - get_exec_time(t); + else + /* avoid overflow */ + return 0; +} + +#define budget_enforced(t) (tsk_rt(t)->task_params.budget_policy != NO_ENFORCEMENT) + +#define budget_precisely_enforced(t) (tsk_rt(t)->task_params.budget_policy \ + == PRECISE_ENFORCEMENT) + +#define is_hrt(t) \ + (tsk_rt(t)->task_params.cls == RT_CLASS_HARD) +#define is_srt(t) \ + (tsk_rt(t)->task_params.cls == RT_CLASS_SOFT) +#define is_be(t) \ + (tsk_rt(t)->task_params.cls == RT_CLASS_BEST_EFFORT) + +/* Our notion of time within LITMUS: kernel monotonic time. */ +static inline lt_t litmus_clock(void) +{ + return ktime_to_ns(ktime_get()); +} + +/* A macro to convert from nanoseconds to ktime_t. */ +#define ns_to_ktime(t) ktime_add_ns(ktime_set(0, 0), t) + +#define get_domain(t) (tsk_rt(t)->domain) + +/* Honor the flag in the preempt_count variable that is set + * when scheduling is in progress. + */ +#define is_running(t) \ + ((t)->state == TASK_RUNNING || \ + task_thread_info(t)->preempt_count & PREEMPT_ACTIVE) + +#define is_blocked(t) \ + (!is_running(t)) +#define is_released(t, now) \ + (lt_before_eq(get_release(t), now)) +#define is_tardy(t, now) \ + (lt_before_eq(tsk_rt(t)->job_params.deadline, now)) + +/* real-time comparison macros */ +#define earlier_deadline(a, b) (lt_before(\ + (a)->rt_param.job_params.deadline,\ + (b)->rt_param.job_params.deadline)) +#define shorter_period(a, b) (lt_before(\ + (a)->rt_param.task_params.period,\ + (b)->rt_param.task_params.period)) +#define earlier_release(a, b) (lt_before(\ + (a)->rt_param.job_params.release,\ + (b)->rt_param.job_params.release)) +void preempt_if_preemptable(struct task_struct* t, int on_cpu); + +#ifdef CONFIG_LITMUS_LOCKING +void srp_ceiling_block(void); +#else +#define srp_ceiling_block() /* nothing */ +#endif + +#define bheap2task(hn) ((struct task_struct*) hn->value) + +#ifdef CONFIG_NP_SECTION + +static inline int is_kernel_np(struct task_struct *t) +{ + return tsk_rt(t)->kernel_np; +} + +static inline int is_user_np(struct task_struct *t) +{ + return tsk_rt(t)->ctrl_page ? tsk_rt(t)->ctrl_page->sched.np.flag : 0; +} + +static inline void request_exit_np(struct task_struct *t) +{ + if (is_user_np(t)) { + /* Set the flag that tells user space to call + * into the kernel at the end of a critical section. */ + if (likely(tsk_rt(t)->ctrl_page)) { + TRACE_TASK(t, "setting delayed_preemption flag\n"); + tsk_rt(t)->ctrl_page->sched.np.preempt = 1; + } + } +} + +static inline void make_np(struct task_struct *t) +{ + tsk_rt(t)->kernel_np++; +} + +/* Caller should check if preemption is necessary when + * the function return 0. + */ +static inline int take_np(struct task_struct *t) +{ + return --tsk_rt(t)->kernel_np; +} + +/* returns 0 if remote CPU needs an IPI to preempt, 1 if no IPI is required */ +static inline int request_exit_np_atomic(struct task_struct *t) +{ + union np_flag old, new; + + if (tsk_rt(t)->ctrl_page) { + old.raw = tsk_rt(t)->ctrl_page->sched.raw; + if (old.np.flag == 0) { + /* no longer non-preemptive */ + return 0; + } else if (old.np.preempt) { + /* already set, nothing for us to do */ + return 1; + } else { + /* non preemptive and flag not set */ + new.raw = old.raw; + new.np.preempt = 1; + /* if we get old back, then we atomically set the flag */ + return cmpxchg(&tsk_rt(t)->ctrl_page->sched.raw, old.raw, new.raw) == old.raw; + /* If we raced with a concurrent change, then so be + * it. Deliver it by IPI. We don't want an unbounded + * retry loop here since tasks might exploit that to + * keep the kernel busy indefinitely. */ + } + } else + return 0; +} + +#else + +static inline int is_kernel_np(struct task_struct* t) +{ + return 0; +} + +static inline int is_user_np(struct task_struct* t) +{ + return 0; +} + +static inline void request_exit_np(struct task_struct *t) +{ + /* request_exit_np() shouldn't be called if !CONFIG_NP_SECTION */ + BUG(); +} + +static inline int request_exit_np_atomic(struct task_struct *t) +{ + return 0; +} + +#endif + +static inline void clear_exit_np(struct task_struct *t) +{ + if (likely(tsk_rt(t)->ctrl_page)) + tsk_rt(t)->ctrl_page->sched.np.preempt = 0; +} + +static inline int is_np(struct task_struct *t) +{ +#ifdef CONFIG_SCHED_DEBUG_TRACE + int kernel, user; + kernel = is_kernel_np(t); + user = is_user_np(t); + if (kernel || user) + TRACE_TASK(t, " is non-preemptive: kernel=%d user=%d\n", + + kernel, user); + return kernel || user; +#else + return unlikely(is_kernel_np(t) || is_user_np(t)); +#endif +} + +static inline int is_present(struct task_struct* t) +{ + return t && tsk_rt(t)->present; +} + + +/* make the unit explicit */ +typedef unsigned long quanta_t; + +enum round { + FLOOR, + CEIL +}; + + +/* Tick period is used to convert ns-specified execution + * costs and periods into tick-based equivalents. + */ +extern ktime_t tick_period; + +static inline quanta_t time2quanta(lt_t time, enum round round) +{ + s64 quantum_length = ktime_to_ns(tick_period); + + if (do_div(time, quantum_length) && round == CEIL) + time++; + return (quanta_t) time; +} + +/* By how much is cpu staggered behind CPU 0? */ +u64 cpu_stagger_offset(int cpu); + +#endif diff --git a/include/litmus/litmus_proc.h b/include/litmus/litmus_proc.h new file mode 100644 index 0000000..6800e72 --- /dev/null +++ b/include/litmus/litmus_proc.h @@ -0,0 +1,25 @@ +#include +#include + +int __init init_litmus_proc(void); +void exit_litmus_proc(void); + +/* + * On success, returns 0 and sets the pointer to the location of the new + * proc dir entry, otherwise returns an error code and sets pde to NULL. + */ +long make_plugin_proc_dir(struct sched_plugin* plugin, + struct proc_dir_entry** pde); + +/* + * Plugins should deallocate all child proc directory entries before + * calling this, to avoid memory leaks. + */ +void remove_plugin_proc_dir(struct sched_plugin* plugin); + + +/* Copy at most size-1 bytes from ubuf into kbuf, null-terminate buf, and + * remove a '\n' if present. Returns the number of bytes that were read or + * -EFAULT. */ +int copy_and_chomp(char *kbuf, unsigned long ksize, + __user const char* ubuf, unsigned long ulength); diff --git a/include/litmus/litmus_softirq.h b/include/litmus/litmus_softirq.h new file mode 100644 index 0000000..1eb5ea1 --- /dev/null +++ b/include/litmus/litmus_softirq.h @@ -0,0 +1,199 @@ +#ifndef __LITMUS_SOFTIRQ_H +#define __LITMUS_SOFTIRQ_H + +#include +#include + +/* + Threaded tasklet handling for Litmus. Tasklets + are scheduled with the priority of the tasklet's + owner---that is, the RT task on behalf the tasklet + runs. + + Tasklets are current scheduled in FIFO order with + NO priority inheritance for "blocked" tasklets. + + klitirqd assumes the priority of the owner of the + tasklet when the tasklet is next to execute. + + Currently, hi-tasklets are scheduled before + low-tasklets, regardless of priority of low-tasklets. + And likewise, low-tasklets are scheduled before work + queue objects. This priority inversion probably needs + to be fixed, though it is not an issue if our work with + GPUs as GPUs are owned (and associated klitirqds) for + exclusive time periods, thus no inversions can + occur. + */ + + + +#define NR_LITMUS_SOFTIRQD CONFIG_NR_LITMUS_SOFTIRQD + +/* Spawns NR_LITMUS_SOFTIRQD klitirqd daemons. + Actual launch of threads is deffered to kworker's + workqueue, so daemons will likely not be immediately + running when this function returns, though the required + data will be initialized. + + @affinity_set: an array expressing the processor affinity + for each of the NR_LITMUS_SOFTIRQD daemons. May be set + to NULL for global scheduling. + + - Examples - + 8-CPU system with two CPU clusters: + affinity[] = {0, 0, 0, 0, 3, 3, 3, 3} + NOTE: Daemons not actually bound to specified CPU, but rather + cluster in which the CPU resides. + + 8-CPU system, partitioned: + affinity[] = {0, 1, 2, 3, 4, 5, 6, 7} + + FIXME: change array to a CPU topology or array of cpumasks + + */ +void spawn_klitirqd(int* affinity); + + +/* Raises a flag to tell klitirqds to terminate. + Termination is async, so some threads may be running + after function return. */ +void kill_klitirqd(void); + + +/* Returns 1 if all NR_LITMUS_SOFTIRQD klitirqs are ready + to handle tasklets. 0, otherwise.*/ +int klitirqd_is_ready(void); + +/* Returns 1 if no NR_LITMUS_SOFTIRQD klitirqs are ready + to handle tasklets. 0, otherwise.*/ +int klitirqd_is_dead(void); + +/* Flushes all pending work out to the OS for regular + * tasklet/work processing of the specified 'owner' + * + * PRECOND: klitirqd_thread must have a clear entry + * in the GPU registry, otherwise this call will become + * a no-op as work will loop back to the klitirqd_thread. + * + * Pass NULL for owner to flush ALL pending items. + */ +void flush_pending(struct task_struct* klitirqd_thread, + struct task_struct* owner); + +struct task_struct* get_klitirqd(unsigned int k_id); + + +extern int __litmus_tasklet_schedule( + struct tasklet_struct *t, + unsigned int k_id); + +/* schedule a tasklet on klitirqd #k_id */ +static inline int litmus_tasklet_schedule( + struct tasklet_struct *t, + unsigned int k_id) +{ + int ret = 0; + if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) + ret = __litmus_tasklet_schedule(t, k_id); + return(ret); +} + +/* for use by __tasklet_schedule() */ +static inline int _litmus_tasklet_schedule( + struct tasklet_struct *t, + unsigned int k_id) +{ + return(__litmus_tasklet_schedule(t, k_id)); +} + + + + +extern int __litmus_tasklet_hi_schedule(struct tasklet_struct *t, + unsigned int k_id); + +/* schedule a hi tasklet on klitirqd #k_id */ +static inline int litmus_tasklet_hi_schedule(struct tasklet_struct *t, + unsigned int k_id) +{ + int ret = 0; + if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) + ret = __litmus_tasklet_hi_schedule(t, k_id); + return(ret); +} + +/* for use by __tasklet_hi_schedule() */ +static inline int _litmus_tasklet_hi_schedule(struct tasklet_struct *t, + unsigned int k_id) +{ + return(__litmus_tasklet_hi_schedule(t, k_id)); +} + + + + + +extern int __litmus_tasklet_hi_schedule_first( + struct tasklet_struct *t, + unsigned int k_id); + +/* schedule a hi tasklet on klitirqd #k_id on next go-around */ +/* PRECONDITION: Interrupts must be disabled. */ +static inline int litmus_tasklet_hi_schedule_first( + struct tasklet_struct *t, + unsigned int k_id) +{ + int ret = 0; + if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) + ret = __litmus_tasklet_hi_schedule_first(t, k_id); + return(ret); +} + +/* for use by __tasklet_hi_schedule_first() */ +static inline int _litmus_tasklet_hi_schedule_first( + struct tasklet_struct *t, + unsigned int k_id) +{ + return(__litmus_tasklet_hi_schedule_first(t, k_id)); +} + + + +////////////// + +extern int __litmus_schedule_work( + struct work_struct* w, + unsigned int k_id); + +static inline int litmus_schedule_work( + struct work_struct* w, + unsigned int k_id) +{ + return(__litmus_schedule_work(w, k_id)); +} + + + +///////////// mutex operations for client threads. + +void down_and_set_stat(struct task_struct* t, + enum klitirqd_sem_status to_set, + struct mutex* sem); + +void __down_and_reset_and_set_stat(struct task_struct* t, + enum klitirqd_sem_status to_reset, + enum klitirqd_sem_status to_set, + struct mutex* sem); + +void up_and_set_stat(struct task_struct* t, + enum klitirqd_sem_status to_set, + struct mutex* sem); + + + +void release_klitirqd_lock(struct task_struct* t); + +int reacquire_klitirqd_lock(struct task_struct* t); + +#endif diff --git a/include/litmus/locking.h b/include/litmus/locking.h new file mode 100644 index 0000000..36647fe --- /dev/null +++ b/include/litmus/locking.h @@ -0,0 +1,160 @@ +#ifndef LITMUS_LOCKING_H +#define LITMUS_LOCKING_H + +#include + +struct litmus_lock_ops; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING +struct nested_info +{ + struct litmus_lock *lock; + struct task_struct *hp_waiter_eff_prio; + struct task_struct **hp_waiter_ptr; + struct binheap_node hp_binheap_node; +}; + +static inline struct task_struct* top_priority(struct binheap_handle* handle) { + if(!binheap_empty(handle)) { + return (struct task_struct*)(binheap_top_entry(handle, struct nested_info, hp_binheap_node)->hp_waiter_eff_prio); + } + return NULL; +} + +void print_hp_waiters(struct binheap_node* n, int depth); +#endif + + +/* Generic base struct for LITMUS^RT userspace semaphores. + * This structure should be embedded in protocol-specific semaphores. + */ +struct litmus_lock { + struct litmus_lock_ops *ops; + int type; + + int ident; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + struct nested_info nest; +//#ifdef CONFIG_DEBUG_SPINLOCK + char cheat_lockdep[2]; + struct lock_class_key key; +//#endif +#endif +}; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + +#define MAX_DGL_SIZE CONFIG_LITMUS_MAX_DGL_SIZE + +typedef struct dgl_wait_state { + struct task_struct *task; /* task waiting on DGL */ + struct litmus_lock *locks[MAX_DGL_SIZE]; /* requested locks in DGL */ + int size; /* size of the DGL */ + int nr_remaining; /* nr locks remainging before DGL is complete */ + int last_primary; /* index lock in locks[] that has active priority */ + wait_queue_t wq_nodes[MAX_DGL_SIZE]; +} dgl_wait_state_t; + +void wake_or_wait_on_next_lock(dgl_wait_state_t *dgl_wait); +void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lock*/); + +void init_dgl_waitqueue_entry(wait_queue_t *wq_node, dgl_wait_state_t* dgl_wait); +int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key); +void __waitqueue_dgl_remove_first(wait_queue_head_t *wq, dgl_wait_state_t** dgl_wait, struct task_struct **task); +#endif + +typedef int (*lock_op_t)(struct litmus_lock *l); +typedef lock_op_t lock_close_t; +typedef lock_op_t lock_lock_t; +typedef lock_op_t lock_unlock_t; + +typedef int (*lock_open_t)(struct litmus_lock *l, void* __user arg); +typedef void (*lock_free_t)(struct litmus_lock *l); + +struct litmus_lock_ops { + /* Current task tries to obtain / drop a reference to a lock. + * Optional methods, allowed by default. */ + lock_open_t open; + lock_close_t close; + + /* Current tries to lock/unlock this lock (mandatory methods). */ + lock_lock_t lock; + lock_unlock_t unlock; + + /* The lock is no longer being referenced (mandatory method). */ + lock_free_t deallocate; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + void (*propagate_increase_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags); + void (*propagate_decrease_inheritance)(struct litmus_lock* l, struct task_struct* t, raw_spinlock_t* to_unlock, unsigned long irqflags); +#endif + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + raw_spinlock_t* (*get_dgl_spin_lock)(struct litmus_lock *l); + int (*dgl_lock)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node); + int (*is_owner)(struct litmus_lock *l, struct task_struct *t); + void (*enable_priority)(struct litmus_lock *l, dgl_wait_state_t* dgl_wait); +#endif +}; + + +/* + Nested inheritance can be achieved with fine-grain locking when there is + no need for DGL support, presuming locks are acquired in a partial order + (no cycles!). However, DGLs allow locks to be acquired in any order. This + makes nested inheritance very difficult (we don't yet know a solution) to + realize with fine-grain locks, so we use a big lock instead. + + Code contains both fine-grain and coarse-grain methods together, side-by-side. + Each lock operation *IS NOT* surrounded by ifdef/endif to help make code more + readable. However, this leads to the odd situation where both code paths + appear together in code as if they were both active together. + + THIS IS NOT REALLY THE CASE! ONLY ONE CODE PATH IS ACTUALLY ACTIVE! + + Example: + lock_global_irqsave(coarseLock, flags); + lock_fine_irqsave(fineLock, flags); + + Reality (coarse): + lock_global_irqsave(coarseLock, flags); + //lock_fine_irqsave(fineLock, flags); + + Reality (fine): + //lock_global_irqsave(coarseLock, flags); + lock_fine_irqsave(fineLock, flags); + + Be careful when you read code involving nested inheritance. + */ +#if defined(CONFIG_LITMUS_DGL_SUPPORT) +/* DGL requires a big lock to implement nested inheritance */ +#define lock_global_irqsave(lock, flags) raw_spin_lock_irqsave((lock), (flags)) +#define lock_global(lock) raw_spin_lock((lock)) +#define unlock_global_irqrestore(lock, flags) raw_spin_unlock_irqrestore((lock), (flags)) +#define unlock_global(lock) raw_spin_unlock((lock)) + +/* fine-grain locking are no-ops with DGL support */ +#define lock_fine_irqsave(lock, flags) +#define lock_fine(lock) +#define unlock_fine_irqrestore(lock, flags) +#define unlock_fine(lock) + +#elif defined(CONFIG_LITMUS_NESTED_LOCKING) +/* Use fine-grain locking when DGLs are disabled. */ +/* global locking are no-ops without DGL support */ +#define lock_global_irqsave(lock, flags) +#define lock_global(lock) +#define unlock_global_irqrestore(lock, flags) +#define unlock_global(lock) + +#define lock_fine_irqsave(lock, flags) raw_spin_lock_irqsave((lock), (flags)) +#define lock_fine(lock) raw_spin_lock((lock)) +#define unlock_fine_irqrestore(lock, flags) raw_spin_unlock_irqrestore((lock), (flags)) +#define unlock_fine(lock) raw_spin_unlock((lock)) + +#endif + + +#endif + diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h new file mode 100644 index 0000000..97c9577 --- /dev/null +++ b/include/litmus/nvidia_info.h @@ -0,0 +1,46 @@ +#ifndef __LITMUS_NVIDIA_H +#define __LITMUS_NVIDIA_H + +#include + + +#include + + +//#define NV_DEVICE_NUM NR_LITMUS_SOFTIRQD +#define NV_DEVICE_NUM CONFIG_NV_DEVICE_NUM +#define NV_MAX_SIMULT_USERS CONFIG_NV_MAX_SIMULT_USERS + +int init_nvidia_info(void); +void shutdown_nvidia_info(void); + +int is_nvidia_func(void* func_addr); + +void dump_nvidia_info(const struct tasklet_struct *t); + + +// Returns the Nvidia device # associated with provided tasklet and work_struct. +u32 get_tasklet_nv_device_num(const struct tasklet_struct *t); +u32 get_work_nv_device_num(const struct work_struct *t); + + +int init_nv_device_reg(void); +//int get_nv_device_id(struct task_struct* owner); + + +int reg_nv_device(int reg_device_id, int register_device, struct task_struct *t); + +struct task_struct* get_nv_max_device_owner(u32 target_device_id); +//int is_nv_device_owner(u32 target_device_id); + +void lock_nv_registry(u32 reg_device_id, unsigned long* flags); +void unlock_nv_registry(u32 reg_device_id, unsigned long* flags); + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD +void pai_check_priority_increase(struct task_struct *t, int reg_device_id); +void pai_check_priority_decrease(struct task_struct *t, int reg_device_id); +#endif + +//void increment_nv_int_count(u32 device); + +#endif diff --git a/include/litmus/preempt.h b/include/litmus/preempt.h new file mode 100644 index 0000000..8f3a9ca --- /dev/null +++ b/include/litmus/preempt.h @@ -0,0 +1,164 @@ +#ifndef LITMUS_PREEMPT_H +#define LITMUS_PREEMPT_H + +#include +#include +#include +#include + +#include + +extern DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, resched_state); + +#ifdef CONFIG_PREEMPT_STATE_TRACE +const char* sched_state_name(int s); +#define TRACE_STATE(fmt, args...) TRACE("SCHED_STATE " fmt, args) +#else +#define TRACE_STATE(fmt, args...) /* ignore */ +#endif + +#define VERIFY_SCHED_STATE(x) \ + do { int __s = get_sched_state(); \ + if ((__s & (x)) == 0) \ + TRACE_STATE("INVALID s=0x%x (%s) not " \ + "in 0x%x (%s) [%s]\n", \ + __s, sched_state_name(__s), \ + (x), #x, __FUNCTION__); \ + } while (0); + +//#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) /* ignore */ +#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) \ + TRACE_STATE("[P%d] 0x%x (%s) -> 0x%x (%s)\n", \ + cpu, (x), sched_state_name(x), \ + (y), sched_state_name(y)) + +typedef enum scheduling_state { + TASK_SCHEDULED = (1 << 0), /* The currently scheduled task is the one that + * should be scheduled, and the processor does not + * plan to invoke schedule(). */ + SHOULD_SCHEDULE = (1 << 1), /* A remote processor has determined that the + * processor should reschedule, but this has not + * been communicated yet (IPI still pending). */ + WILL_SCHEDULE = (1 << 2), /* The processor has noticed that it has to + * reschedule and will do so shortly. */ + TASK_PICKED = (1 << 3), /* The processor is currently executing schedule(), + * has selected a new task to schedule, but has not + * yet performed the actual context switch. */ + PICKED_WRONG_TASK = (1 << 4), /* The processor has not yet performed the context + * switch, but a remote processor has already + * determined that a higher-priority task became + * eligible after the task was picked. */ +} sched_state_t; + +static inline sched_state_t get_sched_state_on(int cpu) +{ + return atomic_read(&per_cpu(resched_state, cpu)); +} + +static inline sched_state_t get_sched_state(void) +{ + return atomic_read(&__get_cpu_var(resched_state)); +} + +static inline int is_in_sched_state(int possible_states) +{ + return get_sched_state() & possible_states; +} + +static inline int cpu_is_in_sched_state(int cpu, int possible_states) +{ + return get_sched_state_on(cpu) & possible_states; +} + +static inline void set_sched_state(sched_state_t s) +{ + TRACE_SCHED_STATE_CHANGE(get_sched_state(), s, smp_processor_id()); + atomic_set(&__get_cpu_var(resched_state), s); +} + +static inline int sched_state_transition(sched_state_t from, sched_state_t to) +{ + sched_state_t old_state; + + old_state = atomic_cmpxchg(&__get_cpu_var(resched_state), from, to); + if (old_state == from) { + TRACE_SCHED_STATE_CHANGE(from, to, smp_processor_id()); + return 1; + } else + return 0; +} + +static inline int sched_state_transition_on(int cpu, + sched_state_t from, + sched_state_t to) +{ + sched_state_t old_state; + + old_state = atomic_cmpxchg(&per_cpu(resched_state, cpu), from, to); + if (old_state == from) { + TRACE_SCHED_STATE_CHANGE(from, to, cpu); + return 1; + } else + return 0; +} + +/* Plugins must call this function after they have decided which job to + * schedule next. IMPORTANT: this function must be called while still holding + * the lock that is used to serialize scheduling decisions. + * + * (Ideally, we would like to use runqueue locks for this purpose, but that + * would lead to deadlocks with the migration code.) + */ +static inline void sched_state_task_picked(void) +{ + VERIFY_SCHED_STATE(WILL_SCHEDULE); + + /* WILL_SCHEDULE has only a local tansition => simple store is ok */ + set_sched_state(TASK_PICKED); +} + +static inline void sched_state_entered_schedule(void) +{ + /* Update state for the case that we entered schedule() not due to + * set_tsk_need_resched() */ + set_sched_state(WILL_SCHEDULE); +} + +/* Called by schedule() to check if the scheduling decision is still valid + * after a context switch. Returns 1 if the CPU needs to reschdule. */ +static inline int sched_state_validate_switch(void) +{ + int left_state_ok = 0; + + VERIFY_SCHED_STATE(PICKED_WRONG_TASK | TASK_PICKED); + + if (is_in_sched_state(TASK_PICKED)) { + /* Might be good; let's try to transition out of this + * state. This must be done atomically since remote processors + * may try to change the state, too. */ + left_state_ok = sched_state_transition(TASK_PICKED, TASK_SCHEDULED); + } + + if (!left_state_ok) { + /* We raced with a higher-priority task arrival => not + * valid. The CPU needs to reschedule. */ + set_sched_state(WILL_SCHEDULE); + return 1; + } else + return 0; +} + +/* State transition events. See litmus/preempt.c for details. */ +void sched_state_will_schedule(struct task_struct* tsk); +void sched_state_ipi(void); +/* Cause a CPU (remote or local) to reschedule. */ +void litmus_reschedule(int cpu); +void litmus_reschedule_local(void); + +#ifdef CONFIG_DEBUG_KERNEL +void sched_state_plugin_check(void); +#else +#define sched_state_plugin_check() /* no check */ +#endif + +#endif diff --git a/include/litmus/rsm_lock.h b/include/litmus/rsm_lock.h new file mode 100644 index 0000000..a151896 --- /dev/null +++ b/include/litmus/rsm_lock.h @@ -0,0 +1,54 @@ +#ifndef LITMUS_RSM_H +#define LITMUS_RSM_H + +#include +#include +#include + +/* struct for semaphore with priority inheritance */ +struct rsm_mutex { + struct litmus_lock litmus_lock; + + /* current resource holder */ + struct task_struct *owner; + + /* highest-priority waiter */ + struct task_struct *hp_waiter; + + /* FIFO queue of waiting tasks -- for now. time stamp in the future. */ + wait_queue_head_t wait; + + /* we do some nesting within spinlocks, so we can't use the normal + sleeplocks found in wait_queue_head_t. */ + raw_spinlock_t lock; +}; + +static inline struct rsm_mutex* rsm_mutex_from_lock(struct litmus_lock* lock) +{ + return container_of(lock, struct rsm_mutex, litmus_lock); +} + +#ifdef CONFIG_LITMUS_DGL_SUPPORT +int rsm_mutex_is_owner(struct litmus_lock *l, struct task_struct *t); +int rsm_mutex_dgl_lock(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, wait_queue_t* wq_node); +void rsm_mutex_enable_priority(struct litmus_lock *l, dgl_wait_state_t* dgl_wait); +#endif + +void rsm_mutex_propagate_increase_inheritance(struct litmus_lock* l, + struct task_struct* t, + raw_spinlock_t* to_unlock, + unsigned long irqflags); + +void rsm_mutex_propagate_decrease_inheritance(struct litmus_lock* l, + struct task_struct* t, + raw_spinlock_t* to_unlock, + unsigned long irqflags); + +int rsm_mutex_lock(struct litmus_lock* l); +int rsm_mutex_unlock(struct litmus_lock* l); +int rsm_mutex_close(struct litmus_lock* l); +void rsm_mutex_free(struct litmus_lock* l); +struct litmus_lock* rsm_mutex_new(struct litmus_lock_ops*); + + +#endif \ No newline at end of file diff --git a/include/litmus/rt_domain.h b/include/litmus/rt_domain.h new file mode 100644 index 0000000..ac24929 --- /dev/null +++ b/include/litmus/rt_domain.h @@ -0,0 +1,182 @@ +/* CLEANUP: Add comments and make it less messy. + * + */ + +#ifndef __UNC_RT_DOMAIN_H__ +#define __UNC_RT_DOMAIN_H__ + +#include + +#define RELEASE_QUEUE_SLOTS 127 /* prime */ + +struct _rt_domain; + +typedef int (*check_resched_needed_t)(struct _rt_domain *rt); +typedef void (*release_jobs_t)(struct _rt_domain *rt, struct bheap* tasks); + +struct release_queue { + /* each slot maintains a list of release heaps sorted + * by release time */ + struct list_head slot[RELEASE_QUEUE_SLOTS]; +}; + +typedef struct _rt_domain { + /* runnable rt tasks are in here */ + raw_spinlock_t ready_lock; + struct bheap ready_queue; + + /* real-time tasks waiting for release are in here */ + raw_spinlock_t release_lock; + struct release_queue release_queue; + +#ifdef CONFIG_RELEASE_MASTER + int release_master; +#endif + + /* for moving tasks to the release queue */ + raw_spinlock_t tobe_lock; + struct list_head tobe_released; + + /* how do we check if we need to kick another CPU? */ + check_resched_needed_t check_resched; + + /* how do we release jobs? */ + release_jobs_t release_jobs; + + /* how are tasks ordered in the ready queue? */ + bheap_prio_t order; +} rt_domain_t; + +struct release_heap { + /* list_head for per-time-slot list */ + struct list_head list; + lt_t release_time; + /* all tasks to be released at release_time */ + struct bheap heap; + /* used to trigger the release */ + struct hrtimer timer; + +#ifdef CONFIG_RELEASE_MASTER + /* used to delegate releases */ + struct hrtimer_start_on_info info; +#endif + /* required for the timer callback */ + rt_domain_t* dom; +}; + + +static inline struct task_struct* __next_ready(rt_domain_t* rt) +{ + struct bheap_node *hn = bheap_peek(rt->order, &rt->ready_queue); + if (hn) + return bheap2task(hn); + else + return NULL; +} + +void rt_domain_init(rt_domain_t *rt, bheap_prio_t order, + check_resched_needed_t check, + release_jobs_t relase); + +void __add_ready(rt_domain_t* rt, struct task_struct *new); +void __merge_ready(rt_domain_t* rt, struct bheap *tasks); +void __add_release(rt_domain_t* rt, struct task_struct *task); + +static inline struct task_struct* __take_ready(rt_domain_t* rt) +{ + struct bheap_node* hn = bheap_take(rt->order, &rt->ready_queue); + if (hn) + return bheap2task(hn); + else + return NULL; +} + +static inline struct task_struct* __peek_ready(rt_domain_t* rt) +{ + struct bheap_node* hn = bheap_peek(rt->order, &rt->ready_queue); + if (hn) + return bheap2task(hn); + else + return NULL; +} + +static inline int is_queued(struct task_struct *t) +{ + BUG_ON(!tsk_rt(t)->heap_node); + return bheap_node_in_heap(tsk_rt(t)->heap_node); +} + +static inline void remove(rt_domain_t* rt, struct task_struct *t) +{ + bheap_delete(rt->order, &rt->ready_queue, tsk_rt(t)->heap_node); +} + +static inline void add_ready(rt_domain_t* rt, struct task_struct *new) +{ + unsigned long flags; + /* first we need the write lock for rt_ready_queue */ + raw_spin_lock_irqsave(&rt->ready_lock, flags); + __add_ready(rt, new); + raw_spin_unlock_irqrestore(&rt->ready_lock, flags); +} + +static inline void merge_ready(rt_domain_t* rt, struct bheap* tasks) +{ + unsigned long flags; + raw_spin_lock_irqsave(&rt->ready_lock, flags); + __merge_ready(rt, tasks); + raw_spin_unlock_irqrestore(&rt->ready_lock, flags); +} + +static inline struct task_struct* take_ready(rt_domain_t* rt) +{ + unsigned long flags; + struct task_struct* ret; + /* first we need the write lock for rt_ready_queue */ + raw_spin_lock_irqsave(&rt->ready_lock, flags); + ret = __take_ready(rt); + raw_spin_unlock_irqrestore(&rt->ready_lock, flags); + return ret; +} + + +static inline void add_release(rt_domain_t* rt, struct task_struct *task) +{ + unsigned long flags; + raw_spin_lock_irqsave(&rt->tobe_lock, flags); + __add_release(rt, task); + raw_spin_unlock_irqrestore(&rt->tobe_lock, flags); +} + +#ifdef CONFIG_RELEASE_MASTER +void __add_release_on(rt_domain_t* rt, struct task_struct *task, + int target_cpu); + +static inline void add_release_on(rt_domain_t* rt, + struct task_struct *task, + int target_cpu) +{ + unsigned long flags; + raw_spin_lock_irqsave(&rt->tobe_lock, flags); + __add_release_on(rt, task, target_cpu); + raw_spin_unlock_irqrestore(&rt->tobe_lock, flags); +} +#endif + +static inline int __jobs_pending(rt_domain_t* rt) +{ + return !bheap_empty(&rt->ready_queue); +} + +static inline int jobs_pending(rt_domain_t* rt) +{ + unsigned long flags; + int ret; + /* first we need the write lock for rt_ready_queue */ + raw_spin_lock_irqsave(&rt->ready_lock, flags); + ret = !bheap_empty(&rt->ready_queue); + raw_spin_unlock_irqrestore(&rt->ready_lock, flags); + return ret; +} + +#endif diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h new file mode 100644 index 0000000..0198884 --- /dev/null +++ b/include/litmus/rt_param.h @@ -0,0 +1,307 @@ +/* + * Definition of the scheduler plugin interface. + * + */ +#ifndef _LINUX_RT_PARAM_H_ +#define _LINUX_RT_PARAM_H_ + +#include + +/* Litmus time type. */ +typedef unsigned long long lt_t; + +static inline int lt_after(lt_t a, lt_t b) +{ + return ((long long) b) - ((long long) a) < 0; +} +#define lt_before(a, b) lt_after(b, a) + +static inline int lt_after_eq(lt_t a, lt_t b) +{ + return ((long long) a) - ((long long) b) >= 0; +} +#define lt_before_eq(a, b) lt_after_eq(b, a) + +/* different types of clients */ +typedef enum { + RT_CLASS_HARD, + RT_CLASS_SOFT, + RT_CLASS_SOFT_W_SLIP, + RT_CLASS_BEST_EFFORT +} task_class_t; + +typedef enum { + NO_ENFORCEMENT, /* job may overrun unhindered */ + QUANTUM_ENFORCEMENT, /* budgets are only checked on quantum boundaries */ + PRECISE_ENFORCEMENT /* budgets are enforced with hrtimers */ +} budget_policy_t; + +struct rt_task { + lt_t exec_cost; + lt_t period; + lt_t phase; + unsigned int cpu; + task_class_t cls; + budget_policy_t budget_policy; /* ignored by pfair */ +}; + +union np_flag { + uint32_t raw; + struct { + /* Is the task currently in a non-preemptive section? */ + uint32_t flag:31; + /* Should the task call into the scheduler? */ + uint32_t preempt:1; + } np; +}; + +struct affinity_observer_args +{ + int lock_od; +}; + +struct gpu_affinity_observer_args +{ + struct affinity_observer_args obs; + int replica_to_gpu_offset; + int nr_simult_users; + int relaxed_rules; +}; + +/* The definition of the data that is shared between the kernel and real-time + * tasks via a shared page (see litmus/ctrldev.c). + * + * WARNING: User space can write to this, so don't trust + * the correctness of the fields! + * + * This servees two purposes: to enable efficient signaling + * of non-preemptive sections (user->kernel) and + * delayed preemptions (kernel->user), and to export + * some real-time relevant statistics such as preemption and + * migration data to user space. We can't use a device to export + * statistics because we want to avoid system call overhead when + * determining preemption/migration overheads). + */ +struct control_page { + volatile union np_flag sched; + + /* to be extended */ +}; + +/* don't export internal data structures to user space (liblitmus) */ +#ifdef __KERNEL__ + +#include +#include + +struct _rt_domain; +struct bheap_node; +struct release_heap; + +struct rt_job { + /* Time instant the the job was or will be released. */ + lt_t release; + /* What is the current deadline? */ + lt_t deadline; + + /* How much service has this job received so far? */ + lt_t exec_time; + + /* Which job is this. This is used to let user space + * specify which job to wait for, which is important if jobs + * overrun. If we just call sys_sleep_next_period() then we + * will unintentionally miss jobs after an overrun. + * + * Increase this sequence number when a job is released. + */ + unsigned int job_no; +}; + +struct pfair_param; + +enum klitirqd_sem_status +{ + NEED_TO_REACQUIRE, + REACQUIRING, + NOT_HELD, + HELD +}; + +typedef enum gpu_migration_dist +{ + // TODO: Make this variable against NR_NVIDIA_GPUS + MIG_LOCAL = 0, + MIG_NEAR = 1, + MIG_MED = 2, + MIG_FAR = 3, // 8 GPUs in a binary tree hierarchy + MIG_NONE = 4, + + MIG_LAST = MIG_NONE +} gpu_migration_dist_t; + +typedef struct feedback_est{ + fp_t est; + fp_t accum_err; +} feedback_est_t; + +/* RT task parameters for scheduling extensions + * These parameters are inherited during clone and therefore must + * be explicitly set up before the task set is launched. + */ +struct rt_param { + /* is the task sleeping? */ + unsigned int flags:8; + + /* do we need to check for srp blocking? */ + unsigned int srp_non_recurse:1; + + /* is the task present? (true if it can be scheduled) */ + unsigned int present:1; + +#ifdef CONFIG_LITMUS_SOFTIRQD + /* proxy threads have minimum priority by default */ + unsigned int is_proxy_thread:1; + + /* pointer to klitirqd currently working on this + task_struct's behalf. only set by the task pointed + to by klitirqd. + + ptr only valid if is_proxy_thread == 0 + */ + struct task_struct* cur_klitirqd; + + /* Used to implement mutual execution exclusion between + * job and klitirqd execution. Job must always hold + * it's klitirqd_sem to execute. klitirqd instance + * must hold the semaphore before executing on behalf + * of a job. + */ + struct mutex klitirqd_sem; + + /* status of held klitirqd_sem, even if the held klitirqd_sem is from + another task (only proxy threads do this though). + */ + atomic_t klitirqd_sem_stat; +#endif + +#ifdef CONFIG_LITMUS_NVIDIA + /* number of top-half interrupts handled on behalf of current job */ + atomic_t nv_int_count; + long unsigned int held_gpus; // bitmap of held GPUs. + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + fp_t gpu_fb_param_a[MIG_LAST+1]; + fp_t gpu_fb_param_b[MIG_LAST+1]; + + gpu_migration_dist_t gpu_migration; + int last_gpu; + feedback_est_t gpu_migration_est[MIG_LAST+1]; // local, near, med, far + + lt_t accum_gpu_time; + lt_t gpu_time_stamp; + + unsigned int suspend_gpu_tracker_on_block:1; +#endif +#endif + +#ifdef CONFIG_LITMUS_LOCKING + /* Is the task being priority-boosted by a locking protocol? */ + unsigned int priority_boosted:1; + /* If so, when did this start? */ + lt_t boost_start_time; +#endif + + /* user controlled parameters */ + struct rt_task task_params; + + /* timing parameters */ + struct rt_job job_params; + + /* task representing the current "inherited" task + * priority, assigned by inherit_priority and + * return priority in the scheduler plugins. + * could point to self if PI does not result in + * an increased task priority. + */ + struct task_struct* inh_task; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + raw_spinlock_t hp_blocked_tasks_lock; + struct binheap_handle hp_blocked_tasks; + + /* pointer to lock upon which is currently blocked */ + struct litmus_lock* blocked_lock; +#endif + +#ifdef CONFIG_NP_SECTION + /* For the FMLP under PSN-EDF, it is required to make the task + * non-preemptive from kernel space. In order not to interfere with + * user space, this counter indicates the kernel space np setting. + * kernel_np > 0 => task is non-preemptive + */ + unsigned int kernel_np; +#endif + + /* This field can be used by plugins to store where the task + * is currently scheduled. It is the responsibility of the + * plugin to avoid race conditions. + * + * This used by GSN-EDF and PFAIR. + */ + volatile int scheduled_on; + + /* Is the stack of the task currently in use? This is updated by + * the LITMUS core. + * + * Be careful to avoid deadlocks! + */ + volatile int stack_in_use; + + /* This field can be used by plugins to store where the task + * is currently linked. It is the responsibility of the plugin + * to avoid race conditions. + * + * Used by GSN-EDF. + */ + volatile int linked_on; + + /* PFAIR/PD^2 state. Allocated on demand. */ + struct pfair_param* pfair; + + /* Fields saved before BE->RT transition. + */ + int old_policy; + int old_prio; + + /* ready queue for this task */ + struct _rt_domain* domain; + + /* heap element for this task + * + * Warning: Don't statically allocate this node. The heap + * implementation swaps these between tasks, thus after + * dequeuing from a heap you may end up with a different node + * then the one you had when enqueuing the task. For the same + * reason, don't obtain and store references to this node + * other than this pointer (which is updated by the heap + * implementation). + */ + struct bheap_node* heap_node; + struct release_heap* rel_heap; + + /* Used by rt_domain to queue task in release list. + */ + struct list_head list; + + /* Pointer to the page shared between userspace and kernel. */ + struct control_page * ctrl_page; +}; + +/* Possible RT flags */ +#define RT_F_RUNNING 0x00000000 +#define RT_F_SLEEP 0x00000001 +#define RT_F_EXIT_SEM 0x00000008 + +#endif + +#endif diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h new file mode 100644 index 0000000..24a6858 --- /dev/null +++ b/include/litmus/sched_plugin.h @@ -0,0 +1,183 @@ +/* + * Definition of the scheduler plugin interface. + * + */ +#ifndef _LINUX_SCHED_PLUGIN_H_ +#define _LINUX_SCHED_PLUGIN_H_ + +#include + +#ifdef CONFIG_LITMUS_LOCKING +#include +#endif + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING +#include +#endif + +#include + +/************************ setup/tear down ********************/ + +typedef long (*activate_plugin_t) (void); +typedef long (*deactivate_plugin_t) (void); + + + +/********************* scheduler invocation ******************/ + +/* Plugin-specific realtime tick handler */ +typedef void (*scheduler_tick_t) (struct task_struct *cur); +/* Novell make sched decision function */ +typedef struct task_struct* (*schedule_t)(struct task_struct * prev); +/* Clean up after the task switch has occured. + * This function is called after every (even non-rt) task switch. + */ +typedef void (*finish_switch_t)(struct task_struct *prev); + +/********************* task state changes ********************/ + +/* Called to setup a new real-time task. + * Release the first job, enqueue, etc. + * Task may already be running. + */ +typedef void (*task_new_t) (struct task_struct *task, + int on_rq, + int running); + +/* Called to re-introduce a task after blocking. + * Can potentially be called multiple times. + */ +typedef void (*task_wake_up_t) (struct task_struct *task); +/* called to notify the plugin of a blocking real-time task + * it will only be called for real-time tasks and before schedule is called */ +typedef void (*task_block_t) (struct task_struct *task); +/* Called when a real-time task exits or changes to a different scheduling + * class. + * Free any allocated resources + */ +typedef void (*task_exit_t) (struct task_struct *); + +/* Called when the current task attempts to create a new lock of a given + * protocol type. */ +typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type, + void* __user config); + +struct affinity_observer; +typedef long (*allocate_affinity_observer_t) ( + struct affinity_observer **aff_obs, int type, + void* __user config); + +typedef void (*increase_prio_t)(struct task_struct* t, struct task_struct* prio_inh); +typedef void (*decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh); +typedef void (*nested_increase_prio_t)(struct task_struct* t, struct task_struct* prio_inh, + raw_spinlock_t *to_unlock, unsigned long irqflags); +typedef void (*nested_decrease_prio_t)(struct task_struct* t, struct task_struct* prio_inh, + raw_spinlock_t *to_unlock, unsigned long irqflags); + +typedef void (*increase_prio_klitirq_t)(struct task_struct* klitirqd, + struct task_struct* old_owner, + struct task_struct* new_owner); +typedef void (*decrease_prio_klitirqd_t)(struct task_struct* klitirqd, + struct task_struct* old_owner); + + +typedef int (*enqueue_pai_tasklet_t)(struct tasklet_struct* tasklet); +typedef void (*change_prio_pai_tasklet_t)(struct task_struct *old_prio, + struct task_struct *new_prio); +typedef void (*run_tasklets_t)(struct task_struct* next); + +typedef raw_spinlock_t* (*get_dgl_spinlock_t) (struct task_struct *t); + + +typedef int (*higher_prio_t)(struct task_struct* a, struct task_struct* b); + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + +typedef enum +{ + BASE, + EFFECTIVE +} comparison_mode_t; + +typedef int (*__higher_prio_t)(struct task_struct* a, comparison_mode_t a_mod, + struct task_struct* b, comparison_mode_t b_mod); +#endif + + +/********************* sys call backends ********************/ +/* This function causes the caller to sleep until the next release */ +typedef long (*complete_job_t) (void); + +typedef long (*admit_task_t)(struct task_struct* tsk); + +typedef void (*release_at_t)(struct task_struct *t, lt_t start); + +struct sched_plugin { + struct list_head list; + /* basic info */ + char *plugin_name; + + /* setup */ + activate_plugin_t activate_plugin; + deactivate_plugin_t deactivate_plugin; + + /* scheduler invocation */ + scheduler_tick_t tick; + schedule_t schedule; + finish_switch_t finish_switch; + + /* syscall backend */ + complete_job_t complete_job; + release_at_t release_at; + + /* task state changes */ + admit_task_t admit_task; + + task_new_t task_new; + task_wake_up_t task_wake_up; + task_block_t task_block; + task_exit_t task_exit; + + higher_prio_t compare; + +#ifdef CONFIG_LITMUS_LOCKING + /* locking protocols */ + allocate_lock_t allocate_lock; + increase_prio_t increase_prio; + decrease_prio_t decrease_prio; +#endif +#ifdef CONFIG_LITMUS_NESTED_LOCKING + nested_increase_prio_t nested_increase_prio; + nested_decrease_prio_t nested_decrease_prio; + __higher_prio_t __compare; +#endif +#ifdef CONFIG_LITMUS_DGL_SUPPORT + get_dgl_spinlock_t get_dgl_spinlock; +#endif + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + allocate_affinity_observer_t allocate_aff_obs; +#endif + +#ifdef CONFIG_LITMUS_SOFTIRQD + increase_prio_klitirq_t increase_prio_klitirqd; + decrease_prio_klitirqd_t decrease_prio_klitirqd; +#endif +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + enqueue_pai_tasklet_t enqueue_pai_tasklet; + change_prio_pai_tasklet_t change_prio_pai_tasklet; + run_tasklets_t run_tasklets; +#endif +} __attribute__ ((__aligned__(SMP_CACHE_BYTES))); + + +extern struct sched_plugin *litmus; + +int register_sched_plugin(struct sched_plugin* plugin); +struct sched_plugin* find_sched_plugin(const char* name); +int print_sched_plugins(char* buf, int max); + +extern struct sched_plugin linux_sched_plugin; + +#endif diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h new file mode 100644 index 0000000..b1b71f6 --- /dev/null +++ b/include/litmus/sched_trace.h @@ -0,0 +1,380 @@ +/* + * sched_trace.h -- record scheduler events to a byte stream for offline analysis. + */ +#ifndef _LINUX_SCHED_TRACE_H_ +#define _LINUX_SCHED_TRACE_H_ + +/* all times in nanoseconds */ + +struct st_trace_header { + u8 type; /* Of what type is this record? */ + u8 cpu; /* On which CPU was it recorded? */ + u16 pid; /* PID of the task. */ + u32 job:24; /* The job sequence number. */ + u8 extra; +} __attribute__((packed)); + +#define ST_NAME_LEN 16 +struct st_name_data { + char cmd[ST_NAME_LEN];/* The name of the executable of this process. */ +} __attribute__((packed)); + +struct st_param_data { /* regular params */ + u32 wcet; + u32 period; + u32 phase; + u8 partition; + u8 class; + u8 __unused[2]; +} __attribute__((packed)); + +struct st_release_data { /* A job is was/is going to be released. */ + u64 release; /* What's the release time? */ + u64 deadline; /* By when must it finish? */ +} __attribute__((packed)); + +struct st_assigned_data { /* A job was asigned to a CPU. */ + u64 when; + u8 target; /* Where should it execute? */ + u8 __unused[7]; +} __attribute__((packed)); + +struct st_switch_to_data { /* A process was switched to on a given CPU. */ + u64 when; /* When did this occur? */ + u32 exec_time; /* Time the current job has executed. */ + u8 __unused[4]; +} __attribute__((packed)); + +struct st_switch_away_data { /* A process was switched away from on a given CPU. */ + u64 when; + u64 exec_time; +} __attribute__((packed)); + +struct st_completion_data { /* A job completed. */ + u64 when; + u8 forced:1; /* Set to 1 if job overran and kernel advanced to the + * next task automatically; set to 0 otherwise. + */ + u8 __uflags:7; + u16 nv_int_count; + u8 __unused[5]; +} __attribute__((packed)); + +struct st_block_data { /* A task blocks. */ + u64 when; + u64 __unused; +} __attribute__((packed)); + +struct st_resume_data { /* A task resumes. */ + u64 when; + u64 __unused; +} __attribute__((packed)); + +struct st_action_data { + u64 when; + u8 action; + u8 __unused[7]; +} __attribute__((packed)); + +struct st_sys_release_data { + u64 when; + u64 release; +} __attribute__((packed)); + + +struct st_tasklet_release_data { + u64 when; + u64 __unused; +} __attribute__((packed)); + +struct st_tasklet_begin_data { + u64 when; + u16 exe_pid; + u8 __unused[6]; +} __attribute__((packed)); + +struct st_tasklet_end_data { + u64 when; + u16 exe_pid; + u8 flushed; + u8 __unused[5]; +} __attribute__((packed)); + + +struct st_work_release_data { + u64 when; + u64 __unused; +} __attribute__((packed)); + +struct st_work_begin_data { + u64 when; + u16 exe_pid; + u8 __unused[6]; +} __attribute__((packed)); + +struct st_work_end_data { + u64 when; + u16 exe_pid; + u8 flushed; + u8 __unused[5]; +} __attribute__((packed)); + +struct st_effective_priority_change_data { + u64 when; + u16 inh_pid; + u8 __unused[6]; +} __attribute__((packed)); + +struct st_nv_interrupt_begin_data { + u64 when; + u32 device; + u32 serialNumber; +} __attribute__((packed)); + +struct st_nv_interrupt_end_data { + u64 when; + u32 device; + u32 serialNumber; +} __attribute__((packed)); + +struct st_prediction_err_data { + u64 distance; + u64 rel_err; +} __attribute__((packed)); + +struct st_migration_data { + u64 observed; + u64 estimated; +} __attribute__((packed)); + +struct migration_info { + u64 observed; + u64 estimated; + u8 distance; +} __attribute__((packed)); + +#define DATA(x) struct st_ ## x ## _data x; + +typedef enum { + ST_NAME = 1, /* Start at one, so that we can spot + * uninitialized records. */ + ST_PARAM, + ST_RELEASE, + ST_ASSIGNED, + ST_SWITCH_TO, + ST_SWITCH_AWAY, + ST_COMPLETION, + ST_BLOCK, + ST_RESUME, + ST_ACTION, + ST_SYS_RELEASE, + ST_TASKLET_RELEASE, + ST_TASKLET_BEGIN, + ST_TASKLET_END, + ST_WORK_RELEASE, + ST_WORK_BEGIN, + ST_WORK_END, + ST_EFF_PRIO_CHANGE, + ST_NV_INTERRUPT_BEGIN, + ST_NV_INTERRUPT_END, + + ST_PREDICTION_ERR, + ST_MIGRATION, +} st_event_record_type_t; + +struct st_event_record { + struct st_trace_header hdr; + union { + u64 raw[2]; + + DATA(name); + DATA(param); + DATA(release); + DATA(assigned); + DATA(switch_to); + DATA(switch_away); + DATA(completion); + DATA(block); + DATA(resume); + DATA(action); + DATA(sys_release); + DATA(tasklet_release); + DATA(tasklet_begin); + DATA(tasklet_end); + DATA(work_release); + DATA(work_begin); + DATA(work_end); + DATA(effective_priority_change); + DATA(nv_interrupt_begin); + DATA(nv_interrupt_end); + + DATA(prediction_err); + DATA(migration); + } data; +} __attribute__((packed)); + +#undef DATA + +#ifdef __KERNEL__ + +#include +#include + +#ifdef CONFIG_SCHED_TASK_TRACE + +#define SCHED_TRACE(id, callback, task) \ + ft_event1(id, callback, task) +#define SCHED_TRACE2(id, callback, task, xtra) \ + ft_event2(id, callback, task, xtra) +#define SCHED_TRACE3(id, callback, task, xtra1, xtra2) \ + ft_event3(id, callback, task, xtra1, xtra2) + +/* provide prototypes; needed on sparc64 */ +#ifndef NO_TASK_TRACE_DECLS +feather_callback void do_sched_trace_task_name(unsigned long id, + struct task_struct* task); +feather_callback void do_sched_trace_task_param(unsigned long id, + struct task_struct* task); +feather_callback void do_sched_trace_task_release(unsigned long id, + struct task_struct* task); +feather_callback void do_sched_trace_task_switch_to(unsigned long id, + struct task_struct* task); +feather_callback void do_sched_trace_task_switch_away(unsigned long id, + struct task_struct* task); +feather_callback void do_sched_trace_task_completion(unsigned long id, + struct task_struct* task, + unsigned long forced); +feather_callback void do_sched_trace_task_block(unsigned long id, + struct task_struct* task); +feather_callback void do_sched_trace_task_resume(unsigned long id, + struct task_struct* task); +feather_callback void do_sched_trace_action(unsigned long id, + struct task_struct* task, + unsigned long action); +feather_callback void do_sched_trace_sys_release(unsigned long id, + lt_t* start); + + +feather_callback void do_sched_trace_tasklet_release(unsigned long id, + struct task_struct* owner); +feather_callback void do_sched_trace_tasklet_begin(unsigned long id, + struct task_struct* owner); +feather_callback void do_sched_trace_tasklet_end(unsigned long id, + struct task_struct* owner, + unsigned long flushed); + +feather_callback void do_sched_trace_work_release(unsigned long id, + struct task_struct* owner); +feather_callback void do_sched_trace_work_begin(unsigned long id, + struct task_struct* owner, + struct task_struct* exe); +feather_callback void do_sched_trace_work_end(unsigned long id, + struct task_struct* owner, + struct task_struct* exe, + unsigned long flushed); + +feather_callback void do_sched_trace_eff_prio_change(unsigned long id, + struct task_struct* task, + struct task_struct* inh); + +feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id, + u32 device); +feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id, + unsigned long unused); + +feather_callback void do_sched_trace_prediction_err(unsigned long id, + struct task_struct* task, + gpu_migration_dist_t* distance, + fp_t* rel_err); + + + + + +feather_callback void do_sched_trace_migration(unsigned long id, + struct task_struct* task, + struct migration_info* mig_info); + + +/* returns true if we're tracing an interrupt on current CPU */ +/* int is_interrupt_tracing_active(void); */ + +#endif + +#else + +#define SCHED_TRACE(id, callback, task) /* no tracing */ +#define SCHED_TRACE2(id, callback, task, xtra) /* no tracing */ +#define SCHED_TRACE3(id, callback, task, xtra1, xtra2) + +#endif + + +#define SCHED_TRACE_BASE_ID 500 + + +#define sched_trace_task_name(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 1, do_sched_trace_task_name, t) +#define sched_trace_task_param(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 2, do_sched_trace_task_param, t) +#define sched_trace_task_release(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 3, do_sched_trace_task_release, t) +#define sched_trace_task_switch_to(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 4, do_sched_trace_task_switch_to, t) +#define sched_trace_task_switch_away(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 5, do_sched_trace_task_switch_away, t) +#define sched_trace_task_completion(t, forced) \ + SCHED_TRACE2(SCHED_TRACE_BASE_ID + 6, do_sched_trace_task_completion, t, \ + (unsigned long) forced) +#define sched_trace_task_block(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 7, do_sched_trace_task_block, t) +#define sched_trace_task_resume(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 8, do_sched_trace_task_resume, t) +#define sched_trace_action(t, action) \ + SCHED_TRACE2(SCHED_TRACE_BASE_ID + 9, do_sched_trace_action, t, \ + (unsigned long) action); +/* when is a pointer, it does not need an explicit cast to unsigned long */ +#define sched_trace_sys_release(when) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 10, do_sched_trace_sys_release, when) + + +#define sched_trace_tasklet_release(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 11, do_sched_trace_tasklet_release, t) + +#define sched_trace_tasklet_begin(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, do_sched_trace_tasklet_begin, t) + +#define sched_trace_tasklet_end(t, flushed) \ + SCHED_TRACE2(SCHED_TRACE_BASE_ID + 13, do_sched_trace_tasklet_end, t, flushed) + + +#define sched_trace_work_release(t) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 14, do_sched_trace_work_release, t) + +#define sched_trace_work_begin(t, e) \ + SCHED_TRACE2(SCHED_TRACE_BASE_ID + 15, do_sched_trace_work_begin, t, e) + +#define sched_trace_work_end(t, e, flushed) \ + SCHED_TRACE3(SCHED_TRACE_BASE_ID + 16, do_sched_trace_work_end, t, e, flushed) + + +#define sched_trace_eff_prio_change(t, inh) \ + SCHED_TRACE2(SCHED_TRACE_BASE_ID + 17, do_sched_trace_eff_prio_change, t, inh) + + +#define sched_trace_nv_interrupt_begin(d) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 18, do_sched_trace_nv_interrupt_begin, d) +#define sched_trace_nv_interrupt_end(d) \ + SCHED_TRACE(SCHED_TRACE_BASE_ID + 19, do_sched_trace_nv_interrupt_end, d) + +#define sched_trace_prediction_err(t, dist, rel_err) \ + SCHED_TRACE3(SCHED_TRACE_BASE_ID + 20, do_sched_trace_prediction_err, t, dist, rel_err) + +#define sched_trace_migration(t, mig_info) \ + SCHED_TRACE2(SCHED_TRACE_BASE_ID + 21, do_sched_trace_migration, t, mig_info) + +#define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */ + +#endif /* __KERNEL__ */ + +#endif diff --git a/include/litmus/sched_trace_external.h b/include/litmus/sched_trace_external.h new file mode 100644 index 0000000..e70e45e --- /dev/null +++ b/include/litmus/sched_trace_external.h @@ -0,0 +1,78 @@ +/* + * sched_trace.h -- record scheduler events to a byte stream for offline analysis. + */ +#ifndef _LINUX_SCHED_TRACE_EXTERNAL_H_ +#define _LINUX_SCHED_TRACE_EXTERNAL_H_ + + +#ifdef CONFIG_SCHED_TASK_TRACE +extern void __sched_trace_tasklet_begin_external(struct task_struct* t); +static inline void sched_trace_tasklet_begin_external(struct task_struct* t) +{ + __sched_trace_tasklet_begin_external(t); +} + +extern void __sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed); +static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed) +{ + __sched_trace_tasklet_end_external(t, flushed); +} + +extern void __sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e); +static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e) +{ + __sched_trace_work_begin_external(t, e); +} + +extern void __sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f); +static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f) +{ + __sched_trace_work_end_external(t, e, f); +} + +#ifdef CONFIG_LITMUS_NVIDIA +extern void __sched_trace_nv_interrupt_begin_external(u32 device); +static inline void sched_trace_nv_interrupt_begin_external(u32 device) +{ + __sched_trace_nv_interrupt_begin_external(device); +} + +extern void __sched_trace_nv_interrupt_end_external(u32 device); +static inline void sched_trace_nv_interrupt_end_external(u32 device) +{ + __sched_trace_nv_interrupt_end_external(device); +} +#endif + +#else + +// no tracing. +static inline void sched_trace_tasklet_begin_external(struct task_struct* t){} +static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed){} +static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e){} +static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f){} + +#ifdef CONFIG_LITMUS_NVIDIA +static inline void sched_trace_nv_interrupt_begin_external(u32 device){} +static inline void sched_trace_nv_interrupt_end_external(u32 device){} +#endif + +#endif + + +#ifdef CONFIG_LITMUS_NVIDIA + +#define EX_TS(evt) \ +extern void __##evt(void); \ +static inline void EX_##evt(void) { __##evt(); } + +EX_TS(TS_NV_TOPISR_START) +EX_TS(TS_NV_TOPISR_END) +EX_TS(TS_NV_BOTISR_START) +EX_TS(TS_NV_BOTISR_END) +EX_TS(TS_NV_RELEASE_BOTISR_START) +EX_TS(TS_NV_RELEASE_BOTISR_END) + +#endif + +#endif diff --git a/include/litmus/srp.h b/include/litmus/srp.h new file mode 100644 index 0000000..c9a4552 --- /dev/null +++ b/include/litmus/srp.h @@ -0,0 +1,28 @@ +#ifndef LITMUS_SRP_H +#define LITMUS_SRP_H + +struct srp_semaphore; + +struct srp_priority { + struct list_head list; + unsigned int priority; + pid_t pid; +}; +#define list2prio(l) list_entry(l, struct srp_priority, list) + +/* struct for uniprocessor SRP "semaphore" */ +struct srp_semaphore { + struct litmus_lock litmus_lock; + struct srp_priority ceiling; + struct task_struct* owner; + int cpu; /* cpu associated with this "semaphore" and resource */ +}; + +/* map a task to its SRP preemption level priority */ +typedef unsigned int (*srp_prioritization_t)(struct task_struct* t); +/* Must be updated by each plugin that uses SRP.*/ +extern srp_prioritization_t get_srp_prio; + +struct srp_semaphore* allocate_srp_semaphore(void); + +#endif diff --git a/include/litmus/trace.h b/include/litmus/trace.h new file mode 100644 index 0000000..e078aee --- /dev/null +++ b/include/litmus/trace.h @@ -0,0 +1,148 @@ +#ifndef _SYS_TRACE_H_ +#define _SYS_TRACE_H_ + +#ifdef CONFIG_SCHED_OVERHEAD_TRACE + +#include +#include + + +/*********************** TIMESTAMPS ************************/ + +enum task_type_marker { + TSK_BE, + TSK_RT, + TSK_UNKNOWN +}; + +struct timestamp { + uint64_t timestamp; + uint32_t seq_no; + uint8_t cpu; + uint8_t event; + uint8_t task_type:2; + uint8_t irq_flag:1; + uint8_t irq_count:5; +}; + +/* tracing callbacks */ +feather_callback void save_timestamp(unsigned long event); +feather_callback void save_timestamp_def(unsigned long event, unsigned long type); +feather_callback void save_timestamp_task(unsigned long event, unsigned long t_ptr); +feather_callback void save_timestamp_cpu(unsigned long event, unsigned long cpu); +feather_callback void save_task_latency(unsigned long event, unsigned long when_ptr); + +#define TIMESTAMP(id) ft_event0(id, save_timestamp) + +#define DTIMESTAMP(id, def) ft_event1(id, save_timestamp_def, (unsigned long) def) + +#define TTIMESTAMP(id, task) \ + ft_event1(id, save_timestamp_task, (unsigned long) task) + +#define CTIMESTAMP(id, cpu) \ + ft_event1(id, save_timestamp_cpu, (unsigned long) cpu) + +#define LTIMESTAMP(id, task) \ + ft_event1(id, save_task_latency, (unsigned long) task) + +#else /* !CONFIG_SCHED_OVERHEAD_TRACE */ + +#define TIMESTAMP(id) /* no tracing */ + +#define DTIMESTAMP(id, def) /* no tracing */ + +#define TTIMESTAMP(id, task) /* no tracing */ + +#define CTIMESTAMP(id, cpu) /* no tracing */ + +#define LTIMESTAMP(id, when_ptr) /* no tracing */ + +#endif + + +/* Convention for timestamps + * ========================= + * + * In order to process the trace files with a common tool, we use the following + * convention to measure execution times: The end time id of a code segment is + * always the next number after the start time event id. + */ + + + +#define TS_SCHED_START DTIMESTAMP(100, TSK_UNKNOWN) /* we only + * care + * about + * next */ +#define TS_SCHED_END(t) TTIMESTAMP(101, t) +#define TS_SCHED2_START(t) TTIMESTAMP(102, t) +#define TS_SCHED2_END(t) TTIMESTAMP(103, t) + +#define TS_CXS_START(t) TTIMESTAMP(104, t) +#define TS_CXS_END(t) TTIMESTAMP(105, t) + +#define TS_RELEASE_START DTIMESTAMP(106, TSK_RT) +#define TS_RELEASE_END DTIMESTAMP(107, TSK_RT) + +#define TS_TICK_START(t) TTIMESTAMP(110, t) +#define TS_TICK_END(t) TTIMESTAMP(111, t) + + +#define TS_PLUGIN_SCHED_START /* TIMESTAMP(120) */ /* currently unused */ +#define TS_PLUGIN_SCHED_END /* TIMESTAMP(121) */ + +#define TS_PLUGIN_TICK_START /* TIMESTAMP(130) */ +#define TS_PLUGIN_TICK_END /* TIMESTAMP(131) */ + +#define TS_ENTER_NP_START TIMESTAMP(140) +#define TS_ENTER_NP_END TIMESTAMP(141) + +#define TS_EXIT_NP_START TIMESTAMP(150) +#define TS_EXIT_NP_END TIMESTAMP(151) + +#define TS_LOCK_START TIMESTAMP(170) +#define TS_LOCK_SUSPEND TIMESTAMP(171) +#define TS_LOCK_RESUME TIMESTAMP(172) +#define TS_LOCK_END TIMESTAMP(173) + +#ifdef CONFIG_LITMUS_DGL_SUPPORT +#define TS_DGL_LOCK_START TIMESTAMP(175) +#define TS_DGL_LOCK_SUSPEND TIMESTAMP(176) +#define TS_DGL_LOCK_RESUME TIMESTAMP(177) +#define TS_DGL_LOCK_END TIMESTAMP(178) +#endif + +#define TS_UNLOCK_START TIMESTAMP(180) +#define TS_UNLOCK_END TIMESTAMP(181) + +#ifdef CONFIG_LITMUS_DGL_SUPPORT +#define TS_DGL_UNLOCK_START TIMESTAMP(185) +#define TS_DGL_UNLOCK_END TIMESTAMP(186) +#endif + +#define TS_SEND_RESCHED_START(c) CTIMESTAMP(190, c) +#define TS_SEND_RESCHED_END DTIMESTAMP(191, TSK_UNKNOWN) + +#define TS_RELEASE_LATENCY(when) LTIMESTAMP(208, &(when)) + + +#ifdef CONFIG_LITMUS_NVIDIA + +#define TS_NV_TOPISR_START TIMESTAMP(200) +#define TS_NV_TOPISR_END TIMESTAMP(201) + +#define TS_NV_BOTISR_START TIMESTAMP(202) +#define TS_NV_BOTISR_END TIMESTAMP(203) + +#define TS_NV_RELEASE_BOTISR_START TIMESTAMP(204) +#define TS_NV_RELEASE_BOTISR_END TIMESTAMP(205) + +#endif + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD +#define TS_NV_SCHED_BOTISR_START TIMESTAMP(206) +#define TS_NV_SCHED_BOTISR_END TIMESTAMP(207) +#endif + + +#endif /* !_SYS_TRACE_H_ */ diff --git a/include/litmus/trace_irq.h b/include/litmus/trace_irq.h new file mode 100644 index 0000000..f18b127 --- /dev/null +++ b/include/litmus/trace_irq.h @@ -0,0 +1,21 @@ +#ifndef _LITMUS_TRACE_IRQ_H_ +#define _LITMUS_TRACE_IRQ_H_ + +#ifdef CONFIG_SCHED_OVERHEAD_TRACE + +extern DEFINE_PER_CPU(atomic_t, irq_fired_count); + +static inline void ft_irq_fired(void) +{ + /* Only called with preemptions disabled. */ + atomic_inc(&__get_cpu_var(irq_fired_count)); +} + + +#else + +#define ft_irq_fired() /* nothing to do */ + +#endif + +#endif diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h new file mode 100644 index 0000000..4fa514c --- /dev/null +++ b/include/litmus/unistd_32.h @@ -0,0 +1,24 @@ +/* + * included from arch/x86/include/asm/unistd_32.h + * + * LITMUS^RT syscalls with "relative" numbers + */ +#define __LSC(x) (__NR_LITMUS + x) + +#define __NR_set_rt_task_param __LSC(0) +#define __NR_get_rt_task_param __LSC(1) +#define __NR_complete_job __LSC(2) +#define __NR_od_open __LSC(3) +#define __NR_od_close __LSC(4) +#define __NR_litmus_lock __LSC(5) +#define __NR_litmus_unlock __LSC(6) +#define __NR_query_job_no __LSC(7) +#define __NR_wait_for_job_release __LSC(8) +#define __NR_wait_for_ts_release __LSC(9) +#define __NR_release_ts __LSC(10) +#define __NR_null_call __LSC(11) +#define __NR_litmus_dgl_lock __LSC(12) +#define __NR_litmus_dgl_unlock __LSC(13) +#define __NR_register_nv_device __LSC(14) + +#define NR_litmus_syscalls 15 diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h new file mode 100644 index 0000000..f80dc45 --- /dev/null +++ b/include/litmus/unistd_64.h @@ -0,0 +1,40 @@ +/* + * included from arch/x86/include/asm/unistd_64.h + * + * LITMUS^RT syscalls with "relative" numbers + */ +#define __LSC(x) (__NR_LITMUS + x) + +#define __NR_set_rt_task_param __LSC(0) +__SYSCALL(__NR_set_rt_task_param, sys_set_rt_task_param) +#define __NR_get_rt_task_param __LSC(1) +__SYSCALL(__NR_get_rt_task_param, sys_get_rt_task_param) +#define __NR_complete_job __LSC(2) +__SYSCALL(__NR_complete_job, sys_complete_job) +#define __NR_od_open __LSC(3) +__SYSCALL(__NR_od_open, sys_od_open) +#define __NR_od_close __LSC(4) +__SYSCALL(__NR_od_close, sys_od_close) +#define __NR_litmus_lock __LSC(5) +__SYSCALL(__NR_litmus_lock, sys_litmus_lock) +#define __NR_litmus_unlock __LSC(6) +__SYSCALL(__NR_litmus_unlock, sys_litmus_unlock) +#define __NR_query_job_no __LSC(7) +__SYSCALL(__NR_query_job_no, sys_query_job_no) +#define __NR_wait_for_job_release __LSC(8) +__SYSCALL(__NR_wait_for_job_release, sys_wait_for_job_release) +#define __NR_wait_for_ts_release __LSC(9) +__SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release) +#define __NR_release_ts __LSC(10) +__SYSCALL(__NR_release_ts, sys_release_ts) +#define __NR_null_call __LSC(11) +__SYSCALL(__NR_null_call, sys_null_call) +#define __NR_litmus_dgl_lock __LSC(12) +__SYSCALL(__NR_litmus_dgl_lock, sys_litmus_dgl_lock) +#define __NR_litmus_dgl_unlock __LSC(13) +__SYSCALL(__NR_litmus_dgl_unlock, sys_litmus_dgl_unlock) +#define __NR_register_nv_device __LSC(14) +__SYSCALL(__NR_register_nv_device, sys_register_nv_device) + + +#define NR_litmus_syscalls 15 diff --git a/kernel/exit.c b/kernel/exit.c index f2b321b..64879bd 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -57,6 +57,8 @@ #include #include +extern void exit_od_table(struct task_struct *t); + static void exit_mm(struct task_struct * tsk); static void __unhash_process(struct task_struct *p, bool group_dead) @@ -980,6 +982,8 @@ NORET_TYPE void do_exit(long code) if (unlikely(tsk->audit_context)) audit_free(tsk); + exit_od_table(tsk); + tsk->exit_code = code; taskstats_exit(tsk, group_dead); diff --git a/kernel/fork.c b/kernel/fork.c index 0276c30..25c6111 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -77,6 +77,9 @@ #include +#include +#include + /* * Protected counters by write_lock_irq(&tasklist_lock) */ @@ -191,6 +194,7 @@ void __put_task_struct(struct task_struct *tsk) WARN_ON(atomic_read(&tsk->usage)); WARN_ON(tsk == current); + exit_litmus(tsk); exit_creds(tsk); delayacct_tsk_free(tsk); put_signal_struct(tsk->signal); @@ -275,6 +279,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) tsk->stack = ti; + /* Don't let the new task be a real-time task. */ + litmus_fork(tsk); + err = prop_local_init_single(&tsk->dirties); if (err) goto out; diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index a9205e3..11e8969 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -46,6 +46,8 @@ #include #include +#include + #include #include @@ -1026,6 +1028,98 @@ hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) } EXPORT_SYMBOL_GPL(hrtimer_start); +#ifdef CONFIG_ARCH_HAS_SEND_PULL_TIMERS + +/** + * hrtimer_start_on_info_init - Initialize hrtimer_start_on_info + */ +void hrtimer_start_on_info_init(struct hrtimer_start_on_info *info) +{ + memset(info, 0, sizeof(struct hrtimer_start_on_info)); + atomic_set(&info->state, HRTIMER_START_ON_INACTIVE); +} + +/** + * hrtimer_pull - PULL_TIMERS_VECTOR callback on remote cpu + */ +void hrtimer_pull(void) +{ + struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases); + struct hrtimer_start_on_info *info; + struct list_head *pos, *safe, list; + + raw_spin_lock(&base->lock); + list_replace_init(&base->to_pull, &list); + raw_spin_unlock(&base->lock); + + list_for_each_safe(pos, safe, &list) { + info = list_entry(pos, struct hrtimer_start_on_info, list); + TRACE("pulled timer 0x%x\n", info->timer); + list_del(pos); + hrtimer_start(info->timer, info->time, info->mode); + } +} + +/** + * hrtimer_start_on - trigger timer arming on remote cpu + * @cpu: remote cpu + * @info: save timer information for enqueuing on remote cpu + * @timer: timer to be pulled + * @time: expire time + * @mode: timer mode + */ +int hrtimer_start_on(int cpu, struct hrtimer_start_on_info* info, + struct hrtimer *timer, ktime_t time, + const enum hrtimer_mode mode) +{ + unsigned long flags; + struct hrtimer_cpu_base* base; + int in_use = 0, was_empty; + + /* serialize access to info through the timer base */ + lock_hrtimer_base(timer, &flags); + + in_use = (atomic_read(&info->state) != HRTIMER_START_ON_INACTIVE); + if (!in_use) { + INIT_LIST_HEAD(&info->list); + info->timer = timer; + info->time = time; + info->mode = mode; + /* mark as in use */ + atomic_set(&info->state, HRTIMER_START_ON_QUEUED); + } + + unlock_hrtimer_base(timer, &flags); + + if (!in_use) { + /* initiate pull */ + preempt_disable(); + if (cpu == smp_processor_id()) { + /* start timer locally; we may get called + * with rq->lock held, do not wake up anything + */ + TRACE("hrtimer_start_on: starting on local CPU\n"); + __hrtimer_start_range_ns(info->timer, info->time, + 0, info->mode, 0); + } else { + TRACE("hrtimer_start_on: pulling to remote CPU\n"); + base = &per_cpu(hrtimer_bases, cpu); + raw_spin_lock_irqsave(&base->lock, flags); + was_empty = list_empty(&base->to_pull); + list_add(&info->list, &base->to_pull); + raw_spin_unlock_irqrestore(&base->lock, flags); + if (was_empty) + /* only send IPI if other no else + * has done so already + */ + smp_send_pull_timers(cpu); + } + preempt_enable(); + } + return in_use; +} + +#endif /** * hrtimer_try_to_cancel - try to deactivate a timer @@ -1625,6 +1719,7 @@ static void __cpuinit init_hrtimers_cpu(int cpu) } hrtimer_init_hres(cpu_base); + INIT_LIST_HEAD(&cpu_base->to_pull); } #ifdef CONFIG_HOTPLUG_CPU diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 298c927..2bdcdc3 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -542,7 +542,7 @@ static void print_lock(struct held_lock *hlock) print_ip_sym(hlock->acquire_ip); } -static void lockdep_print_held_locks(struct task_struct *curr) +void lockdep_print_held_locks(struct task_struct *curr) { int i, depth = curr->lockdep_depth; @@ -558,6 +558,7 @@ static void lockdep_print_held_locks(struct task_struct *curr) print_lock(curr->held_locks + i); } } +EXPORT_SYMBOL(lockdep_print_held_locks); static void print_kernel_version(void) { @@ -583,6 +584,10 @@ static int static_obj(void *obj) end = (unsigned long) &_end, addr = (unsigned long) obj; + // GLENN + return 1; + + /* * static variable? */ diff --git a/kernel/mutex.c b/kernel/mutex.c index d607ed5..96bcecd 100644 --- a/kernel/mutex.c +++ b/kernel/mutex.c @@ -498,3 +498,128 @@ int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) return 1; } EXPORT_SYMBOL(atomic_dec_and_mutex_lock); + + + + +void mutex_lock_sfx(struct mutex *lock, + side_effect_t pre, unsigned long pre_arg, + side_effect_t post, unsigned long post_arg) +{ + long state = TASK_UNINTERRUPTIBLE; + + struct task_struct *task = current; + struct mutex_waiter waiter; + unsigned long flags; + + preempt_disable(); + mutex_acquire(&lock->dep_map, subclass, 0, ip); + + spin_lock_mutex(&lock->wait_lock, flags); + + if(pre) + { + if(unlikely(pre(pre_arg))) + { + // this will fuck with lockdep's CONFIG_PROVE_LOCKING... + spin_unlock_mutex(&lock->wait_lock, flags); + preempt_enable(); + return; + } + } + + debug_mutex_lock_common(lock, &waiter); + debug_mutex_add_waiter(lock, &waiter, task_thread_info(task)); + + /* add waiting tasks to the end of the waitqueue (FIFO): */ + list_add_tail(&waiter.list, &lock->wait_list); + waiter.task = task; + + if (atomic_xchg(&lock->count, -1) == 1) + goto done; + + lock_contended(&lock->dep_map, ip); + + for (;;) { + /* + * Lets try to take the lock again - this is needed even if + * we get here for the first time (shortly after failing to + * acquire the lock), to make sure that we get a wakeup once + * it's unlocked. Later on, if we sleep, this is the + * operation that gives us the lock. We xchg it to -1, so + * that when we release the lock, we properly wake up the + * other waiters: + */ + if (atomic_xchg(&lock->count, -1) == 1) + break; + + __set_task_state(task, state); + + /* didnt get the lock, go to sleep: */ + spin_unlock_mutex(&lock->wait_lock, flags); + preempt_enable_no_resched(); + schedule(); + preempt_disable(); + spin_lock_mutex(&lock->wait_lock, flags); + } + +done: + lock_acquired(&lock->dep_map, ip); + /* got the lock - rejoice! */ + mutex_remove_waiter(lock, &waiter, current_thread_info()); + mutex_set_owner(lock); + + /* set it to 0 if there are no waiters left: */ + if (likely(list_empty(&lock->wait_list))) + atomic_set(&lock->count, 0); + + if(post) + post(post_arg); + + spin_unlock_mutex(&lock->wait_lock, flags); + + debug_mutex_free_waiter(&waiter); + preempt_enable(); +} +EXPORT_SYMBOL(mutex_lock_sfx); + +void mutex_unlock_sfx(struct mutex *lock, + side_effect_t pre, unsigned long pre_arg, + side_effect_t post, unsigned long post_arg) +{ + unsigned long flags; + + spin_lock_mutex(&lock->wait_lock, flags); + + if(pre) + pre(pre_arg); + + //mutex_release(&lock->dep_map, nested, _RET_IP_); + mutex_release(&lock->dep_map, 1, _RET_IP_); + debug_mutex_unlock(lock); + + /* + * some architectures leave the lock unlocked in the fastpath failure + * case, others need to leave it locked. In the later case we have to + * unlock it here + */ + if (__mutex_slowpath_needs_to_unlock()) + atomic_set(&lock->count, 1); + + if (!list_empty(&lock->wait_list)) { + /* get the first entry from the wait-list: */ + struct mutex_waiter *waiter = + list_entry(lock->wait_list.next, + struct mutex_waiter, list); + + debug_mutex_wake_waiter(lock, waiter); + + wake_up_process(waiter->task); + } + + if(post) + post(post_arg); + + spin_unlock_mutex(&lock->wait_lock, flags); +} +EXPORT_SYMBOL(mutex_unlock_sfx); diff --git a/kernel/printk.c b/kernel/printk.c index 3518539..b799a2e 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -70,6 +70,13 @@ int console_printk[4] = { }; /* + * divert printk() messages when there is a LITMUS^RT debug listener + */ +#include +int trace_override = 0; +int trace_recurse = 0; + +/* * Low level drivers may need that to know if they can schedule in * their unblank() callback or not. So let's export it. */ @@ -871,6 +878,9 @@ asmlinkage int vprintk(const char *fmt, va_list args) /* Emit the output into the temporary buffer */ printed_len += vscnprintf(printk_buf + printed_len, sizeof(printk_buf) - printed_len, fmt, args); + /* if LITMUS^RT tracer is active divert printk() msgs */ + if (trace_override && !trace_recurse) + TRACE("%s", printk_buf); p = printk_buf; @@ -947,7 +957,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) * Try to acquire and then immediately release the * console semaphore. The release will do all the * actual magic (print out buffers, wake up klogd, - * etc). + * etc). * * The console_trylock_for_printk() function * will release 'logbuf_lock' regardless of whether it @@ -1220,7 +1230,7 @@ int printk_needs_cpu(int cpu) void wake_up_klogd(void) { - if (waitqueue_active(&log_wait)) + if (!trace_override && waitqueue_active(&log_wait)) this_cpu_write(printk_pending, 1); } diff --git a/kernel/sched.c b/kernel/sched.c index fde6ff9..2f990b4 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -80,6 +80,15 @@ #include "workqueue_sched.h" #include "sched_autogroup.h" +#include +#include + +#ifdef CONFIG_LITMUS_SOFTIRQD +#include +#endif + +static void litmus_tick(struct rq*, struct task_struct*); + #define CREATE_TRACE_POINTS #include @@ -410,6 +419,12 @@ struct rt_rq { #endif }; +/* Litmus related fields in a runqueue */ +struct litmus_rq { + unsigned long nr_running; + struct task_struct *prev; +}; + #ifdef CONFIG_SMP /* @@ -475,6 +490,7 @@ struct rq { struct cfs_rq cfs; struct rt_rq rt; + struct litmus_rq litmus; #ifdef CONFIG_FAIR_GROUP_SCHED /* list of leaf cfs_rq on this cpu: */ @@ -1045,6 +1061,7 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer) raw_spin_lock(&rq->lock); update_rq_clock(rq); rq->curr->sched_class->task_tick(rq, rq->curr, 1); + litmus_tick(rq, rq->curr); raw_spin_unlock(&rq->lock); return HRTIMER_NORESTART; @@ -1773,7 +1790,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) static const struct sched_class rt_sched_class; -#define sched_class_highest (&stop_sched_class) +#define sched_class_highest (&litmus_sched_class) #define for_each_class(class) \ for (class = sched_class_highest; class; class = class->next) @@ -2031,6 +2048,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) #include "sched_rt.c" #include "sched_autogroup.c" #include "sched_stoptask.c" +#include "../litmus/sched_litmus.c" #ifdef CONFIG_SCHED_DEBUG # include "sched_debug.c" #endif @@ -2153,6 +2171,10 @@ static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) * A queue event has occurred, and we're going to schedule. In * this case, we can save a useless back to back clock update. */ + /* LITMUS^RT: + * The "disable-clock-update" approach was buggy in Linux 2.6.36. + * The issue has been solved in 2.6.37. + */ if (rq->curr->on_rq && test_tsk_need_resched(rq->curr)) rq->skip_clock_update = 1; } @@ -2643,7 +2665,12 @@ static void ttwu_queue(struct task_struct *p, int cpu) struct rq *rq = cpu_rq(cpu); #if defined(CONFIG_SMP) - if (sched_feat(TTWU_QUEUE) && cpu != smp_processor_id()) { + /* + * LITMUS^RT: whether to send an IPI to the remote CPU + * is plugin specific. + */ + if (!is_realtime(p) && + sched_feat(TTWU_QUEUE) && cpu != smp_processor_id()) { sched_clock_cpu(cpu); /* sync clocks x-cpu */ ttwu_queue_remote(p, cpu); return; @@ -2676,6 +2703,9 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) unsigned long flags; int cpu, success = 0; + if (is_realtime(p)) + TRACE_TASK(p, "try_to_wake_up() state:%d\n", p->state); + smp_wmb(); raw_spin_lock_irqsave(&p->pi_lock, flags); if (!(p->state & state)) @@ -2712,6 +2742,12 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_rmb(); + /* LITMUS^RT: once the task can be safely referenced by this + * CPU, don't mess up with Linux load balancing stuff. + */ + if (is_realtime(p)) + goto litmus_out_activate; + p->sched_contributes_to_load = !!task_contributes_to_load(p); p->state = TASK_WAKING; @@ -2723,12 +2759,16 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) wake_flags |= WF_MIGRATED; set_task_cpu(p, cpu); } + +litmus_out_activate: #endif /* CONFIG_SMP */ ttwu_queue(p, cpu); stat: ttwu_stat(p, cpu, wake_flags); out: + if (is_realtime(p)) + TRACE_TASK(p, "try_to_wake_up() done state:%d\n", p->state); raw_spin_unlock_irqrestore(&p->pi_lock, flags); return success; @@ -2839,7 +2879,8 @@ void sched_fork(struct task_struct *p) * Revert to default priority/policy on fork if requested. */ if (unlikely(p->sched_reset_on_fork)) { - if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) { + if (p->policy == SCHED_FIFO || p->policy == SCHED_RR || + p->policy == SCHED_LITMUS) { p->policy = SCHED_NORMAL; p->normal_prio = p->static_prio; } @@ -3050,6 +3091,8 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev) */ prev_state = prev->state; finish_arch_switch(prev); + litmus->finish_switch(prev); + prev->rt_param.stack_in_use = NO_CPU; #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW local_irq_disable(); #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */ @@ -3079,6 +3122,15 @@ static inline void pre_schedule(struct rq *rq, struct task_struct *prev) { if (prev->sched_class->pre_schedule) prev->sched_class->pre_schedule(rq, prev); + + /* LITMUS^RT not very clean hack: we need to save the prev task + * as our scheduling decision rely on it (as we drop the rq lock + * something in prev can change...); there is no way to escape + * this ack apart from modifying pick_nex_task(rq, _prev_) or + * falling back on the previous solution of decoupling + * scheduling decisions + */ + rq->litmus.prev = prev; } /* rq->lock is NOT held, but preemption is disabled */ @@ -4094,18 +4146,26 @@ void scheduler_tick(void) sched_clock_tick(); + TS_TICK_START(current); + raw_spin_lock(&rq->lock); update_rq_clock(rq); update_cpu_load_active(rq); curr->sched_class->task_tick(rq, curr, 0); + + /* litmus_tick may force current to resched */ + litmus_tick(rq, curr); + raw_spin_unlock(&rq->lock); perf_event_task_tick(); #ifdef CONFIG_SMP rq->idle_at_tick = idle_cpu(cpu); - trigger_load_balance(rq, cpu); + if (!is_realtime(current)) + trigger_load_balance(rq, cpu); #endif + TS_TICK_END(current); } notrace unsigned long get_parent_ip(unsigned long addr) @@ -4225,12 +4285,20 @@ pick_next_task(struct rq *rq) /* * Optimization: we know that if all tasks are in * the fair class we can call that function directly: - */ - if (likely(rq->nr_running == rq->cfs.nr_running)) { + + * NOT IN LITMUS^RT! + + * This breaks many assumptions in the plugins. + * Do not uncomment without thinking long and hard + * about how this affects global plugins such as GSN-EDF. + + if (rq->nr_running == rq->cfs.nr_running) { + TRACE("taking shortcut in pick_next_task()\n"); p = fair_sched_class.pick_next_task(rq); if (likely(p)) return p; } + */ for_each_class(class) { p = class->pick_next_task(rq); @@ -4241,6 +4309,7 @@ pick_next_task(struct rq *rq) BUG(); /* the idle class will always have a runnable task */ } + /* * schedule() is the main scheduler function. */ @@ -4253,11 +4322,23 @@ asmlinkage void __sched schedule(void) need_resched: preempt_disable(); + sched_state_entered_schedule(); cpu = smp_processor_id(); rq = cpu_rq(cpu); rcu_note_context_switch(cpu); prev = rq->curr; +#ifdef CONFIG_LITMUS_SOFTIRQD + release_klitirqd_lock(prev); +#endif + + /* LITMUS^RT: quickly re-evaluate the scheduling decision + * if the previous one is no longer valid after CTX. + */ +litmus_need_resched_nonpreemptible: + TS_SCHED_START; + sched_trace_task_switch_away(prev); + schedule_debug(prev); if (sched_feat(HRTICK)) @@ -4314,7 +4395,10 @@ need_resched: rq->curr = next; ++*switch_count; + TS_SCHED_END(next); + TS_CXS_START(next); context_switch(rq, prev, next); /* unlocks the rq */ + TS_CXS_END(current); /* * The context switch have flipped the stack from under us * and restored the local variables which were saved when @@ -4323,17 +4407,37 @@ need_resched: */ cpu = smp_processor_id(); rq = cpu_rq(cpu); - } else + } else { + TS_SCHED_END(prev); raw_spin_unlock_irq(&rq->lock); + } + + sched_trace_task_switch_to(current); post_schedule(rq); + if (sched_state_validate_switch()) + goto litmus_need_resched_nonpreemptible; + preempt_enable_no_resched(); + if (need_resched()) goto need_resched; + +#ifdef LITMUS_SOFTIRQD + reacquire_klitirqd_lock(prev); +#endif + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + litmus->run_tasklets(prev); +#endif + + srp_ceiling_block(); } EXPORT_SYMBOL(schedule); + + #ifdef CONFIG_MUTEX_SPIN_ON_OWNER static inline bool owner_running(struct mutex *lock, struct task_struct *owner) @@ -4477,6 +4581,7 @@ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, } } + /** * __wake_up - wake up threads blocked on a waitqueue. * @q: the waitqueue @@ -4600,6 +4705,17 @@ void complete_all(struct completion *x) } EXPORT_SYMBOL(complete_all); +void complete_n(struct completion *x, int n) +{ + unsigned long flags; + + spin_lock_irqsave(&x->wait.lock, flags); + x->done += n; + __wake_up_common(&x->wait, TASK_NORMAL, n, 0, NULL); + spin_unlock_irqrestore(&x->wait.lock, flags); +} +EXPORT_SYMBOL(complete_n); + static inline long __sched do_wait_for_common(struct completion *x, long timeout, int state) { @@ -4652,6 +4768,12 @@ void __sched wait_for_completion(struct completion *x) } EXPORT_SYMBOL(wait_for_completion); +void __sched __wait_for_completion_locked(struct completion *x) +{ + do_wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(__wait_for_completion_locked); + /** * wait_for_completion_timeout: - waits for completion of a task (w/timeout) * @x: holds the state of this particular completion @@ -5039,7 +5161,9 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio) p->normal_prio = normal_prio(p); /* we are holding p->pi_lock already */ p->prio = rt_mutex_getprio(p); - if (rt_prio(p->prio)) + if (p->policy == SCHED_LITMUS) + p->sched_class = &litmus_sched_class; + else if (rt_prio(p->prio)) p->sched_class = &rt_sched_class; else p->sched_class = &fair_sched_class; @@ -5087,7 +5211,7 @@ recheck: if (policy != SCHED_FIFO && policy != SCHED_RR && policy != SCHED_NORMAL && policy != SCHED_BATCH && - policy != SCHED_IDLE) + policy != SCHED_IDLE && policy != SCHED_LITMUS) return -EINVAL; } @@ -5102,6 +5226,8 @@ recheck: return -EINVAL; if (rt_policy(policy) != (param->sched_priority != 0)) return -EINVAL; + if (policy == SCHED_LITMUS && policy == p->policy) + return -EINVAL; /* * Allow unprivileged RT tasks to decrease priority: @@ -5145,6 +5271,12 @@ recheck: return retval; } + if (policy == SCHED_LITMUS) { + retval = litmus_admit_task(p); + if (retval) + return retval; + } + /* * make sure no PI-waiters arrive (or leave) while we are * changing the priority of the task: @@ -5203,10 +5335,19 @@ recheck: p->sched_reset_on_fork = reset_on_fork; + if (p->policy == SCHED_LITMUS) + litmus_exit_task(p); + oldprio = p->prio; prev_class = p->sched_class; __setscheduler(rq, p, policy, param->sched_priority); + if (policy == SCHED_LITMUS) { + p->rt_param.stack_in_use = running ? rq->cpu : NO_CPU; + p->rt_param.present = running; + litmus->task_new(p, on_rq, running); + } + if (running) p->sched_class->set_curr_task(rq); if (on_rq) @@ -5374,10 +5515,11 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) rcu_read_lock(); p = find_process_by_pid(pid); - if (!p) { + /* Don't set affinity if task not found and for LITMUS tasks */ + if (!p || is_realtime(p)) { rcu_read_unlock(); put_online_cpus(); - return -ESRCH; + return p ? -EPERM : -ESRCH; } /* Prevent p going away */ diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index c768588..334eb47 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -1890,6 +1890,9 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ int scale = cfs_rq->nr_running >= sched_nr_latency; int next_buddy_marked = 0; + if (unlikely(rt_prio(p->prio)) || p->policy == SCHED_LITMUS) + goto preempt; + if (unlikely(se == pse)) return; diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index 10d0182..58cf5d1 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -1078,7 +1078,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) */ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags) { - if (p->prio < rq->curr->prio) { + if (p->prio < rq->curr->prio || p->policy == SCHED_LITMUS) { resched_task(rq->curr); return; } diff --git a/kernel/semaphore.c b/kernel/semaphore.c index 94a62c0..c947a04 100644 --- a/kernel/semaphore.c +++ b/kernel/semaphore.c @@ -33,11 +33,11 @@ #include #include -static noinline void __down(struct semaphore *sem); +noinline void __down(struct semaphore *sem); static noinline int __down_interruptible(struct semaphore *sem); static noinline int __down_killable(struct semaphore *sem); static noinline int __down_timeout(struct semaphore *sem, long jiffies); -static noinline void __up(struct semaphore *sem); +noinline void __up(struct semaphore *sem); /** * down - acquire the semaphore @@ -190,11 +190,13 @@ EXPORT_SYMBOL(up); /* Functions for the contended case */ +/* struct semaphore_waiter { struct list_head list; struct task_struct *task; int up; }; + */ /* * Because this function is inlined, the 'state' parameter will be @@ -233,10 +235,12 @@ static inline int __sched __down_common(struct semaphore *sem, long state, return -EINTR; } -static noinline void __sched __down(struct semaphore *sem) +noinline void __sched __down(struct semaphore *sem) { __down_common(sem, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); } +EXPORT_SYMBOL(__down); + static noinline int __sched __down_interruptible(struct semaphore *sem) { @@ -253,7 +257,7 @@ static noinline int __sched __down_timeout(struct semaphore *sem, long jiffies) return __down_common(sem, TASK_UNINTERRUPTIBLE, jiffies); } -static noinline void __sched __up(struct semaphore *sem) +noinline void __sched __up(struct semaphore *sem) { struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list, struct semaphore_waiter, list); @@ -261,3 +265,4 @@ static noinline void __sched __up(struct semaphore *sem) waiter->up = 1; wake_up_process(waiter->task); } +EXPORT_SYMBOL(__up); \ No newline at end of file diff --git a/kernel/softirq.c b/kernel/softirq.c index fca82c3..4d7b1a3 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -29,6 +29,15 @@ #include #include + +#include +#include + +#ifdef CONFIG_LITMUS_NVIDIA +#include +#include +#endif + /* - No shared variables, all the data are CPU local. - If a softirq needs serialization, let it serialize itself @@ -67,7 +76,7 @@ char *softirq_to_name[NR_SOFTIRQS] = { * to the pending events, so lets the scheduler to balance * the softirq load for us. */ -static void wakeup_softirqd(void) +void wakeup_softirqd(void) { /* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk = __this_cpu_read(ksoftirqd); @@ -193,6 +202,7 @@ void local_bh_enable_ip(unsigned long ip) } EXPORT_SYMBOL(local_bh_enable_ip); + /* * We restart softirq processing MAX_SOFTIRQ_RESTART times, * and we fall back to softirqd after that. @@ -206,65 +216,65 @@ EXPORT_SYMBOL(local_bh_enable_ip); asmlinkage void __do_softirq(void) { - struct softirq_action *h; - __u32 pending; - int max_restart = MAX_SOFTIRQ_RESTART; - int cpu; + struct softirq_action *h; + __u32 pending; + int max_restart = MAX_SOFTIRQ_RESTART; + int cpu; - pending = local_softirq_pending(); - account_system_vtime(current); + pending = local_softirq_pending(); + account_system_vtime(current); - __local_bh_disable((unsigned long)__builtin_return_address(0), - SOFTIRQ_OFFSET); - lockdep_softirq_enter(); + __local_bh_disable((unsigned long)__builtin_return_address(0), + SOFTIRQ_OFFSET); + lockdep_softirq_enter(); - cpu = smp_processor_id(); + cpu = smp_processor_id(); restart: - /* Reset the pending bitmask before enabling irqs */ - set_softirq_pending(0); + /* Reset the pending bitmask before enabling irqs */ + set_softirq_pending(0); - local_irq_enable(); + local_irq_enable(); - h = softirq_vec; - - do { - if (pending & 1) { - unsigned int vec_nr = h - softirq_vec; - int prev_count = preempt_count(); - - kstat_incr_softirqs_this_cpu(vec_nr); - - trace_softirq_entry(vec_nr); - h->action(h); - trace_softirq_exit(vec_nr); - if (unlikely(prev_count != preempt_count())) { - printk(KERN_ERR "huh, entered softirq %u %s %p" - "with preempt_count %08x," - " exited with %08x?\n", vec_nr, - softirq_to_name[vec_nr], h->action, - prev_count, preempt_count()); - preempt_count() = prev_count; - } + h = softirq_vec; - rcu_bh_qs(cpu); - } - h++; - pending >>= 1; - } while (pending); + do { + if (pending & 1) { + unsigned int vec_nr = h - softirq_vec; + int prev_count = preempt_count(); - local_irq_disable(); + kstat_incr_softirqs_this_cpu(vec_nr); - pending = local_softirq_pending(); - if (pending && --max_restart) - goto restart; + trace_softirq_entry(vec_nr); + h->action(h); + trace_softirq_exit(vec_nr); + if (unlikely(prev_count != preempt_count())) { + printk(KERN_ERR "huh, entered softirq %u %s %p" + "with preempt_count %08x," + " exited with %08x?\n", vec_nr, + softirq_to_name[vec_nr], h->action, + prev_count, preempt_count()); + preempt_count() = prev_count; + } - if (pending) - wakeup_softirqd(); + rcu_bh_qs(cpu); + } + h++; + pending >>= 1; + } while (pending); - lockdep_softirq_exit(); + local_irq_disable(); - account_system_vtime(current); - __local_bh_enable(SOFTIRQ_OFFSET); + pending = local_softirq_pending(); + if (pending && --max_restart) + goto restart; + + if (pending) + wakeup_softirqd(); + + lockdep_softirq_exit(); + + account_system_vtime(current); + __local_bh_enable(SOFTIRQ_OFFSET); } #ifndef __ARCH_HAS_DO_SOFTIRQ @@ -402,8 +412,99 @@ struct tasklet_head static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); +#ifdef CONFIG_LITMUS_NVIDIA +static int __do_nv_now(struct tasklet_struct* tasklet) +{ + int success = 1; + + if(tasklet_trylock(tasklet)) { + if (!atomic_read(&tasklet->count)) { + if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) { + BUG(); + } + tasklet->func(tasklet->data); + tasklet_unlock(tasklet); + } + else { + success = 0; + } + + tasklet_unlock(tasklet); + } + else { + success = 0; + } + + return success; +} +#endif + + void __tasklet_schedule(struct tasklet_struct *t) { +#ifdef CONFIG_LITMUS_NVIDIA + if(is_nvidia_func(t->func)) + { +#if 0 + // do nvidia tasklets right away and return + if(__do_nv_now(t)) + return; +#else + u32 nvidia_device = get_tasklet_nv_device_num(t); + // TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n", + // __FUNCTION__, nvidia_device,litmus_clock()); + + unsigned long flags; + struct task_struct* device_owner; + + lock_nv_registry(nvidia_device, &flags); + + device_owner = get_nv_max_device_owner(nvidia_device); + + if(device_owner==NULL) + { + t->owner = NULL; + } + else + { + if(is_realtime(device_owner)) + { + TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n", + __FUNCTION__, nvidia_device,litmus_clock()); + TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n", + __FUNCTION__,device_owner->pid,nvidia_device); + + t->owner = device_owner; + sched_trace_tasklet_release(t->owner); + + if(likely(_litmus_tasklet_schedule(t,nvidia_device))) + { + unlock_nv_registry(nvidia_device, &flags); + return; + } + else + { + t->owner = NULL; /* fall through to normal scheduling */ + } + } + else + { + t->owner = NULL; + } + } + unlock_nv_registry(nvidia_device, &flags); +#endif + } + +#endif + + ___tasklet_schedule(t); +} +EXPORT_SYMBOL(__tasklet_schedule); + + +void ___tasklet_schedule(struct tasklet_struct *t) +{ unsigned long flags; local_irq_save(flags); @@ -413,11 +514,65 @@ void __tasklet_schedule(struct tasklet_struct *t) raise_softirq_irqoff(TASKLET_SOFTIRQ); local_irq_restore(flags); } +EXPORT_SYMBOL(___tasklet_schedule); -EXPORT_SYMBOL(__tasklet_schedule); void __tasklet_hi_schedule(struct tasklet_struct *t) { +#ifdef CONFIG_LITMUS_NVIDIA + if(is_nvidia_func(t->func)) + { + u32 nvidia_device = get_tasklet_nv_device_num(t); + // TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n", + // __FUNCTION__, nvidia_device,litmus_clock()); + + unsigned long flags; + struct task_struct* device_owner; + + lock_nv_registry(nvidia_device, &flags); + + device_owner = get_nv_max_device_owner(nvidia_device); + + if(device_owner==NULL) + { + t->owner = NULL; + } + else + { + if( is_realtime(device_owner)) + { + TRACE("%s: Handling NVIDIA tasklet for device %u\tat %llu\n", + __FUNCTION__, nvidia_device,litmus_clock()); + TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n", + __FUNCTION__,device_owner->pid,nvidia_device); + + t->owner = device_owner; + sched_trace_tasklet_release(t->owner); + if(likely(_litmus_tasklet_hi_schedule(t,nvidia_device))) + { + unlock_nv_registry(nvidia_device, &flags); + return; + } + else + { + t->owner = NULL; /* fall through to normal scheduling */ + } + } + else + { + t->owner = NULL; + } + } + unlock_nv_registry(nvidia_device, &flags); + } +#endif + + ___tasklet_hi_schedule(t); +} +EXPORT_SYMBOL(__tasklet_hi_schedule); + +void ___tasklet_hi_schedule(struct tasklet_struct* t) +{ unsigned long flags; local_irq_save(flags); @@ -427,19 +582,72 @@ void __tasklet_hi_schedule(struct tasklet_struct *t) raise_softirq_irqoff(HI_SOFTIRQ); local_irq_restore(flags); } - -EXPORT_SYMBOL(__tasklet_hi_schedule); +EXPORT_SYMBOL(___tasklet_hi_schedule); void __tasklet_hi_schedule_first(struct tasklet_struct *t) { BUG_ON(!irqs_disabled()); +#ifdef CONFIG_LITMUS_NVIDIA + if(is_nvidia_func(t->func)) + { + u32 nvidia_device = get_tasklet_nv_device_num(t); + // TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n", + // __FUNCTION__, nvidia_device,litmus_clock()); + unsigned long flags; + struct task_struct* device_owner; + + lock_nv_registry(nvidia_device, &flags); + + device_owner = get_nv_max_device_owner(nvidia_device); + + if(device_owner==NULL) + { + t->owner = NULL; + } + else + { + if(is_realtime(device_owner)) + { + TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n", + __FUNCTION__, nvidia_device,litmus_clock()); + + TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n", + __FUNCTION__,device_owner->pid,nvidia_device); + + t->owner = device_owner; + sched_trace_tasklet_release(t->owner); + if(likely(_litmus_tasklet_hi_schedule_first(t,nvidia_device))) + { + unlock_nv_registry(nvidia_device, &flags); + return; + } + else + { + t->owner = NULL; /* fall through to normal scheduling */ + } + } + else + { + t->owner = NULL; + } + } + unlock_nv_registry(nvidia_device, &flags); + } +#endif + + ___tasklet_hi_schedule_first(t); +} +EXPORT_SYMBOL(__tasklet_hi_schedule_first); + +void ___tasklet_hi_schedule_first(struct tasklet_struct* t) +{ + BUG_ON(!irqs_disabled()); t->next = __this_cpu_read(tasklet_hi_vec.head); __this_cpu_write(tasklet_hi_vec.head, t); __raise_softirq_irqoff(HI_SOFTIRQ); } - -EXPORT_SYMBOL(__tasklet_hi_schedule_first); +EXPORT_SYMBOL(___tasklet_hi_schedule_first); static void tasklet_action(struct softirq_action *a) { @@ -495,6 +703,7 @@ static void tasklet_hi_action(struct softirq_action *a) if (!atomic_read(&t->count)) { if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) BUG(); + t->func(t->data); tasklet_unlock(t); continue; @@ -518,8 +727,13 @@ void tasklet_init(struct tasklet_struct *t, t->next = NULL; t->state = 0; atomic_set(&t->count, 0); + t->func = func; t->data = data; + +#ifdef CONFIG_LITMUS_SOFTIRQD + t->owner = NULL; +#endif } EXPORT_SYMBOL(tasklet_init); @@ -534,6 +748,7 @@ void tasklet_kill(struct tasklet_struct *t) yield(); } while (test_bit(TASKLET_STATE_SCHED, &t->state)); } + tasklet_unlock_wait(t); clear_bit(TASKLET_STATE_SCHED, &t->state); } @@ -808,6 +1023,7 @@ void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu) for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) { if (*i == t) { *i = t->next; + /* If this was the tail element, move the tail ptr */ if (*i == NULL) per_cpu(tasklet_vec, cpu).tail = i; diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index d5097c4..0c0e02f 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -766,12 +766,53 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) } /** + * tick_set_quanta_type - get the quanta type as a boot option + * Default is standard setup with ticks staggered over first + * half of tick period. + */ +int quanta_type = LINUX_DEFAULT_TICKS; +static int __init tick_set_quanta_type(char *str) +{ + if (strcmp("aligned", str) == 0) { + quanta_type = LITMUS_ALIGNED_TICKS; + printk(KERN_INFO "LITMUS^RT: setting aligned quanta\n"); + } + else if (strcmp("staggered", str) == 0) { + quanta_type = LITMUS_STAGGERED_TICKS; + printk(KERN_INFO "LITMUS^RT: setting staggered quanta\n"); + } + return 1; +} +__setup("quanta=", tick_set_quanta_type); + +u64 cpu_stagger_offset(int cpu) +{ + u64 offset = 0; + switch (quanta_type) { + case LITMUS_ALIGNED_TICKS: + offset = 0; + break; + case LITMUS_STAGGERED_TICKS: + offset = ktime_to_ns(tick_period); + do_div(offset, num_possible_cpus()); + offset *= cpu; + break; + default: + offset = ktime_to_ns(tick_period) >> 1; + do_div(offset, num_possible_cpus()); + offset *= cpu; + } + return offset; +} + +/** * tick_setup_sched_timer - setup the tick emulation timer */ void tick_setup_sched_timer(void) { struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); ktime_t now = ktime_get(); + u64 offset; /* * Emulate tick processing via per-CPU hrtimers: @@ -782,6 +823,12 @@ void tick_setup_sched_timer(void) /* Get the next period (per cpu) */ hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); + /* Offset must be set correctly to achieve desired quanta type. */ + offset = cpu_stagger_offset(smp_processor_id()); + + /* Add the correct offset to expiration time */ + hrtimer_add_expires_ns(&ts->sched_timer, offset); + for (;;) { hrtimer_forward(&ts->sched_timer, now, tick_period); hrtimer_start_expires(&ts->sched_timer, diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 0400553..6b59d59 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -44,6 +44,13 @@ #include "workqueue_sched.h" +#ifdef CONFIG_LITMUS_NVIDIA +#include +#include +#include +#endif + + enum { /* global_cwq flags */ GCWQ_MANAGE_WORKERS = 1 << 0, /* need to manage workers */ @@ -1047,9 +1054,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, work_flags |= WORK_STRUCT_DELAYED; worklist = &cwq->delayed_works; } - insert_work(cwq, work, worklist, work_flags); - spin_unlock_irqrestore(&gcwq->lock, flags); } @@ -2687,10 +2692,70 @@ EXPORT_SYMBOL(cancel_delayed_work_sync); */ int schedule_work(struct work_struct *work) { - return queue_work(system_wq, work); +#if 0 +#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) + if(is_nvidia_func(work->func)) + { + u32 nvidiaDevice = get_work_nv_device_num(work); + + //1) Ask Litmus which task owns GPU . (API to be defined.) + unsigned long flags; + struct task_struct* device_owner; + + lock_nv_registry(nvidiaDevice, &flags); + + device_owner = get_nv_max_device_owner(nvidiaDevice); + + //2) If there is an owner, set work->owner to the owner's task struct. + if(device_owner==NULL) + { + work->owner = NULL; + //TRACE("%s: the owner task of NVIDIA Device %u is NULL\n",__FUNCTION__,nvidiaDevice); + } + else + { + if( is_realtime(device_owner)) + { + TRACE("%s: Handling NVIDIA work for device\t%u\tat\t%llu\n", + __FUNCTION__, nvidiaDevice,litmus_clock()); + TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n", + __FUNCTION__, + device_owner->pid, + nvidiaDevice); + + //3) Call litmus_schedule_work() and return (don't execute the rest + // of schedule_schedule()). + work->owner = device_owner; + sched_trace_work_release(work->owner); + if(likely(litmus_schedule_work(work, nvidiaDevice))) + { + unlock_nv_registry(nvidiaDevice, &flags); + return 1; + } + else + { + work->owner = NULL; /* fall through to normal work scheduling */ + } + } + else + { + work->owner = NULL; + } + } + unlock_nv_registry(nvidiaDevice, &flags); + } +#endif +#endif + return(__schedule_work(work)); } EXPORT_SYMBOL(schedule_work); +int __schedule_work(struct work_struct* work) +{ + return queue_work(system_wq, work); +} +EXPORT_SYMBOL(__schedule_work); + /* * schedule_work_on - put work task on a specific cpu * @cpu: cpu to put the work task on diff --git a/litmus/Kconfig b/litmus/Kconfig new file mode 100644 index 0000000..03cc92c --- /dev/null +++ b/litmus/Kconfig @@ -0,0 +1,364 @@ +menu "LITMUS^RT" + +menu "Scheduling" + +config PLUGIN_CEDF + bool "Clustered-EDF" + depends on X86 && SYSFS + default y + help + Include the Clustered EDF (C-EDF) plugin in the kernel. + This is appropriate for large platforms with shared caches. + On smaller platforms (e.g., ARM PB11MPCore), using C-EDF + makes little sense since there aren't any shared caches. + +config PLUGIN_PFAIR + bool "PFAIR" + depends on HIGH_RES_TIMERS && !NO_HZ + default y + help + Include the PFAIR plugin (i.e., the PD^2 scheduler) in the kernel. + The PFAIR plugin requires high resolution timers (for staggered quanta) + and does not support NO_HZ (quanta could be missed when the system is idle). + + If unsure, say Yes. + +config RELEASE_MASTER + bool "Release-master Support" + depends on ARCH_HAS_SEND_PULL_TIMERS + default n + help + Allow one processor to act as a dedicated interrupt processor + that services all timer interrupts, but that does not schedule + real-time tasks. See RTSS'09 paper for details + (http://www.cs.unc.edu/~anderson/papers.html). + Currently only supported by GSN-EDF. + +endmenu + +menu "Real-Time Synchronization" + +config NP_SECTION + bool "Non-preemptive section support" + default n + help + Allow tasks to become non-preemptable. + Note that plugins still need to explicitly support non-preemptivity. + Currently, only GSN-EDF and PSN-EDF have such support. + + This is required to support locking protocols such as the FMLP. + If disabled, all tasks will be considered preemptable at all times. + +config LITMUS_LOCKING + bool "Support for real-time locking protocols" + depends on NP_SECTION + default n + help + Enable LITMUS^RT's deterministic multiprocessor real-time + locking protocols. + + Say Yes if you want to include locking protocols such as the FMLP and + Baker's SRP. + +config LITMUS_AFFINITY_LOCKING + bool "Enable affinity infrastructure in k-exclusion locking protocols." + depends on LITMUS_LOCKING + default n + help + Enable affinity tracking infrastructure in k-exclusion locking protocols. + This only enabled the *infrastructure* not actual affinity algorithms. + + If unsure, say No. + +config LITMUS_NESTED_LOCKING + bool "Support for nested inheritance in locking protocols" + depends on LITMUS_LOCKING + default n + help + Enable nested priority inheritance. + +config LITMUS_DGL_SUPPORT + bool "Support for dynamic group locks" + depends on LITMUS_NESTED_LOCKING + default n + help + Enable dynamic group lock support. + +config LITMUS_MAX_DGL_SIZE + int "Maximum size of a dynamic group lock." + depends on LITMUS_DGL_SUPPORT + range 1 128 + default "10" + help + Dynamic group lock data structures are allocated on the process + stack when a group is requested. We set a maximum size of + locks in a dynamic group lock to avoid dynamic allocation. + + TODO: Batch DGL requests exceeding LITMUS_MAX_DGL_SIZE. + +endmenu + +menu "Performance Enhancements" + +config SCHED_CPU_AFFINITY + bool "Local Migration Affinity" + depends on X86 + default y + help + Rescheduled tasks prefer CPUs near to their previously used CPU. This + may improve performance through possible preservation of cache affinity. + + Warning: May make bugs harder to find since tasks may migrate less often. + + NOTES: + * Feature is not utilized by PFair/PD^2. + + Say Yes if unsure. + +endmenu + +menu "Tracing" + +config FEATHER_TRACE + bool "Feather-Trace Infrastructure" + default y + help + Feather-Trace basic tracing infrastructure. Includes device file + driver and instrumentation point support. + + There are actually two implementations of Feather-Trace. + 1) A slower, but portable, default implementation. + 2) Architecture-specific implementations that rewrite kernel .text at runtime. + + If enabled, Feather-Trace will be based on 2) if available (currently only for x86). + However, if DEBUG_RODATA=y, then Feather-Trace will choose option 1) in any case + to avoid problems with write-protected .text pages. + + Bottom line: to avoid increased overheads, choose DEBUG_RODATA=n. + + Note that this option only enables the basic Feather-Trace infrastructure; + you still need to enable SCHED_TASK_TRACE and/or SCHED_OVERHEAD_TRACE to + actually enable any events. + +config SCHED_TASK_TRACE + bool "Trace real-time tasks" + depends on FEATHER_TRACE + default y + help + Include support for the sched_trace_XXX() tracing functions. This + allows the collection of real-time task events such as job + completions, job releases, early completions, etc. This results in a + small overhead in the scheduling code. Disable if the overhead is not + acceptable (e.g., benchmarking). + + Say Yes for debugging. + Say No for overhead tracing. + +config SCHED_TASK_TRACE_SHIFT + int "Buffer size for sched_trace_xxx() events" + depends on SCHED_TASK_TRACE + range 8 15 + default 9 + help + + Select the buffer size of sched_trace_xxx() events as a power of two. + These buffers are statically allocated as per-CPU data. Each event + requires 24 bytes storage plus one additional flag byte. Too large + buffers can cause issues with the per-cpu allocator (and waste + memory). Too small buffers can cause scheduling events to be lost. The + "right" size is workload dependent and depends on the number of tasks, + each task's period, each task's number of suspensions, and how often + the buffer is flushed. + + Examples: 12 => 4k events + 10 => 1k events + 8 => 512 events + +config SCHED_OVERHEAD_TRACE + bool "Record timestamps for overhead measurements" + depends on FEATHER_TRACE + default n + help + Export event stream for overhead tracing. + Say Yes for overhead tracing. + +config SCHED_DEBUG_TRACE + bool "TRACE() debugging" + default y + help + Include support for sched_trace_log_messageg(), which is used to + implement TRACE(). If disabled, no TRACE() messages will be included + in the kernel, and no overheads due to debugging statements will be + incurred by the scheduler. Disable if the overhead is not acceptable + (e.g. benchmarking). + + Say Yes for debugging. + Say No for overhead tracing. + +config SCHED_DEBUG_TRACE_SHIFT + int "Buffer size for TRACE() buffer" + depends on SCHED_DEBUG_TRACE + range 14 22 + default 18 + help + + Select the amount of memory needed per for the TRACE() buffer, as a + power of two. The TRACE() buffer is global and statically allocated. If + the buffer is too small, there will be holes in the TRACE() log if the + buffer-flushing task is starved. + + The default should be sufficient for most systems. Increase the buffer + size if the log contains holes. Reduce the buffer size when running on + a memory-constrained system. + + Examples: 14 => 16KB + 18 => 256KB + 20 => 1MB + + This buffer is exported to usespace using a misc device as + 'litmus/log'. On a system with default udev rules, a corresponding + character device node should be created at /dev/litmus/log. The buffer + can be flushed using cat, e.g., 'cat /dev/litmus/log > my_log_file.txt'. + +config SCHED_DEBUG_TRACE_CALLER + bool "Include [function@file:line] tag in TRACE() log" + depends on SCHED_DEBUG_TRACE + default n + help + With this option enabled, TRACE() prepends + + "[@:]" + + to each message in the debug log. Enable this to aid in figuring out + what was called in which order. The downside is that it adds a lot of + clutter. + + If unsure, say No. + +config PREEMPT_STATE_TRACE + bool "Trace preemption state machine transitions" + depends on SCHED_DEBUG_TRACE + default n + help + With this option enabled, each CPU will log when it transitions + states in the preemption state machine. This state machine is + used to determine how to react to IPIs (avoid races with in-flight IPIs). + + Warning: this creates a lot of information in the debug trace. Only + recommended when you are debugging preemption-related races. + + If unsure, say No. + +endmenu + +menu "Interrupt Handling" + +choice + prompt "Scheduling of interrupt bottom-halves in Litmus." + default LITMUS_SOFTIRQD_NONE + depends on LITMUS_LOCKING && !LITMUS_THREAD_ALL_SOFTIRQ + help + Schedule tasklets with known priorities in Litmus. + +config LITMUS_SOFTIRQD_NONE + bool "No tasklet scheduling in Litmus." + help + Don't schedule tasklets in Litmus. Default. + +config LITMUS_SOFTIRQD + bool "Spawn klitirqd interrupt handling threads." + help + Create klitirqd interrupt handling threads. Work must be + specifically dispatched to these workers. (Softirqs for + Litmus tasks are not magically redirected to klitirqd.) + + G-EDF/RM, C-EDF/RM ONLY for now! + + +config LITMUS_PAI_SOFTIRQD + bool "Defer tasklets to context switch points." + help + Only execute scheduled tasklet bottom halves at + scheduling points. Trades context switch overhead + at the cost of non-preemptive durations of bottom half + processing. + + G-EDF/RM, C-EDF/RM ONLY for now! + +endchoice + + +config NR_LITMUS_SOFTIRQD + int "Number of klitirqd." + depends on LITMUS_SOFTIRQD + range 1 4096 + default "1" + help + Should be <= to the number of CPUs in your system. + +config LITMUS_NVIDIA + bool "Litmus handling of NVIDIA interrupts." + default n + help + Direct tasklets from NVIDIA devices to Litmus's klitirqd + or PAI interrupt handling routines. + + If unsure, say No. + +config LITMUS_AFFINITY_AWARE_GPU_ASSINGMENT + bool "Enable affinity-aware heuristics to improve GPU assignment." + depends on LITMUS_NVIDIA && LITMUS_AFFINITY_LOCKING + default n + help + Enable several heuristics to improve the assignment + of GPUs to real-time tasks to reduce the overheads + of memory migrations. + + If unsure, say No. + +config NV_DEVICE_NUM + int "Number of NVIDIA GPUs." + depends on LITMUS_SOFTIRQD || LITMUS_PAI_SOFTIRQD + range 1 4096 + default "1" + help + Should be (<= to the number of CPUs) and + (<= to the number of GPUs) in your system. + +config NV_MAX_SIMULT_USERS + int "Maximum number of threads sharing a GPU simultanously" + depends on LITMUS_SOFTIRQD || LITMUS_PAI_SOFTIRQD + range 1 3 + default "2" + help + Should be equal to the #copy_engines + #execution_engines + of the GPUs in your system. + + Scientific/Professional GPUs = 3 (ex. M2070, Quadro 6000?) + Consumer Fermi/Kepler GPUs = 2 (GTX-4xx thru -6xx) + Older = 1 (ex. GTX-2xx) + +choice + prompt "CUDA/Driver Version Support" + default CUDA_4_0 + depends on LITMUS_NVIDIA + help + Select the version of CUDA/driver to support. + +config CUDA_4_0 + bool "CUDA 4.0" + depends on LITMUS_NVIDIA + help + Support CUDA 4.0 RC2 (dev. driver version: x86_64-270.40) + +config CUDA_3_2 + bool "CUDA 3.2" + depends on LITMUS_NVIDIA + help + Support CUDA 3.2 (dev. driver version: x86_64-260.24) + +endchoice + +endmenu + +endmenu diff --git a/litmus/Makefile b/litmus/Makefile new file mode 100644 index 0000000..080cbf6 --- /dev/null +++ b/litmus/Makefile @@ -0,0 +1,38 @@ +# +# Makefile for LITMUS^RT +# + +obj-y = sched_plugin.o litmus.o \ + preempt.o \ + litmus_proc.o \ + budget.o \ + clustered.o \ + jobs.o \ + sync.o \ + rt_domain.o \ + edf_common.o \ + fdso.o \ + locking.o \ + srp.o \ + bheap.o \ + binheap.o \ + ctrldev.o \ + sched_gsn_edf.o \ + sched_psn_edf.o \ + kfmlp_lock.o + +obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o +obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o +obj-$(CONFIG_SCHED_CPU_AFFINITY) += affinity.o + +obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o +obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o +obj-$(CONFIG_SCHED_DEBUG_TRACE) += sched_trace.o +obj-$(CONFIG_SCHED_OVERHEAD_TRACE) += trace.o + +obj-$(CONFIG_LITMUS_NESTED_LOCKING) += rsm_lock.o ikglp_lock.o +obj-$(CONFIG_LITMUS_SOFTIRQD) += litmus_softirq.o +obj-$(CONFIG_LITMUS_PAI_SOFTIRQD) += litmus_pai_softirq.o +obj-$(CONFIG_LITMUS_NVIDIA) += nvidia_info.o sched_trace_external.o + +obj-$(CONFIG_LITMUS_AFFINITY_LOCKING) += kexclu_affinity.o gpu_affinity.o diff --git a/litmus/affinity.c b/litmus/affinity.c new file mode 100644 index 0000000..cd93249 --- /dev/null +++ b/litmus/affinity.c @@ -0,0 +1,42 @@ +#include + +#include + +struct neighborhood neigh_info[NR_CPUS]; + +/* called by _init_litmus() */ +void init_topology(void) { + int cpu; + int i; + int chk; + int depth = num_cache_leaves; + + if (depth > NUM_CACHE_LEVELS) + depth = NUM_CACHE_LEVELS; + + for_each_online_cpu(cpu) { + for (i = 0; i < depth; ++i) { + chk = get_shared_cpu_map((struct cpumask *)&neigh_info[cpu].neighbors[i], cpu, i); + if (chk) { + /* failed */ + neigh_info[cpu].size[i] = 0; + } else { + /* size = num bits in mask */ + neigh_info[cpu].size[i] = + cpumask_weight((struct cpumask *)&neigh_info[cpu].neighbors[i]); + } + printk("CPU %d has %d neighbors at level %d. (mask = %lx)\n", + cpu, neigh_info[cpu].size[i], i, + *cpumask_bits(neigh_info[cpu].neighbors[i])); + } + + /* set data for non-existent levels */ + for (; i < NUM_CACHE_LEVELS; ++i) { + neigh_info[cpu].size[i] = 0; + + printk("CPU %d has %d neighbors at level %d. (mask = %lx)\n", + cpu, neigh_info[cpu].size[i], i, 0lu); + } + } +} + diff --git a/litmus/bheap.c b/litmus/bheap.c new file mode 100644 index 0000000..528af97 --- /dev/null +++ b/litmus/bheap.c @@ -0,0 +1,314 @@ +#include "linux/kernel.h" +#include "litmus/bheap.h" + +void bheap_init(struct bheap* heap) +{ + heap->head = NULL; + heap->min = NULL; +} + +void bheap_node_init(struct bheap_node** _h, void* value) +{ + struct bheap_node* h = *_h; + h->parent = NULL; + h->next = NULL; + h->child = NULL; + h->degree = NOT_IN_HEAP; + h->value = value; + h->ref = _h; +} + + +/* make child a subtree of root */ +static void __bheap_link(struct bheap_node* root, + struct bheap_node* child) +{ + child->parent = root; + child->next = root->child; + root->child = child; + root->degree++; +} + +/* merge root lists */ +static struct bheap_node* __bheap_merge(struct bheap_node* a, + struct bheap_node* b) +{ + struct bheap_node* head = NULL; + struct bheap_node** pos = &head; + + while (a && b) { + if (a->degree < b->degree) { + *pos = a; + a = a->next; + } else { + *pos = b; + b = b->next; + } + pos = &(*pos)->next; + } + if (a) + *pos = a; + else + *pos = b; + return head; +} + +/* reverse a linked list of nodes. also clears parent pointer */ +static struct bheap_node* __bheap_reverse(struct bheap_node* h) +{ + struct bheap_node* tail = NULL; + struct bheap_node* next; + + if (!h) + return h; + + h->parent = NULL; + while (h->next) { + next = h->next; + h->next = tail; + tail = h; + h = next; + h->parent = NULL; + } + h->next = tail; + return h; +} + +static void __bheap_min(bheap_prio_t higher_prio, struct bheap* heap, + struct bheap_node** prev, struct bheap_node** node) +{ + struct bheap_node *_prev, *cur; + *prev = NULL; + + if (!heap->head) { + *node = NULL; + return; + } + + *node = heap->head; + _prev = heap->head; + cur = heap->head->next; + while (cur) { + if (higher_prio(cur, *node)) { + *node = cur; + *prev = _prev; + } + _prev = cur; + cur = cur->next; + } +} + +static void __bheap_union(bheap_prio_t higher_prio, struct bheap* heap, + struct bheap_node* h2) +{ + struct bheap_node* h1; + struct bheap_node *prev, *x, *next; + if (!h2) + return; + h1 = heap->head; + if (!h1) { + heap->head = h2; + return; + } + h1 = __bheap_merge(h1, h2); + prev = NULL; + x = h1; + next = x->next; + while (next) { + if (x->degree != next->degree || + (next->next && next->next->degree == x->degree)) { + /* nothing to do, advance */ + prev = x; + x = next; + } else if (higher_prio(x, next)) { + /* x becomes the root of next */ + x->next = next->next; + __bheap_link(x, next); + } else { + /* next becomes the root of x */ + if (prev) + prev->next = next; + else + h1 = next; + __bheap_link(next, x); + x = next; + } + next = x->next; + } + heap->head = h1; +} + +static struct bheap_node* __bheap_extract_min(bheap_prio_t higher_prio, + struct bheap* heap) +{ + struct bheap_node *prev, *node; + __bheap_min(higher_prio, heap, &prev, &node); + if (!node) + return NULL; + if (prev) + prev->next = node->next; + else + heap->head = node->next; + __bheap_union(higher_prio, heap, __bheap_reverse(node->child)); + return node; +} + +/* insert (and reinitialize) a node into the heap */ +void bheap_insert(bheap_prio_t higher_prio, struct bheap* heap, + struct bheap_node* node) +{ + struct bheap_node *min; + node->child = NULL; + node->parent = NULL; + node->next = NULL; + node->degree = 0; + if (heap->min && higher_prio(node, heap->min)) { + /* swap min cache */ + min = heap->min; + min->child = NULL; + min->parent = NULL; + min->next = NULL; + min->degree = 0; + __bheap_union(higher_prio, heap, min); + heap->min = node; + } else + __bheap_union(higher_prio, heap, node); +} + +void bheap_uncache_min(bheap_prio_t higher_prio, struct bheap* heap) +{ + struct bheap_node* min; + if (heap->min) { + min = heap->min; + heap->min = NULL; + bheap_insert(higher_prio, heap, min); + } +} + +/* merge addition into target */ +void bheap_union(bheap_prio_t higher_prio, + struct bheap* target, struct bheap* addition) +{ + /* first insert any cached minima, if necessary */ + bheap_uncache_min(higher_prio, target); + bheap_uncache_min(higher_prio, addition); + __bheap_union(higher_prio, target, addition->head); + /* this is a destructive merge */ + addition->head = NULL; +} + +struct bheap_node* bheap_peek(bheap_prio_t higher_prio, + struct bheap* heap) +{ + if (!heap->min) + heap->min = __bheap_extract_min(higher_prio, heap); + return heap->min; +} + +struct bheap_node* bheap_take(bheap_prio_t higher_prio, + struct bheap* heap) +{ + struct bheap_node *node; + if (!heap->min) + heap->min = __bheap_extract_min(higher_prio, heap); + node = heap->min; + heap->min = NULL; + if (node) + node->degree = NOT_IN_HEAP; + return node; +} + +int bheap_decrease(bheap_prio_t higher_prio, struct bheap_node* node) +{ + struct bheap_node *parent; + struct bheap_node** tmp_ref; + void* tmp; + + /* bubble up */ + parent = node->parent; + while (parent && higher_prio(node, parent)) { + /* swap parent and node */ + tmp = parent->value; + parent->value = node->value; + node->value = tmp; + /* swap references */ + *(parent->ref) = node; + *(node->ref) = parent; + tmp_ref = parent->ref; + parent->ref = node->ref; + node->ref = tmp_ref; + /* step up */ + node = parent; + parent = node->parent; + } + + return parent != NULL; +} + +void bheap_delete(bheap_prio_t higher_prio, struct bheap* heap, + struct bheap_node* node) +{ + struct bheap_node *parent, *prev, *pos; + struct bheap_node** tmp_ref; + void* tmp; + + if (heap->min != node) { + /* bubble up */ + parent = node->parent; + while (parent) { + /* swap parent and node */ + tmp = parent->value; + parent->value = node->value; + node->value = tmp; + /* swap references */ + *(parent->ref) = node; + *(node->ref) = parent; + tmp_ref = parent->ref; + parent->ref = node->ref; + node->ref = tmp_ref; + /* step up */ + node = parent; + parent = node->parent; + } + /* now delete: + * first find prev */ + prev = NULL; + pos = heap->head; + while (pos != node) { + prev = pos; + pos = pos->next; + } + /* we have prev, now remove node */ + if (prev) + prev->next = node->next; + else + heap->head = node->next; + __bheap_union(higher_prio, heap, __bheap_reverse(node->child)); + } else + heap->min = NULL; + node->degree = NOT_IN_HEAP; +} + +/* allocate a heap node for value and insert into the heap */ +int bheap_add(bheap_prio_t higher_prio, struct bheap* heap, + void* value, int gfp_flags) +{ + struct bheap_node* hn = bheap_node_alloc(gfp_flags); + if (likely(hn)) { + bheap_node_init(&hn, value); + bheap_insert(higher_prio, heap, hn); + } + return hn != NULL; +} + +void* bheap_take_del(bheap_prio_t higher_prio, + struct bheap* heap) +{ + struct bheap_node* hn = bheap_take(higher_prio, heap); + void* ret = NULL; + if (hn) { + ret = hn->value; + bheap_node_free(hn); + } + return ret; +} diff --git a/litmus/binheap.c b/litmus/binheap.c new file mode 100644 index 0000000..8d42403 --- /dev/null +++ b/litmus/binheap.c @@ -0,0 +1,443 @@ +#include + +//extern void dump_node_data(struct binheap_node* parent, struct binheap_node* child); +//extern void dump_node_data2(struct binheap_handle *handle, struct binheap_node* bad_node); + +int binheap_is_in_this_heap(struct binheap_node *node, + struct binheap_handle* heap) +{ + if(!binheap_is_in_heap(node)) { + return 0; + } + + while(node->parent != NULL) { + node = node->parent; + } + + return (node == heap->root); +} + +/* Update the node reference pointers. Same logic as Litmus binomial heap. */ +static void __update_ref(struct binheap_node *parent, + struct binheap_node *child) +{ + *(parent->ref_ptr) = child; + *(child->ref_ptr) = parent; + + swap(parent->ref_ptr, child->ref_ptr); +} + +/* Swaps data between two nodes. */ +static void __binheap_swap(struct binheap_node *parent, + struct binheap_node *child) +{ +// if(parent == BINHEAP_POISON || child == BINHEAP_POISON) { +// dump_node_data(parent, child); +// BUG(); +// } + + swap(parent->data, child->data); + __update_ref(parent, child); +} + + +/* Swaps memory and data between two nodes. Actual nodes swap instead of + * just data. Needed when we delete nodes from the heap. + */ +static void __binheap_swap_safe(struct binheap_handle *handle, + struct binheap_node *a, + struct binheap_node *b) +{ + swap(a->data, b->data); + __update_ref(a, b); + + if((a->parent != NULL) && (a->parent == b->parent)) { + /* special case: shared parent */ + swap(a->parent->left, a->parent->right); + } + else { + /* Update pointers to swap parents. */ + + if(a->parent) { + if(a == a->parent->left) { + a->parent->left = b; + } + else { + a->parent->right = b; + } + } + + if(b->parent) { + if(b == b->parent->left) { + b->parent->left = a; + } + else { + b->parent->right = a; + } + } + + swap(a->parent, b->parent); + } + + /* swap children */ + + if(a->left) { + a->left->parent = b; + + if(a->right) { + a->right->parent = b; + } + } + + if(b->left) { + b->left->parent = a; + + if(b->right) { + b->right->parent = a; + } + } + + swap(a->left, b->left); + swap(a->right, b->right); + + + /* update next/last/root pointers */ + + if(a == handle->next) { + handle->next = b; + } + else if(b == handle->next) { + handle->next = a; + } + + if(a == handle->last) { + handle->last = b; + } + else if(b == handle->last) { + handle->last = a; + } + + if(a == handle->root) { + handle->root = b; + } + else if(b == handle->root) { + handle->root = a; + } +} + + +/** + * Update the pointer to the last node in the complete binary tree. + * Called internally after the root node has been deleted. + */ +static void __binheap_update_last(struct binheap_handle *handle) +{ + struct binheap_node *temp = handle->last; + + /* find a "bend" in the tree. */ + while(temp->parent && (temp == temp->parent->left)) { + temp = temp->parent; + } + + /* step over to sibling if we're not at root */ + if(temp->parent != NULL) { + temp = temp->parent->left; + } + + /* now travel right as far as possible. */ + while(temp->right != NULL) { + temp = temp->right; + } + + /* take one step to the left if we're not at the bottom-most level. */ + if(temp->left != NULL) { + temp = temp->left; + } + + //BUG_ON(!(temp->left == NULL && temp->right == NULL)); + + handle->last = temp; +} + +/** + * Update the pointer to the node that will take the next inserted node. + * Called internally after a node has been inserted. + */ +static void __binheap_update_next(struct binheap_handle *handle) +{ + struct binheap_node *temp = handle->next; + + /* find a "bend" in the tree. */ + while(temp->parent && (temp == temp->parent->right)) { + temp = temp->parent; + } + + /* step over to sibling if we're not at root */ + if(temp->parent != NULL) { + temp = temp->parent->right; + } + + /* now travel left as far as possible. */ + while(temp->left != NULL) { + temp = temp->left; + } + + handle->next = temp; +} + + + +/* bubble node up towards root */ +static void __binheap_bubble_up( + struct binheap_handle *handle, + struct binheap_node *node) +{ + //BUG_ON(!binheap_is_in_heap(node)); +// if(!binheap_is_in_heap(node)) +// { +// dump_node_data2(handle, node); +// BUG(); +// } + + while((node->parent != NULL) && + ((node->data == BINHEAP_POISON) /* let BINHEAP_POISON data bubble to the top */ || + handle->compare(node, node->parent))) { + __binheap_swap(node->parent, node); + node = node->parent; + +// if(!binheap_is_in_heap(node)) +// { +// dump_node_data2(handle, node); +// BUG(); +// } + } +} + + +/* bubble node down, swapping with min-child */ +static void __binheap_bubble_down(struct binheap_handle *handle) +{ + struct binheap_node *node = handle->root; + + while(node->left != NULL) { + if(node->right && handle->compare(node->right, node->left)) { + if(handle->compare(node->right, node)) { + __binheap_swap(node, node->right); + node = node->right; + } + else { + break; + } + } + else { + if(handle->compare(node->left, node)) { + __binheap_swap(node, node->left); + node = node->left; + } + else { + break; + } + } + } +} + + + +void __binheap_add(struct binheap_node *new_node, + struct binheap_handle *handle, + void *data) +{ +// if(binheap_is_in_heap(new_node)) +// { +// dump_node_data2(handle, new_node); +// BUG(); +// } + + new_node->data = data; + new_node->ref = new_node; + new_node->ref_ptr = &(new_node->ref); + + if(!binheap_empty(handle)) { + /* insert left side first */ + if(handle->next->left == NULL) { + handle->next->left = new_node; + new_node->parent = handle->next; + new_node->left = NULL; + new_node->right = NULL; + + handle->last = new_node; + + __binheap_bubble_up(handle, new_node); + } + else { + /* left occupied. insert right. */ + handle->next->right = new_node; + new_node->parent = handle->next; + new_node->left = NULL; + new_node->right = NULL; + + handle->last = new_node; + + __binheap_update_next(handle); + __binheap_bubble_up(handle, new_node); + } + } + else { + /* first node in heap */ + + new_node->parent = NULL; + new_node->left = NULL; + new_node->right = NULL; + + handle->root = new_node; + handle->next = new_node; + handle->last = new_node; + } +} + + + +/** + * Removes the root node from the heap. The node is removed after coalescing + * the binheap_node with its original data pointer at the root of the tree. + * + * The 'last' node in the tree is then swapped up to the root and bubbled + * down. + */ +void __binheap_delete_root(struct binheap_handle *handle, + struct binheap_node *container) +{ + struct binheap_node *root = handle->root; + +// if(!binheap_is_in_heap(container)) +// { +// dump_node_data2(handle, container); +// BUG(); +// } + + if(root != container) { + /* coalesce */ + __binheap_swap_safe(handle, root, container); + root = container; + } + + if(handle->last != root) { + /* swap 'last' node up to root and bubble it down. */ + + struct binheap_node *to_move = handle->last; + + if(to_move->parent != root) { + handle->next = to_move->parent; + + if(handle->next->right == to_move) { + /* disconnect from parent */ + to_move->parent->right = NULL; + handle->last = handle->next->left; + } + else { + /* find new 'last' before we disconnect */ + __binheap_update_last(handle); + + /* disconnect from parent */ + to_move->parent->left = NULL; + } + } + else { + /* 'last' is direct child of root */ + + handle->next = to_move; + + if(to_move == to_move->parent->right) { + to_move->parent->right = NULL; + handle->last = to_move->parent->left; + } + else { + to_move->parent->left = NULL; + handle->last = to_move; + } + } + to_move->parent = NULL; + + /* reconnect as root. We can't just swap data ptrs since root node + * may be freed after this function returns. + */ + to_move->left = root->left; + to_move->right = root->right; + if(to_move->left != NULL) { + to_move->left->parent = to_move; + } + if(to_move->right != NULL) { + to_move->right->parent = to_move; + } + + handle->root = to_move; + + /* bubble down */ + __binheap_bubble_down(handle); + } + else { + /* removing last node in tree */ + handle->root = NULL; + handle->next = NULL; + handle->last = NULL; + } + + /* mark as removed */ + container->parent = BINHEAP_POISON; +} + + +/** + * Delete an arbitrary node. Bubble node to delete up to the root, + * and then delete to root. + */ +void __binheap_delete(struct binheap_node *node_to_delete, + struct binheap_handle *handle) +{ + struct binheap_node *target = node_to_delete->ref; + void *temp_data = target->data; + +// if(!binheap_is_in_heap(node_to_delete)) +// { +// dump_node_data2(handle, node_to_delete); +// BUG(); +// } +// +// if(!binheap_is_in_heap(target)) +// { +// dump_node_data2(handle, target); +// BUG(); +// } + + /* temporarily set data to null to allow node to bubble up to the top. */ + target->data = BINHEAP_POISON; + + __binheap_bubble_up(handle, target); + __binheap_delete_root(handle, node_to_delete); + + node_to_delete->data = temp_data; /* restore node data pointer */ + //node_to_delete->parent = BINHEAP_POISON; /* poison the node */ +} + +/** + * Bubble up a node whose pointer has decreased in value. + */ +void __binheap_decrease(struct binheap_node *orig_node, + struct binheap_handle *handle) +{ + struct binheap_node *target = orig_node->ref; + +// if(!binheap_is_in_heap(orig_node)) +// { +// dump_node_data2(handle, orig_node); +// BUG(); +// } +// +// if(!binheap_is_in_heap(target)) +// { +// dump_node_data2(handle, target); +// BUG(); +// } +// + __binheap_bubble_up(handle, target); +} diff --git a/litmus/budget.c b/litmus/budget.c new file mode 100644 index 0000000..310e9a3 --- /dev/null +++ b/litmus/budget.c @@ -0,0 +1,111 @@ +#include +#include +#include + +#include +#include + +struct enforcement_timer { + /* The enforcement timer is used to accurately police + * slice budgets. */ + struct hrtimer timer; + int armed; +}; + +DEFINE_PER_CPU(struct enforcement_timer, budget_timer); + +static enum hrtimer_restart on_enforcement_timeout(struct hrtimer *timer) +{ + struct enforcement_timer* et = container_of(timer, + struct enforcement_timer, + timer); + unsigned long flags; + + local_irq_save(flags); + TRACE("enforcement timer fired.\n"); + et->armed = 0; + /* activate scheduler */ + litmus_reschedule_local(); + local_irq_restore(flags); + + return HRTIMER_NORESTART; +} + +/* assumes called with IRQs off */ +static void cancel_enforcement_timer(struct enforcement_timer* et) +{ + int ret; + + TRACE("cancelling enforcement timer.\n"); + + /* Since interrupts are disabled and et->armed is only + * modified locally, we do not need any locks. + */ + + if (et->armed) { + ret = hrtimer_try_to_cancel(&et->timer); + /* Should never be inactive. */ + BUG_ON(ret == 0); + /* Should never be running concurrently. */ + BUG_ON(ret == -1); + + et->armed = 0; + } +} + +/* assumes called with IRQs off */ +static void arm_enforcement_timer(struct enforcement_timer* et, + struct task_struct* t) +{ + lt_t when_to_fire; + TRACE_TASK(t, "arming enforcement timer.\n"); + + /* Calling this when there is no budget left for the task + * makes no sense, unless the task is non-preemptive. */ + BUG_ON(budget_exhausted(t) && (!is_np(t))); + + /* __hrtimer_start_range_ns() cancels the timer + * anyway, so we don't have to check whether it is still armed */ + + if (likely(!is_np(t))) { + when_to_fire = litmus_clock() + budget_remaining(t); + __hrtimer_start_range_ns(&et->timer, + ns_to_ktime(when_to_fire), + 0 /* delta */, + HRTIMER_MODE_ABS_PINNED, + 0 /* no wakeup */); + et->armed = 1; + } +} + + +/* expects to be called with IRQs off */ +void update_enforcement_timer(struct task_struct* t) +{ + struct enforcement_timer* et = &__get_cpu_var(budget_timer); + + if (t && budget_precisely_enforced(t)) { + /* Make sure we call into the scheduler when this budget + * expires. */ + arm_enforcement_timer(et, t); + } else if (et->armed) { + /* Make sure we don't cause unnecessary interrupts. */ + cancel_enforcement_timer(et); + } +} + + +static int __init init_budget_enforcement(void) +{ + int cpu; + struct enforcement_timer* et; + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + et = &per_cpu(budget_timer, cpu); + hrtimer_init(&et->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + et->timer.function = on_enforcement_timeout; + } + return 0; +} + +module_init(init_budget_enforcement); diff --git a/litmus/clustered.c b/litmus/clustered.c new file mode 100644 index 0000000..6fe1b51 --- /dev/null +++ b/litmus/clustered.c @@ -0,0 +1,111 @@ +#include +#include +#include + +#include + +#ifndef CONFIG_X86 +/* fake get_shared_cpu_map() on non-x86 architectures */ + +int get_shared_cpu_map(cpumask_var_t mask, unsigned int cpu, int index) +{ + if (index != 1) + return 1; + else { + /* Fake L1: CPU is all by itself. */ + cpumask_clear(mask); + cpumask_set_cpu(cpu, mask); + return 0; + } +} + +#endif + +int get_cluster_size(enum cache_level level) +{ + cpumask_var_t mask; + int ok; + int num_cpus; + + if (level == GLOBAL_CLUSTER) + return num_online_cpus(); + else { + if (!zalloc_cpumask_var(&mask, GFP_ATOMIC)) + return -ENOMEM; + /* assumes CPU 0 is representative of all CPUs */ + ok = get_shared_cpu_map(mask, 0, level); + /* ok == 0 means we got the map; otherwise it's an invalid cache level */ + if (ok == 0) + num_cpus = cpumask_weight(mask); + free_cpumask_var(mask); + + if (ok == 0) + return num_cpus; + else + return -EINVAL; + } +} + +int assign_cpus_to_clusters(enum cache_level level, + struct scheduling_cluster* clusters[], + unsigned int num_clusters, + struct cluster_cpu* cpus[], + unsigned int num_cpus) +{ + cpumask_var_t mask; + unsigned int i, free_cluster = 0, low_cpu; + int err = 0; + + if (!zalloc_cpumask_var(&mask, GFP_ATOMIC)) + return -ENOMEM; + + /* clear cluster pointers */ + for (i = 0; i < num_cpus; i++) { + cpus[i]->id = i; + cpus[i]->cluster = NULL; + } + + /* initialize clusters */ + for (i = 0; i < num_clusters; i++) { + clusters[i]->id = i; + INIT_LIST_HEAD(&clusters[i]->cpus); + } + + /* Assign each CPU. Two assumtions are made: + * 1) The index of a cpu in cpus corresponds to its processor id (i.e., the index in a cpu mask). + * 2) All cpus that belong to some cluster are online. + */ + for_each_online_cpu(i) { + /* get lowest-id CPU in cluster */ + if (level != GLOBAL_CLUSTER) { + err = get_shared_cpu_map(mask, cpus[i]->id, level); + if (err != 0) { + /* ugh... wrong cache level? Either caller screwed up + * or the CPU topology is weird. */ + printk(KERN_ERR "Could not set up clusters for L%d sharing (max: L%d).\n", + level, err); + err = -EINVAL; + goto out; + } + low_cpu = cpumask_first(mask); + } else + low_cpu = 0; + if (low_cpu == i) { + /* caller must provide an appropriate number of clusters */ + BUG_ON(free_cluster >= num_clusters); + + /* create new cluster */ + cpus[i]->cluster = clusters[free_cluster++]; + } else { + /* low_cpu points to the right cluster + * Assumption: low_cpu is actually online and was processed earlier. */ + cpus[i]->cluster = cpus[low_cpu]->cluster; + } + /* enqueue in cpus list */ + list_add_tail(&cpus[i]->cluster_list, &cpus[i]->cluster->cpus); + printk(KERN_INFO "Assigning CPU%u to cluster %u\n.", i, cpus[i]->cluster->id); + } +out: + free_cpumask_var(mask); + return err; +} diff --git a/litmus/ctrldev.c b/litmus/ctrldev.c new file mode 100644 index 0000000..6677a67 --- /dev/null +++ b/litmus/ctrldev.c @@ -0,0 +1,150 @@ +#include +#include +#include +#include +#include + +#include + +/* only one page for now, but we might want to add a RO version at some point */ + +#define CTRL_NAME "litmus/ctrl" + +/* allocate t->rt_param.ctrl_page*/ +static int alloc_ctrl_page(struct task_struct *t) +{ + int err = 0; + + /* only allocate if the task doesn't have one yet */ + if (!tsk_rt(t)->ctrl_page) { + tsk_rt(t)->ctrl_page = (void*) get_zeroed_page(GFP_KERNEL); + if (!tsk_rt(t)->ctrl_page) + err = -ENOMEM; + /* will get de-allocated in task teardown */ + TRACE_TASK(t, "%s ctrl_page = %p\n", __FUNCTION__, + tsk_rt(t)->ctrl_page); + } + return err; +} + +static int map_ctrl_page(struct task_struct *t, struct vm_area_struct* vma) +{ + int err; + unsigned long pfn; + + struct page* ctrl = virt_to_page(tsk_rt(t)->ctrl_page); + + /* Increase ref count. Is decreased when vma is destroyed. */ + get_page(ctrl); + + /* compute page frame number */ + pfn = page_to_pfn(ctrl); + + TRACE_CUR(CTRL_NAME + ": mapping %p (pfn:%lx, %lx) to 0x%lx (prot:%lx)\n", + tsk_rt(t)->ctrl_page, pfn, page_to_pfn(ctrl), vma->vm_start, + vma->vm_page_prot); + + /* Map it into the vma. Make sure to use PAGE_SHARED, otherwise + * userspace actually gets a copy-on-write page. */ + err = remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, PAGE_SHARED); + + if (err) + TRACE_CUR(CTRL_NAME ": remap_pfn_range() failed (%d)\n", err); + + return err; +} + +static void litmus_ctrl_vm_close(struct vm_area_struct* vma) +{ + TRACE_CUR("%s flags=0x%x prot=0x%x\n", __FUNCTION__, + vma->vm_flags, vma->vm_page_prot); + + TRACE_CUR(CTRL_NAME + ": %p:%p vma:%p vma->vm_private_data:%p closed.\n", + (void*) vma->vm_start, (void*) vma->vm_end, vma, + vma->vm_private_data, current->comm, + current->pid); +} + +static int litmus_ctrl_vm_fault(struct vm_area_struct* vma, + struct vm_fault* vmf) +{ + /* This function should never be called, since + * all pages should have been mapped by mmap() + * already. */ + TRACE_CUR("%s flags=0x%x\n", __FUNCTION__, vma->vm_flags); + + /* nope, you only get one page */ + return VM_FAULT_SIGBUS; +} + +static struct vm_operations_struct litmus_ctrl_vm_ops = { + .close = litmus_ctrl_vm_close, + .fault = litmus_ctrl_vm_fault, +}; + +static int litmus_ctrl_mmap(struct file* filp, struct vm_area_struct* vma) +{ + int err = 0; + + /* first make sure mapper knows what he's doing */ + + /* you can only get one page */ + if (vma->vm_end - vma->vm_start != PAGE_SIZE) + return -EINVAL; + + /* you can only map the "first" page */ + if (vma->vm_pgoff != 0) + return -EINVAL; + + /* you can't share it with anyone */ + if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED)) + return -EINVAL; + + vma->vm_ops = &litmus_ctrl_vm_ops; + /* this mapping should not be kept across forks, + * and cannot be expanded */ + vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND; + + err = alloc_ctrl_page(current); + if (!err) + err = map_ctrl_page(current, vma); + + TRACE_CUR("%s flags=0x%x prot=0x%lx\n", + __FUNCTION__, vma->vm_flags, vma->vm_page_prot); + + return err; +} + +static struct file_operations litmus_ctrl_fops = { + .owner = THIS_MODULE, + .mmap = litmus_ctrl_mmap, +}; + +static struct miscdevice litmus_ctrl_dev = { + .name = CTRL_NAME, + .minor = MISC_DYNAMIC_MINOR, + .fops = &litmus_ctrl_fops, +}; + +static int __init init_litmus_ctrl_dev(void) +{ + int err; + + BUILD_BUG_ON(sizeof(struct control_page) > PAGE_SIZE); + + printk("Initializing LITMUS^RT control device.\n"); + err = misc_register(&litmus_ctrl_dev); + if (err) + printk("Could not allocate %s device (%d).\n", CTRL_NAME, err); + return err; +} + +static void __exit exit_litmus_ctrl_dev(void) +{ + misc_deregister(&litmus_ctrl_dev); +} + +module_init(init_litmus_ctrl_dev); +module_exit(exit_litmus_ctrl_dev); diff --git a/litmus/edf_common.c b/litmus/edf_common.c new file mode 100644 index 0000000..b346bdd --- /dev/null +++ b/litmus/edf_common.c @@ -0,0 +1,211 @@ +/* + * kernel/edf_common.c + * + * Common functions for EDF based scheduler. + */ + +#include +#include +#include + +#include +#include +#include + +#ifdef CONFIG_LITMUS_NESTED_LOCKING +#include +#endif + +#include + + + +/* edf_higher_prio - returns true if first has a higher EDF priority + * than second. Deadline ties are broken by PID. + * + * both first and second may be NULL + */ +#ifdef CONFIG_LITMUS_NESTED_LOCKING +int __edf_higher_prio( + struct task_struct* first, comparison_mode_t first_mode, + struct task_struct* second, comparison_mode_t second_mode) +#else +int edf_higher_prio(struct task_struct* first, struct task_struct* second) +#endif +{ + struct task_struct *first_task = first; + struct task_struct *second_task = second; + + /* There is no point in comparing a task to itself. */ + if (first && first == second) { + TRACE_CUR("WARNING: pointless edf priority comparison: %s/%d\n", first->comm, first->pid); + WARN_ON(1); + return 0; + } + + + /* check for NULL tasks */ + if (!first || !second) { + return first && !second; + } + +#ifdef CONFIG_LITMUS_LOCKING + /* Check for EFFECTIVE priorities. Change task + * used for comparison in such a case. + */ + if (unlikely(first->rt_param.inh_task) +#ifdef CONFIG_LITMUS_NESTED_LOCKING + && (first_mode == EFFECTIVE) +#endif + ) { + first_task = first->rt_param.inh_task; + } + if (unlikely(second->rt_param.inh_task) +#ifdef CONFIG_LITMUS_NESTED_LOCKING + && (second_mode == EFFECTIVE) +#endif + ) { + second_task = second->rt_param.inh_task; + } + + /* Check for priority boosting. Tie-break by start of boosting. + */ + if (unlikely(is_priority_boosted(first_task))) { + /* first_task is boosted, how about second_task? */ + if (!is_priority_boosted(second_task) || + lt_before(get_boost_start(first_task), + get_boost_start(second_task))) { + return 1; + } + else { + return 0; + } + } + else if (unlikely(is_priority_boosted(second_task))) { + /* second_task is boosted, first is not*/ + return 0; + } + +#endif + +// // rate-monotonic for testing +// if (!is_realtime(second_task)) { +// return true; +// } +// +// if (shorter_period(first_task, second_task)) { +// return true; +// } +// +// if (get_period(first_task) == get_period(second_task)) { +// if (first_task->pid < second_task->pid) { +// return true; +// } +// else if (first_task->pid == second_task->pid) { +// return !second->rt_param.inh_task; +// } +// } + + if (!is_realtime(second_task)) { + return true; + } + + if (earlier_deadline(first_task, second_task)) { + return true; + } + if (get_deadline(first_task) == get_deadline(second_task)) { + + if (shorter_period(first_task, second_task)) { + return true; + } + if (get_rt_period(first_task) == get_rt_period(second_task)) { + if (first_task->pid < second_task->pid) { + return true; + } + if (first_task->pid == second_task->pid) { +#ifdef CONFIG_LITMUS_SOFTIRQD + if (first_task->rt_param.is_proxy_thread < + second_task->rt_param.is_proxy_thread) { + return true; + } + if(first_task->rt_param.is_proxy_thread == second_task->rt_param.is_proxy_thread) { + return !second->rt_param.inh_task; + } +#else + return !second->rt_param.inh_task; +#endif + } + + } + } + + return false; +} + + +#ifdef CONFIG_LITMUS_NESTED_LOCKING +int edf_higher_prio(struct task_struct* first, struct task_struct* second) +{ + return __edf_higher_prio(first, EFFECTIVE, second, EFFECTIVE); +} + +int edf_max_heap_order(struct binheap_node *a, struct binheap_node *b) +{ + struct nested_info *l_a = (struct nested_info *)binheap_entry(a, struct nested_info, hp_binheap_node); + struct nested_info *l_b = (struct nested_info *)binheap_entry(b, struct nested_info, hp_binheap_node); + + return __edf_higher_prio(l_a->hp_waiter_eff_prio, EFFECTIVE, l_b->hp_waiter_eff_prio, EFFECTIVE); +} + +int edf_min_heap_order(struct binheap_node *a, struct binheap_node *b) +{ + return edf_max_heap_order(b, a); // swap comparison +} + +int edf_max_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b) +{ + struct nested_info *l_a = (struct nested_info *)binheap_entry(a, struct nested_info, hp_binheap_node); + struct nested_info *l_b = (struct nested_info *)binheap_entry(b, struct nested_info, hp_binheap_node); + + return __edf_higher_prio(l_a->hp_waiter_eff_prio, BASE, l_b->hp_waiter_eff_prio, BASE); +} + +int edf_min_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b) +{ + return edf_max_heap_base_priority_order(b, a); // swap comparison +} +#endif + + +int edf_ready_order(struct bheap_node* a, struct bheap_node* b) +{ + return edf_higher_prio(bheap2task(a), bheap2task(b)); +} + +void edf_domain_init(rt_domain_t* rt, check_resched_needed_t resched, + release_jobs_t release) +{ + rt_domain_init(rt, edf_ready_order, resched, release); +} + +/* need_to_preempt - check whether the task t needs to be preempted + * call only with irqs disabled and with ready_lock acquired + * THIS DOES NOT TAKE NON-PREEMPTIVE SECTIONS INTO ACCOUNT! + */ +int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t) +{ + /* we need the read lock for edf_ready_queue */ + /* no need to preempt if there is nothing pending */ + if (!__jobs_pending(rt)) + return 0; + /* we need to reschedule if t doesn't exist */ + if (!t) + return 1; + + /* NOTE: We cannot check for non-preemptibility since we + * don't know what address space we're currently in. + */ + + /* make sure to get non-rt stuff out of the way */ + return !is_realtime(t) || edf_higher_prio(__next_ready(rt), t); +} diff --git a/litmus/fdso.c b/litmus/fdso.c new file mode 100644 index 0000000..18fc61b --- /dev/null +++ b/litmus/fdso.c @@ -0,0 +1,306 @@ +/* fdso.c - file descriptor attached shared objects + * + * (c) 2007 B. Brandenburg, LITMUS^RT project + * + * Notes: + * - objects descriptor (OD) tables are not cloned during a fork. + * - objects are created on-demand, and freed after the last reference + * is dropped. + * - for now, object types are hard coded. + * - As long as we have live objects, we keep a reference to the inode. + */ + +#include +#include +#include +#include +#include + +#include + +extern struct fdso_ops generic_lock_ops; + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING +extern struct fdso_ops generic_affinity_ops; +#endif + +static const struct fdso_ops* fdso_ops[] = { + &generic_lock_ops, /* FMLP_SEM */ + &generic_lock_ops, /* SRP_SEM */ + &generic_lock_ops, /* RSM_MUTEX */ + &generic_lock_ops, /* IKGLP_SEM */ + &generic_lock_ops, /* KFMLP_SEM */ +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + &generic_affinity_ops, /* IKGLP_SIMPLE_GPU_AFF_OBS */ + &generic_affinity_ops, /* IKGLP_GPU_AFF_OBS */ + &generic_affinity_ops, /* KFMLP_SIMPLE_GPU_AFF_OBS */ + &generic_affinity_ops, /* KFMLP_GPU_AFF_OBS */ +#endif +}; + +static int fdso_create(void** obj_ref, obj_type_t type, void* __user config) +{ + if (fdso_ops[type]->create) + return fdso_ops[type]->create(obj_ref, type, config); + else + return -EINVAL; +} + +static void fdso_destroy(obj_type_t type, void* obj) +{ + fdso_ops[type]->destroy(type, obj); +} + +static int fdso_open(struct od_table_entry* entry, void* __user config) +{ + if (fdso_ops[entry->obj->type]->open) + return fdso_ops[entry->obj->type]->open(entry, config); + else + return 0; +} + +static int fdso_close(struct od_table_entry* entry) +{ + if (fdso_ops[entry->obj->type]->close) + return fdso_ops[entry->obj->type]->close(entry); + else + return 0; +} + +/* inode must be locked already */ +static int alloc_inode_obj(struct inode_obj_id** obj_ref, + struct inode* inode, + obj_type_t type, + unsigned int id, + void* __user config) +{ + struct inode_obj_id* obj; + void* raw_obj; + int err; + + obj = kmalloc(sizeof(*obj), GFP_KERNEL); + if (!obj) { + return -ENOMEM; + } + + err = fdso_create(&raw_obj, type, config); + if (err != 0) { + kfree(obj); + return err; + } + + INIT_LIST_HEAD(&obj->list); + atomic_set(&obj->count, 1); + obj->type = type; + obj->id = id; + obj->obj = raw_obj; + obj->inode = inode; + + list_add(&obj->list, &inode->i_obj_list); + atomic_inc(&inode->i_count); + + printk(KERN_DEBUG "alloc_inode_obj(%p, %d, %d): object created\n", inode, type, id); + + *obj_ref = obj; + return 0; +} + +/* inode must be locked already */ +static struct inode_obj_id* get_inode_obj(struct inode* inode, + obj_type_t type, + unsigned int id) +{ + struct list_head* pos; + struct inode_obj_id* obj = NULL; + + list_for_each(pos, &inode->i_obj_list) { + obj = list_entry(pos, struct inode_obj_id, list); + if (obj->id == id && obj->type == type) { + atomic_inc(&obj->count); + return obj; + } + } + printk(KERN_DEBUG "get_inode_obj(%p, %d, %d): couldn't find object\n", inode, type, id); + return NULL; +} + + +static void put_inode_obj(struct inode_obj_id* obj) +{ + struct inode* inode; + int let_go = 0; + + inode = obj->inode; + if (atomic_dec_and_test(&obj->count)) { + + mutex_lock(&inode->i_obj_mutex); + /* no new references can be obtained */ + if (!atomic_read(&obj->count)) { + list_del(&obj->list); + fdso_destroy(obj->type, obj->obj); + kfree(obj); + let_go = 1; + } + mutex_unlock(&inode->i_obj_mutex); + if (let_go) + iput(inode); + } +} + +static struct od_table_entry* get_od_entry(struct task_struct* t) +{ + struct od_table_entry* table; + int i; + + + table = t->od_table; + if (!table) { + table = kzalloc(sizeof(*table) * MAX_OBJECT_DESCRIPTORS, + GFP_KERNEL); + t->od_table = table; + } + + for (i = 0; table && i < MAX_OBJECT_DESCRIPTORS; i++) + if (!table[i].used) { + table[i].used = 1; + return table + i; + } + return NULL; +} + +static int put_od_entry(struct od_table_entry* od) +{ + put_inode_obj(od->obj); + od->used = 0; + return 0; +} + +void exit_od_table(struct task_struct* t) +{ + int i; + + if (t->od_table) { + for (i = 0; i < MAX_OBJECT_DESCRIPTORS; i++) + if (t->od_table[i].used) + put_od_entry(t->od_table + i); + kfree(t->od_table); + t->od_table = NULL; + } +} + +static int do_sys_od_open(struct file* file, obj_type_t type, int id, + void* __user config) +{ + int idx = 0, err = 0; + struct inode* inode; + struct inode_obj_id* obj = NULL; + struct od_table_entry* entry; + + inode = file->f_dentry->d_inode; + + entry = get_od_entry(current); + if (!entry) + return -ENOMEM; + + mutex_lock(&inode->i_obj_mutex); + obj = get_inode_obj(inode, type, id); + if (!obj) + err = alloc_inode_obj(&obj, inode, type, id, config); + if (err != 0) { + obj = NULL; + idx = err; + entry->used = 0; + } else { + entry->obj = obj; + entry->class = fdso_ops[type]; + idx = entry - current->od_table; + } + + mutex_unlock(&inode->i_obj_mutex); + + /* open only if creation succeeded */ + if (!err) + err = fdso_open(entry, config); + if (err < 0) { + /* The class rejected the open call. + * We need to clean up and tell user space. + */ + if (obj) + put_od_entry(entry); + idx = err; + } + + return idx; +} + + +struct od_table_entry* get_entry_for_od(int od) +{ + struct task_struct *t = current; + + if (!t->od_table) + return NULL; + if (od < 0 || od >= MAX_OBJECT_DESCRIPTORS) + return NULL; + if (!t->od_table[od].used) + return NULL; + return t->od_table + od; +} + + +asmlinkage long sys_od_open(int fd, int type, int obj_id, void* __user config) +{ + int ret = 0; + struct file* file; + + /* + 1) get file from fd, get inode from file + 2) lock inode + 3) try to lookup object + 4) if not present create and enqueue object, inc inode refcnt + 5) increment refcnt of object + 6) alloc od_table_entry, setup ptrs + 7) unlock inode + 8) return offset in od_table as OD + */ + + if (type < MIN_OBJ_TYPE || type > MAX_OBJ_TYPE) { + ret = -EINVAL; + goto out; + } + + file = fget(fd); + if (!file) { + ret = -EBADF; + goto out; + } + + ret = do_sys_od_open(file, type, obj_id, config); + + fput(file); + +out: + return ret; +} + + +asmlinkage long sys_od_close(int od) +{ + int ret = -EINVAL; + struct task_struct *t = current; + + if (od < 0 || od >= MAX_OBJECT_DESCRIPTORS) + return ret; + + if (!t->od_table || !t->od_table[od].used) + return ret; + + + /* give the class a chance to reject the close + */ + ret = fdso_close(t->od_table + od); + if (ret == 0) + ret = put_od_entry(t->od_table + od); + + return ret; +} diff --git a/litmus/ft_event.c b/litmus/ft_event.c new file mode 100644 index 0000000..399a07b --- /dev/null +++ b/litmus/ft_event.c @@ -0,0 +1,43 @@ +#include + +#include + +#if !defined(CONFIG_ARCH_HAS_FEATHER_TRACE) || defined(CONFIG_DEBUG_RODATA) +/* provide dummy implementation */ + +int ft_events[MAX_EVENTS]; + +int ft_enable_event(unsigned long id) +{ + if (id < MAX_EVENTS) { + ft_events[id]++; + return 1; + } else + return 0; +} + +int ft_disable_event(unsigned long id) +{ + if (id < MAX_EVENTS && ft_events[id]) { + ft_events[id]--; + return 1; + } else + return 0; +} + +int ft_disable_all_events(void) +{ + int i; + + for (i = 0; i < MAX_EVENTS; i++) + ft_events[i] = 0; + + return MAX_EVENTS; +} + +int ft_is_event_enabled(unsigned long id) +{ + return id < MAX_EVENTS && ft_events[id]; +} + +#endif diff --git a/litmus/ftdev.c b/litmus/ftdev.c new file mode 100644 index 0000000..06fcf4c --- /dev/null +++ b/litmus/ftdev.c @@ -0,0 +1,439 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +struct ft_buffer* alloc_ft_buffer(unsigned int count, size_t size) +{ + struct ft_buffer* buf; + size_t total = (size + 1) * count; + char* mem; + int order = 0, pages = 1; + + buf = kmalloc(sizeof(*buf), GFP_KERNEL); + if (!buf) + return NULL; + + total = (total / PAGE_SIZE) + (total % PAGE_SIZE != 0); + while (pages < total) { + order++; + pages *= 2; + } + + mem = (char*) __get_free_pages(GFP_KERNEL, order); + if (!mem) { + kfree(buf); + return NULL; + } + + if (!init_ft_buffer(buf, count, size, + mem + (count * size), /* markers at the end */ + mem)) { /* buffer objects */ + free_pages((unsigned long) mem, order); + kfree(buf); + return NULL; + } + return buf; +} + +void free_ft_buffer(struct ft_buffer* buf) +{ + int order = 0, pages = 1; + size_t total; + + if (buf) { + total = (buf->slot_size + 1) * buf->slot_count; + total = (total / PAGE_SIZE) + (total % PAGE_SIZE != 0); + while (pages < total) { + order++; + pages *= 2; + } + free_pages((unsigned long) buf->buffer_mem, order); + kfree(buf); + } +} + +struct ftdev_event { + int id; + struct ftdev_event* next; +}; + +static int activate(struct ftdev_event** chain, int id) +{ + struct ftdev_event* ev = kmalloc(sizeof(*ev), GFP_KERNEL); + if (ev) { + printk(KERN_INFO + "Enabling feather-trace event %d.\n", (int) id); + ft_enable_event(id); + ev->id = id; + ev->next = *chain; + *chain = ev; + } + return ev ? 0 : -ENOMEM; +} + +static void deactivate(struct ftdev_event** chain, int id) +{ + struct ftdev_event **cur = chain; + struct ftdev_event *nxt; + while (*cur) { + if ((*cur)->id == id) { + nxt = (*cur)->next; + kfree(*cur); + *cur = nxt; + printk(KERN_INFO + "Disabling feather-trace event %d.\n", (int) id); + ft_disable_event(id); + break; + } + cur = &(*cur)->next; + } +} + +static int ftdev_open(struct inode *in, struct file *filp) +{ + struct ftdev* ftdev; + struct ftdev_minor* ftdm; + unsigned int buf_idx = iminor(in); + int err = 0; + + ftdev = container_of(in->i_cdev, struct ftdev, cdev); + + if (buf_idx >= ftdev->minor_cnt) { + err = -ENODEV; + goto out; + } + if (ftdev->can_open && (err = ftdev->can_open(ftdev, buf_idx))) + goto out; + + ftdm = ftdev->minor + buf_idx; + ftdm->ftdev = ftdev; + filp->private_data = ftdm; + + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; + } + + if (!ftdm->readers && ftdev->alloc) + err = ftdev->alloc(ftdev, buf_idx); + if (0 == err) + ftdm->readers++; + + mutex_unlock(&ftdm->lock); +out: + return err; +} + +static int ftdev_release(struct inode *in, struct file *filp) +{ + struct ftdev* ftdev; + struct ftdev_minor* ftdm; + unsigned int buf_idx = iminor(in); + int err = 0; + + ftdev = container_of(in->i_cdev, struct ftdev, cdev); + + if (buf_idx >= ftdev->minor_cnt) { + err = -ENODEV; + goto out; + } + ftdm = ftdev->minor + buf_idx; + + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; + } + + if (ftdm->readers == 1) { + while (ftdm->events) + deactivate(&ftdm->events, ftdm->events->id); + + /* wait for any pending events to complete */ + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(HZ); + + printk(KERN_ALERT "Failed trace writes: %u\n", + ftdm->buf->failed_writes); + + if (ftdev->free) + ftdev->free(ftdev, buf_idx); + } + + ftdm->readers--; + mutex_unlock(&ftdm->lock); +out: + return err; +} + +/* based on ft_buffer_read + * @returns < 0 : page fault + * = 0 : no data available + * = 1 : one slot copied + */ +static int ft_buffer_copy_to_user(struct ft_buffer* buf, char __user *dest) +{ + unsigned int idx; + int err = 0; + if (buf->free_count != buf->slot_count) { + /* data available */ + idx = buf->read_idx % buf->slot_count; + if (buf->slots[idx] == SLOT_READY) { + err = copy_to_user(dest, ((char*) buf->buffer_mem) + + idx * buf->slot_size, + buf->slot_size); + if (err == 0) { + /* copy ok */ + buf->slots[idx] = SLOT_FREE; + buf->read_idx++; + fetch_and_inc(&buf->free_count); + err = 1; + } + } + } + return err; +} + +static ssize_t ftdev_read(struct file *filp, + char __user *to, size_t len, loff_t *f_pos) +{ + /* we ignore f_pos, this is strictly sequential */ + + ssize_t err = 0; + size_t chunk; + int copied; + struct ftdev_minor* ftdm = filp->private_data; + + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; + } + + + chunk = ftdm->buf->slot_size; + while (len >= chunk) { + copied = ft_buffer_copy_to_user(ftdm->buf, to); + if (copied == 1) { + len -= chunk; + to += chunk; + err += chunk; + } else if (err == 0 && copied == 0 && ftdm->events) { + /* Only wait if there are any events enabled and only + * if we haven't copied some data yet. We cannot wait + * here with copied data because that data would get + * lost if the task is interrupted (e.g., killed). + */ + set_current_state(TASK_INTERRUPTIBLE); + schedule_timeout(50); + if (signal_pending(current)) { + if (err == 0) + /* nothing read yet, signal problem */ + err = -ERESTARTSYS; + break; + } + } else if (copied < 0) { + /* page fault */ + err = copied; + break; + } else + /* nothing left to get, return to user space */ + break; + } + mutex_unlock(&ftdm->lock); +out: + return err; +} + +static long ftdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + long err = -ENOIOCTLCMD; + struct ftdev_minor* ftdm = filp->private_data; + + if (mutex_lock_interruptible(&ftdm->lock)) { + err = -ERESTARTSYS; + goto out; + } + + /* FIXME: check id against list of acceptable events */ + + switch (cmd) { + case FTDEV_ENABLE_CMD: + if (activate(&ftdm->events, arg)) + err = -ENOMEM; + else + err = 0; + break; + + case FTDEV_DISABLE_CMD: + deactivate(&ftdm->events, arg); + err = 0; + break; + + default: + printk(KERN_DEBUG "ftdev: strange ioctl (%u, %lu)\n", cmd, arg); + }; + + mutex_unlock(&ftdm->lock); +out: + return err; +} + +static ssize_t ftdev_write(struct file *filp, const char __user *from, + size_t len, loff_t *f_pos) +{ + struct ftdev_minor* ftdm = filp->private_data; + ssize_t err = -EINVAL; + struct ftdev* ftdev = ftdm->ftdev; + + /* dispatch write to buffer-specific code, if available */ + if (ftdev->write) + err = ftdev->write(ftdm->buf, len, from); + + return err; +} + +struct file_operations ftdev_fops = { + .owner = THIS_MODULE, + .open = ftdev_open, + .release = ftdev_release, + .write = ftdev_write, + .read = ftdev_read, + .unlocked_ioctl = ftdev_ioctl, +}; + +int ftdev_init( struct ftdev* ftdev, struct module* owner, + const int minor_cnt, const char* name) +{ + int i, err; + + BUG_ON(minor_cnt < 1); + + cdev_init(&ftdev->cdev, &ftdev_fops); + ftdev->name = name; + ftdev->minor_cnt = minor_cnt; + ftdev->cdev.owner = owner; + ftdev->cdev.ops = &ftdev_fops; + ftdev->alloc = NULL; + ftdev->free = NULL; + ftdev->can_open = NULL; + ftdev->write = NULL; + + ftdev->minor = kcalloc(ftdev->minor_cnt, sizeof(*ftdev->minor), + GFP_KERNEL); + if (!ftdev->minor) { + printk(KERN_WARNING "ftdev(%s): Could not allocate memory\n", + ftdev->name); + err = -ENOMEM; + goto err_out; + } + + for (i = 0; i < ftdev->minor_cnt; i++) { + mutex_init(&ftdev->minor[i].lock); + ftdev->minor[i].readers = 0; + ftdev->minor[i].buf = NULL; + ftdev->minor[i].events = NULL; + } + + ftdev->class = class_create(owner, ftdev->name); + if (IS_ERR(ftdev->class)) { + err = PTR_ERR(ftdev->class); + printk(KERN_WARNING "ftdev(%s): " + "Could not create device class.\n", ftdev->name); + goto err_dealloc; + } + + return 0; + +err_dealloc: + kfree(ftdev->minor); +err_out: + return err; +} + +/* + * Destroy minor devices up to, but not including, up_to. + */ +static void ftdev_device_destroy(struct ftdev* ftdev, unsigned int up_to) +{ + dev_t minor_cntr; + + if (up_to < 1) + up_to = (ftdev->minor_cnt < 1) ? 0 : ftdev->minor_cnt; + + for (minor_cntr = 0; minor_cntr < up_to; ++minor_cntr) + device_destroy(ftdev->class, MKDEV(ftdev->major, minor_cntr)); +} + +void ftdev_exit(struct ftdev* ftdev) +{ + printk("ftdev(%s): Exiting\n", ftdev->name); + ftdev_device_destroy(ftdev, -1); + cdev_del(&ftdev->cdev); + unregister_chrdev_region(MKDEV(ftdev->major, 0), ftdev->minor_cnt); + class_destroy(ftdev->class); + kfree(ftdev->minor); +} + +int register_ftdev(struct ftdev* ftdev) +{ + struct device **device; + dev_t trace_dev_tmp, minor_cntr; + int err; + + err = alloc_chrdev_region(&trace_dev_tmp, 0, ftdev->minor_cnt, + ftdev->name); + if (err) { + printk(KERN_WARNING "ftdev(%s): " + "Could not allocate char. device region (%d minors)\n", + ftdev->name, ftdev->minor_cnt); + goto err_out; + } + + ftdev->major = MAJOR(trace_dev_tmp); + + err = cdev_add(&ftdev->cdev, trace_dev_tmp, ftdev->minor_cnt); + if (err) { + printk(KERN_WARNING "ftdev(%s): " + "Could not add cdev for major %u with %u minor(s).\n", + ftdev->name, ftdev->major, ftdev->minor_cnt); + goto err_unregister; + } + + /* create the minor device(s) */ + for (minor_cntr = 0; minor_cntr < ftdev->minor_cnt; ++minor_cntr) + { + trace_dev_tmp = MKDEV(ftdev->major, minor_cntr); + device = &ftdev->minor[minor_cntr].device; + + *device = device_create(ftdev->class, NULL, trace_dev_tmp, NULL, + "litmus/%s%d", ftdev->name, minor_cntr); + if (IS_ERR(*device)) { + err = PTR_ERR(*device); + printk(KERN_WARNING "ftdev(%s): " + "Could not create device major/minor number " + "%u/%u\n", ftdev->name, ftdev->major, + minor_cntr); + printk(KERN_WARNING "ftdev(%s): " + "will attempt deletion of allocated devices.\n", + ftdev->name); + goto err_minors; + } + } + + return 0; + +err_minors: + ftdev_device_destroy(ftdev, minor_cntr); + cdev_del(&ftdev->cdev); +err_unregister: + unregister_chrdev_region(MKDEV(ftdev->major, 0), ftdev->minor_cnt); +err_out: + return err; +} diff --git a/litmus/gpu_affinity.c b/litmus/gpu_affinity.c new file mode 100644 index 0000000..9762be1 --- /dev/null +++ b/litmus/gpu_affinity.c @@ -0,0 +1,113 @@ + +#ifdef CONFIG_LITMUS_NVIDIA + +#include +#include +#include + +#include + +#define OBSERVATION_CAP 2*1e9 + +static fp_t update_estimate(feedback_est_t* fb, fp_t a, fp_t b, lt_t observed) +{ + fp_t relative_err; + fp_t err, new; + fp_t actual = _integer_to_fp(observed); + + err = _sub(actual, fb->est); + new = _add(_mul(a, err), _mul(b, fb->accum_err)); + + relative_err = _div(err, actual); + + fb->est = new; + fb->accum_err = _add(fb->accum_err, err); + + return relative_err; +} + +void update_gpu_estimate(struct task_struct *t, lt_t observed) +{ + feedback_est_t *fb = &(tsk_rt(t)->gpu_migration_est[tsk_rt(t)->gpu_migration]); + + BUG_ON(tsk_rt(t)->gpu_migration > MIG_LAST); + + if(unlikely(fb->est.val == 0)) { + // kludge-- cap observed values to prevent whacky estimations. + // whacky stuff happens during the first few jobs. + if(unlikely(observed > OBSERVATION_CAP)) { + TRACE_TASK(t, "Crazy observation was capped: %llu -> %llu\n", + observed, OBSERVATION_CAP); + observed = OBSERVATION_CAP; + } + + // take the first observation as our estimate + // (initial value of 0 was bogus anyhow) + fb->est = _integer_to_fp(observed); + fb->accum_err = _div(fb->est, _integer_to_fp(2)); // ...seems to work. + } + else { + fp_t rel_err = update_estimate(fb, + tsk_rt(t)->gpu_fb_param_a[tsk_rt(t)->gpu_migration], + tsk_rt(t)->gpu_fb_param_b[tsk_rt(t)->gpu_migration], + observed); + + if(unlikely(_fp_to_integer(fb->est) <= 0)) { + TRACE_TASK(t, "Invalid estimate. Patching.\n"); + fb->est = _integer_to_fp(observed); + fb->accum_err = _div(fb->est, _integer_to_fp(2)); // ...seems to work. + } + else { +// struct migration_info mig_info; + + sched_trace_prediction_err(t, + &(tsk_rt(t)->gpu_migration), + &rel_err); + +// mig_info.observed = observed; +// mig_info.estimated = get_gpu_estimate(t, tsk_rt(t)->gpu_migration); +// mig_info.distance = tsk_rt(t)->gpu_migration; +// +// sched_trace_migration(t, &mig_info); + } + } + + TRACE_TASK(t, "GPU est update after (dist = %d, obs = %llu): %d.%d\n", + tsk_rt(t)->gpu_migration, + observed, + _fp_to_integer(fb->est), + _point(fb->est)); +} + +gpu_migration_dist_t gpu_migration_distance(int a, int b) +{ + // GPUs organized in a binary hierarchy, no more than 2^MIG_FAR GPUs + int i; + int dist; + + if(likely(a >= 0 && b >= 0)) { + for(i = 0; i <= MIG_FAR; ++i) { + if(a>>i == b>>i) { + dist = i; + goto out; + } + } + dist = MIG_NONE; // hopefully never reached. + TRACE_CUR("WARNING: GPU distance too far! %d -> %d\n", a, b); + } + else { + dist = MIG_NONE; + } + +out: + TRACE_CUR("Distance %d -> %d is %d\n", + a, b, dist); + + return dist; +} + + + + +#endif + diff --git a/litmus/ikglp_lock.c b/litmus/ikglp_lock.c new file mode 100644 index 0000000..83b708a --- /dev/null +++ b/litmus/ikglp_lock.c @@ -0,0 +1,2838 @@ +#include +#include + +#include +#include +#include + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) +#include +#include +#endif + +#include + +// big signed value. +#define IKGLP_INVAL_DISTANCE 0x7FFFFFFF + +int ikglp_max_heap_base_priority_order(struct binheap_node *a, + struct binheap_node *b) +{ + ikglp_heap_node_t *d_a = binheap_entry(a, ikglp_heap_node_t, node); + ikglp_heap_node_t *d_b = binheap_entry(b, ikglp_heap_node_t, node); + + BUG_ON(!d_a); + BUG_ON(!d_b); + + return litmus->__compare(d_a->task, BASE, d_b->task, BASE); +} + +int ikglp_min_heap_base_priority_order(struct binheap_node *a, + struct binheap_node *b) +{ + ikglp_heap_node_t *d_a = binheap_entry(a, ikglp_heap_node_t, node); + ikglp_heap_node_t *d_b = binheap_entry(b, ikglp_heap_node_t, node); + + return litmus->__compare(d_b->task, BASE, d_a->task, BASE); +} + +int ikglp_donor_max_heap_base_priority_order(struct binheap_node *a, + struct binheap_node *b) +{ + ikglp_wait_state_t *d_a = binheap_entry(a, ikglp_wait_state_t, node); + ikglp_wait_state_t *d_b = binheap_entry(b, ikglp_wait_state_t, node); + + return litmus->__compare(d_a->task, BASE, d_b->task, BASE); +} + + +int ikglp_min_heap_donee_order(struct binheap_node *a, + struct binheap_node *b) +{ + struct task_struct *prio_a, *prio_b; + + ikglp_donee_heap_node_t *d_a = + binheap_entry(a, ikglp_donee_heap_node_t, node); + ikglp_donee_heap_node_t *d_b = + binheap_entry(b, ikglp_donee_heap_node_t, node); + + if(!d_a->donor_info) { + prio_a = d_a->task; + } + else { + prio_a = d_a->donor_info->task; + BUG_ON(d_a->task != d_a->donor_info->donee_info->task); + } + + if(!d_b->donor_info) { + prio_b = d_b->task; + } + else { + prio_b = d_b->donor_info->task; + BUG_ON(d_b->task != d_b->donor_info->donee_info->task); + } + + // note reversed order + return litmus->__compare(prio_b, BASE, prio_a, BASE); +} + + + +static inline int ikglp_get_idx(struct ikglp_semaphore *sem, + struct fifo_queue *queue) +{ + return (queue - &sem->fifo_queues[0]); +} + +static inline struct fifo_queue* ikglp_get_queue(struct ikglp_semaphore *sem, + struct task_struct *holder) +{ + int i; + for(i = 0; i < sem->nr_replicas; ++i) + if(sem->fifo_queues[i].owner == holder) + return(&sem->fifo_queues[i]); + return(NULL); +} + + + +static struct task_struct* ikglp_find_hp_waiter(struct fifo_queue *kqueue, + struct task_struct *skip) +{ + struct list_head *pos; + struct task_struct *queued, *found = NULL; + + list_for_each(pos, &kqueue->wait.task_list) { + queued = (struct task_struct*) list_entry(pos, + wait_queue_t, task_list)->private; + + /* Compare task prios, find high prio task. */ + if(queued != skip && litmus->compare(queued, found)) + found = queued; + } + return found; +} + +static struct fifo_queue* ikglp_find_shortest(struct ikglp_semaphore *sem, + struct fifo_queue *search_start) +{ + // we start our search at search_start instead of at the beginning of the + // queue list to load-balance across all resources. + struct fifo_queue* step = search_start; + struct fifo_queue* shortest = sem->shortest_fifo_queue; + + do { + step = (step+1 != &sem->fifo_queues[sem->nr_replicas]) ? + step+1 : &sem->fifo_queues[0]; + + if(step->count < shortest->count) { + shortest = step; + if(step->count == 0) + break; /* can't get any shorter */ + } + + }while(step != search_start); + + return(shortest); +} + +static inline struct task_struct* ikglp_mth_highest(struct ikglp_semaphore *sem) +{ + return binheap_top_entry(&sem->top_m, ikglp_heap_node_t, node)->task; +} + + + +#if 0 +static void print_global_list(struct binheap_node* n, int depth) +{ + ikglp_heap_node_t *global_heap_node; + char padding[81] = " "; + + if(n == NULL) { + TRACE_CUR("+-> %p\n", NULL); + return; + } + + global_heap_node = binheap_entry(n, ikglp_heap_node_t, node); + + if(depth*2 <= 80) + padding[depth*2] = '\0'; + + TRACE_CUR("%s+-> %s/%d\n", + padding, + global_heap_node->task->comm, + global_heap_node->task->pid); + + if(n->left) print_global_list(n->left, depth+1); + if(n->right) print_global_list(n->right, depth+1); +} + +static void print_donees(struct ikglp_semaphore *sem, struct binheap_node *n, int depth) +{ + ikglp_donee_heap_node_t *donee_node; + char padding[81] = " "; + struct task_struct* donor = NULL; + + if(n == NULL) { + TRACE_CUR("+-> %p\n", NULL); + return; + } + + donee_node = binheap_entry(n, ikglp_donee_heap_node_t, node); + + if(depth*2 <= 80) + padding[depth*2] = '\0'; + + if(donee_node->donor_info) { + donor = donee_node->donor_info->task; + } + + TRACE_CUR("%s+-> %s/%d (d: %s/%d) (fq: %d)\n", + padding, + donee_node->task->comm, + donee_node->task->pid, + (donor) ? donor->comm : "nil", + (donor) ? donor->pid : -1, + ikglp_get_idx(sem, donee_node->fq)); + + if(n->left) print_donees(sem, n->left, depth+1); + if(n->right) print_donees(sem, n->right, depth+1); +} + +static void print_donors(struct binheap_node *n, int depth) +{ + ikglp_wait_state_t *donor_node; + char padding[81] = " "; + + if(n == NULL) { + TRACE_CUR("+-> %p\n", NULL); + return; + } + + donor_node = binheap_entry(n, ikglp_wait_state_t, node); + + if(depth*2 <= 80) + padding[depth*2] = '\0'; + + + TRACE_CUR("%s+-> %s/%d (donee: %s/%d)\n", + padding, + donor_node->task->comm, + donor_node->task->pid, + donor_node->donee_info->task->comm, + donor_node->donee_info->task->pid); + + if(n->left) print_donors(n->left, depth+1); + if(n->right) print_donors(n->right, depth+1); +} +#endif + +static void ikglp_add_global_list(struct ikglp_semaphore *sem, + struct task_struct *t, + ikglp_heap_node_t *node) +{ + + + node->task = t; + INIT_BINHEAP_NODE(&node->node); + + if(sem->top_m_size < sem->m) { + TRACE_CUR("Trivially adding %s/%d to top-m global list.\n", + t->comm, t->pid); +// TRACE_CUR("Top-M Before (size = %d):\n", sem->top_m_size); +// print_global_list(sem->top_m.root, 1); + + binheap_add(&node->node, &sem->top_m, ikglp_heap_node_t, node); + ++(sem->top_m_size); + +// TRACE_CUR("Top-M After (size = %d):\n", sem->top_m_size); +// print_global_list(sem->top_m.root, 1); + } + else if(litmus->__compare(t, BASE, ikglp_mth_highest(sem), BASE)) { + ikglp_heap_node_t *evicted = + binheap_top_entry(&sem->top_m, ikglp_heap_node_t, node); + + TRACE_CUR("Adding %s/%d to top-m and evicting %s/%d.\n", + t->comm, t->pid, + evicted->task->comm, evicted->task->pid); + +// TRACE_CUR("Not-Top-M Before:\n"); +// print_global_list(sem->not_top_m.root, 1); +// TRACE_CUR("Top-M Before (size = %d):\n", sem->top_m_size); +// print_global_list(sem->top_m.root, 1); + + + binheap_delete_root(&sem->top_m, ikglp_heap_node_t, node); + INIT_BINHEAP_NODE(&evicted->node); + binheap_add(&evicted->node, &sem->not_top_m, ikglp_heap_node_t, node); + + binheap_add(&node->node, &sem->top_m, ikglp_heap_node_t, node); + +// TRACE_CUR("Top-M After (size = %d):\n", sem->top_m_size); +// print_global_list(sem->top_m.root, 1); +// TRACE_CUR("Not-Top-M After:\n"); +// print_global_list(sem->not_top_m.root, 1); + } + else { + TRACE_CUR("Trivially adding %s/%d to not-top-m global list.\n", + t->comm, t->pid); +// TRACE_CUR("Not-Top-M Before:\n"); +// print_global_list(sem->not_top_m.root, 1); + + binheap_add(&node->node, &sem->not_top_m, ikglp_heap_node_t, node); + +// TRACE_CUR("Not-Top-M After:\n"); +// print_global_list(sem->not_top_m.root, 1); + } +} + + +static void ikglp_del_global_list(struct ikglp_semaphore *sem, + struct task_struct *t, + ikglp_heap_node_t *node) +{ + BUG_ON(!binheap_is_in_heap(&node->node)); + + TRACE_CUR("Removing %s/%d from global list.\n", t->comm, t->pid); + + if(binheap_is_in_this_heap(&node->node, &sem->top_m)) { + TRACE_CUR("%s/%d is in top-m\n", t->comm, t->pid); + +// TRACE_CUR("Not-Top-M Before:\n"); +// print_global_list(sem->not_top_m.root, 1); +// TRACE_CUR("Top-M Before (size = %d):\n", sem->top_m_size); +// print_global_list(sem->top_m.root, 1); + + + binheap_delete(&node->node, &sem->top_m); + + if(!binheap_empty(&sem->not_top_m)) { + ikglp_heap_node_t *promoted = + binheap_top_entry(&sem->not_top_m, ikglp_heap_node_t, node); + + TRACE_CUR("Promoting %s/%d to top-m\n", + promoted->task->comm, promoted->task->pid); + + binheap_delete_root(&sem->not_top_m, ikglp_heap_node_t, node); + INIT_BINHEAP_NODE(&promoted->node); + + binheap_add(&promoted->node, &sem->top_m, ikglp_heap_node_t, node); + } + else { + TRACE_CUR("No one to promote to top-m.\n"); + --(sem->top_m_size); + } + +// TRACE_CUR("Top-M After (size = %d):\n", sem->top_m_size); +// print_global_list(sem->top_m.root, 1); +// TRACE_CUR("Not-Top-M After:\n"); +// print_global_list(sem->not_top_m.root, 1); + } + else { + TRACE_CUR("%s/%d is in not-top-m\n", t->comm, t->pid); +// TRACE_CUR("Not-Top-M Before:\n"); +// print_global_list(sem->not_top_m.root, 1); + + binheap_delete(&node->node, &sem->not_top_m); + +// TRACE_CUR("Not-Top-M After:\n"); +// print_global_list(sem->not_top_m.root, 1); + } +} + + +static void ikglp_add_donees(struct ikglp_semaphore *sem, + struct fifo_queue *fq, + struct task_struct *t, + ikglp_donee_heap_node_t* node) +{ +// TRACE_CUR("Adding %s/%d to donee list.\n", t->comm, t->pid); +// TRACE_CUR("donees Before:\n"); +// print_donees(sem, sem->donees.root, 1); + + node->task = t; + node->donor_info = NULL; + node->fq = fq; + INIT_BINHEAP_NODE(&node->node); + + binheap_add(&node->node, &sem->donees, ikglp_donee_heap_node_t, node); + +// TRACE_CUR("donees After:\n"); +// print_donees(sem, sem->donees.root, 1); +} + + +static void ikglp_refresh_owners_prio_increase(struct task_struct *t, + struct fifo_queue *fq, + struct ikglp_semaphore *sem, + unsigned long flags) +{ + // priority of 't' has increased (note: 't' might already be hp_waiter). + if ((t == fq->hp_waiter) || litmus->compare(t, fq->hp_waiter)) { + struct task_struct *old_max_eff_prio; + struct task_struct *new_max_eff_prio; + struct task_struct *new_prio = NULL; + struct task_struct *owner = fq->owner; + + if(fq->hp_waiter) + TRACE_TASK(t, "has higher prio than hp_waiter (%s/%d).\n", + fq->hp_waiter->comm, fq->hp_waiter->pid); + else + TRACE_TASK(t, "has higher prio than hp_waiter (NIL).\n"); + + if(owner) + { + raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock); + +// TRACE_TASK(owner, "Heap Before:\n"); +// print_hp_waiters(tsk_rt(owner)->hp_blocked_tasks.root, 0); + + old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + fq->hp_waiter = t; + fq->nest.hp_waiter_eff_prio = effective_priority(fq->hp_waiter); + + binheap_decrease(&fq->nest.hp_binheap_node, + &tsk_rt(owner)->hp_blocked_tasks); + +// TRACE_TASK(owner, "Heap After:\n"); +// print_hp_waiters(tsk_rt(owner)->hp_blocked_tasks.root, 0); + + new_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + if(new_max_eff_prio != old_max_eff_prio) { + TRACE_TASK(t, "is new hp_waiter.\n"); + + if ((effective_priority(owner) == old_max_eff_prio) || + (litmus->__compare(new_max_eff_prio, BASE, + owner, EFFECTIVE))){ + new_prio = new_max_eff_prio; + } + } + else { + TRACE_TASK(t, "no change in max_eff_prio of heap.\n"); + } + + if(new_prio) { + // set new inheritance and propagate + TRACE_TASK(t, "Effective priority changed for owner %s/%d to %s/%d\n", + owner->comm, owner->pid, + new_prio->comm, new_prio->pid); + litmus->nested_increase_prio(owner, new_prio, &sem->lock, + flags); // unlocks lock. + } + else { + TRACE_TASK(t, "No change in effective priority (is %s/%d). Propagation halted.\n", + new_max_eff_prio->comm, new_max_eff_prio->pid); + raw_spin_unlock(&tsk_rt(owner)->hp_blocked_tasks_lock); + unlock_fine_irqrestore(&sem->lock, flags); + } + } + else { + fq->hp_waiter = t; + fq->nest.hp_waiter_eff_prio = effective_priority(fq->hp_waiter); + + TRACE_TASK(t, "no owner.\n"); + unlock_fine_irqrestore(&sem->lock, flags); + } + } + else { + TRACE_TASK(t, "hp_waiter is unaffected.\n"); + unlock_fine_irqrestore(&sem->lock, flags); + } +} + +// hp_waiter has decreased +static void ikglp_refresh_owners_prio_decrease(struct fifo_queue *fq, + struct ikglp_semaphore *sem, + unsigned long flags) +{ + struct task_struct *owner = fq->owner; + + struct task_struct *old_max_eff_prio; + struct task_struct *new_max_eff_prio; + + if(!owner) { + TRACE_CUR("No owner. Returning.\n"); + unlock_fine_irqrestore(&sem->lock, flags); + return; + } + + TRACE_CUR("ikglp_refresh_owners_prio_decrease\n"); + + raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock); + + old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + binheap_delete(&fq->nest.hp_binheap_node, &tsk_rt(owner)->hp_blocked_tasks); + fq->nest.hp_waiter_eff_prio = fq->hp_waiter; + binheap_add(&fq->nest.hp_binheap_node, &tsk_rt(owner)->hp_blocked_tasks, + struct nested_info, hp_binheap_node); + + new_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + if((old_max_eff_prio != new_max_eff_prio) && + (effective_priority(owner) == old_max_eff_prio)) + { + // Need to set new effective_priority for owner + struct task_struct *decreased_prio; + + TRACE_CUR("Propagating decreased inheritance to holder of fq %d.\n", + ikglp_get_idx(sem, fq)); + + if(litmus->__compare(new_max_eff_prio, BASE, owner, BASE)) { + TRACE_CUR("%s/%d has greater base priority than base priority of owner (%s/%d) of fq %d.\n", + (new_max_eff_prio) ? new_max_eff_prio->comm : "nil", + (new_max_eff_prio) ? new_max_eff_prio->pid : -1, + owner->comm, + owner->pid, + ikglp_get_idx(sem, fq)); + + decreased_prio = new_max_eff_prio; + } + else { + TRACE_CUR("%s/%d has lesser base priority than base priority of owner (%s/%d) of fq %d.\n", + (new_max_eff_prio) ? new_max_eff_prio->comm : "nil", + (new_max_eff_prio) ? new_max_eff_prio->pid : -1, + owner->comm, + owner->pid, + ikglp_get_idx(sem, fq)); + + decreased_prio = NULL; + } + + // beware: recursion + litmus->nested_decrease_prio(owner, decreased_prio, &sem->lock, flags); // will unlock mutex->lock + } + else { + TRACE_TASK(owner, "No need to propagate priority decrease forward.\n"); + raw_spin_unlock(&tsk_rt(owner)->hp_blocked_tasks_lock); + unlock_fine_irqrestore(&sem->lock, flags); + } +} + + +static void ikglp_remove_donation_from_owner(struct binheap_node *n, + struct fifo_queue *fq, + struct ikglp_semaphore *sem, + unsigned long flags) +{ + struct task_struct *owner = fq->owner; + + struct task_struct *old_max_eff_prio; + struct task_struct *new_max_eff_prio; + + BUG_ON(!owner); + + raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock); + + old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + binheap_delete(n, &tsk_rt(owner)->hp_blocked_tasks); + + new_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + if((old_max_eff_prio != new_max_eff_prio) && + (effective_priority(owner) == old_max_eff_prio)) + { + // Need to set new effective_priority for owner + struct task_struct *decreased_prio; + + TRACE_CUR("Propagating decreased inheritance to holder of fq %d.\n", + ikglp_get_idx(sem, fq)); + + if(litmus->__compare(new_max_eff_prio, BASE, owner, BASE)) { + TRACE_CUR("has greater base priority than base priority of owner of fq %d.\n", + ikglp_get_idx(sem, fq)); + decreased_prio = new_max_eff_prio; + } + else { + TRACE_CUR("has lesser base priority than base priority of owner of fq %d.\n", + ikglp_get_idx(sem, fq)); + decreased_prio = NULL; + } + + // beware: recursion + litmus->nested_decrease_prio(owner, decreased_prio, &sem->lock, flags); // will unlock mutex->lock + } + else { + TRACE_TASK(owner, "No need to propagate priority decrease forward.\n"); + raw_spin_unlock(&tsk_rt(owner)->hp_blocked_tasks_lock); + unlock_fine_irqrestore(&sem->lock, flags); + } +} + +static void ikglp_remove_donation_from_fq_waiter(struct task_struct *t, + struct binheap_node *n) +{ + struct task_struct *old_max_eff_prio; + struct task_struct *new_max_eff_prio; + + raw_spin_lock(&tsk_rt(t)->hp_blocked_tasks_lock); + + old_max_eff_prio = top_priority(&tsk_rt(t)->hp_blocked_tasks); + + binheap_delete(n, &tsk_rt(t)->hp_blocked_tasks); + + new_max_eff_prio = top_priority(&tsk_rt(t)->hp_blocked_tasks); + + if((old_max_eff_prio != new_max_eff_prio) && + (effective_priority(t) == old_max_eff_prio)) + { + // Need to set new effective_priority for owner + struct task_struct *decreased_prio; + + if(litmus->__compare(new_max_eff_prio, BASE, t, BASE)) { + decreased_prio = new_max_eff_prio; + } + else { + decreased_prio = NULL; + } + + tsk_rt(t)->inh_task = decreased_prio; + } + + raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); +} + +static void ikglp_get_immediate(struct task_struct* t, + struct fifo_queue *fq, + struct ikglp_semaphore *sem, + unsigned long flags) +{ + // resource available now + TRACE_CUR("queue %d: acquired immediately\n", ikglp_get_idx(sem, fq)); + + fq->owner = t; + + raw_spin_lock(&tsk_rt(t)->hp_blocked_tasks_lock); + binheap_add(&fq->nest.hp_binheap_node, &tsk_rt(t)->hp_blocked_tasks, + struct nested_info, hp_binheap_node); + raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); + + ++(fq->count); + + ikglp_add_global_list(sem, t, &fq->global_heap_node); + ikglp_add_donees(sem, fq, t, &fq->donee_heap_node); + + sem->shortest_fifo_queue = ikglp_find_shortest(sem, sem->shortest_fifo_queue); + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_enqueue(sem->aff_obs, fq, t); + sem->aff_obs->ops->notify_acquired(sem->aff_obs, fq, t); + } +#endif + + unlock_fine_irqrestore(&sem->lock, flags); +} + + + + + +static void __ikglp_enqueue_on_fq(struct ikglp_semaphore *sem, + struct fifo_queue* fq, + struct task_struct* t, + wait_queue_t *wait, + ikglp_heap_node_t *global_heap_node, + ikglp_donee_heap_node_t *donee_heap_node) +{ + /* resource is not free => must suspend and wait */ + TRACE_TASK(t, "Enqueuing on fq %d.\n", + ikglp_get_idx(sem, fq)); + + init_waitqueue_entry(wait, t); + + __add_wait_queue_tail_exclusive(&fq->wait, wait); + + ++(fq->count); + ++(sem->nr_in_fifos); + + // update global list. + if(likely(global_heap_node)) { + if(binheap_is_in_heap(&global_heap_node->node)) { + WARN_ON(1); + ikglp_del_global_list(sem, t, global_heap_node); + } + ikglp_add_global_list(sem, t, global_heap_node); + } + // update donor eligiblity list. + if(likely(donee_heap_node)) { +// if(binheap_is_in_heap(&donee_heap_node->node)) { +// WARN_ON(1); +// } + ikglp_add_donees(sem, fq, t, donee_heap_node); + } + + if(sem->shortest_fifo_queue == fq) { + sem->shortest_fifo_queue = ikglp_find_shortest(sem, fq); + } + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_enqueue(sem->aff_obs, fq, t); + } +#endif + + TRACE_TASK(t, "shortest queue is now %d\n", ikglp_get_idx(sem, fq)); +} + + +static void ikglp_enqueue_on_fq( + struct ikglp_semaphore *sem, + struct fifo_queue *fq, + ikglp_wait_state_t *wait, + unsigned long flags) +{ + /* resource is not free => must suspend and wait */ + TRACE_TASK(wait->task, "queue %d: Resource is not free => must suspend and wait.\n", + ikglp_get_idx(sem, fq)); + + INIT_BINHEAP_NODE(&wait->global_heap_node.node); + INIT_BINHEAP_NODE(&wait->donee_heap_node.node); + + __ikglp_enqueue_on_fq(sem, fq, wait->task, &wait->fq_node, + &wait->global_heap_node, &wait->donee_heap_node); + + ikglp_refresh_owners_prio_increase(wait->task, fq, sem, flags); // unlocks sem->lock +} + + +static void __ikglp_enqueue_on_pq(struct ikglp_semaphore *sem, + ikglp_wait_state_t *wait) +{ + TRACE_TASK(wait->task, "goes to PQ.\n"); + + wait->pq_node.task = wait->task; // copy over task (little redundant...) + + binheap_add(&wait->pq_node.node, &sem->priority_queue, + ikglp_heap_node_t, node); +} + +static void ikglp_enqueue_on_pq(struct ikglp_semaphore *sem, + ikglp_wait_state_t *wait) +{ + INIT_BINHEAP_NODE(&wait->global_heap_node.node); + INIT_BINHEAP_NODE(&wait->donee_heap_node.node); + INIT_BINHEAP_NODE(&wait->pq_node.node); + + __ikglp_enqueue_on_pq(sem, wait); +} + +static void ikglp_enqueue_on_donor(struct ikglp_semaphore *sem, + ikglp_wait_state_t* wait, + unsigned long flags) +{ + struct task_struct *t = wait->task; + ikglp_donee_heap_node_t *donee_node = NULL; + struct task_struct *donee; + + struct task_struct *old_max_eff_prio; + struct task_struct *new_max_eff_prio; + struct task_struct *new_prio = NULL; + + INIT_BINHEAP_NODE(&wait->global_heap_node.node); + INIT_BINHEAP_NODE(&wait->donee_heap_node.node); + INIT_BINHEAP_NODE(&wait->pq_node.node); + INIT_BINHEAP_NODE(&wait->node); + +// TRACE_CUR("Adding %s/%d as donor.\n", t->comm, t->pid); +// TRACE_CUR("donors Before:\n"); +// print_donors(sem->donors.root, 1); + + // Add donor to the global list. + ikglp_add_global_list(sem, t, &wait->global_heap_node); + + // Select a donee +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + donee_node = (sem->aff_obs) ? + sem->aff_obs->ops->advise_donee_selection(sem->aff_obs, t) : + binheap_top_entry(&sem->donees, ikglp_donee_heap_node_t, node); +#else + donee_node = binheap_top_entry(&sem->donees, ikglp_donee_heap_node_t, node); +#endif + + donee = donee_node->task; + + TRACE_TASK(t, "Donee selected: %s/%d\n", donee->comm, donee->pid); + + TRACE_CUR("Temporarily removing %s/%d to donee list.\n", + donee->comm, donee->pid); +// TRACE_CUR("donees Before:\n"); +// print_donees(sem, sem->donees.root, 1); + + //binheap_delete_root(&sem->donees, ikglp_donee_heap_node_t, node); // will re-add it shortly + binheap_delete(&donee_node->node, &sem->donees); + +// TRACE_CUR("donees After:\n"); +// print_donees(sem, sem->donees.root, 1); + + + wait->donee_info = donee_node; + + // Add t to donor heap. + binheap_add(&wait->node, &sem->donors, ikglp_wait_state_t, node); + + // Now adjust the donee's priority. + + // Lock the donee's inheritance heap. + raw_spin_lock(&tsk_rt(donee)->hp_blocked_tasks_lock); + + old_max_eff_prio = top_priority(&tsk_rt(donee)->hp_blocked_tasks); + + if(donee_node->donor_info) { + // Steal donation relation. Evict old donor to PQ. + + // Remove old donor from donor heap + ikglp_wait_state_t *old_wait = donee_node->donor_info; + struct task_struct *old_donor = old_wait->task; + + TRACE_TASK(t, "Donee (%s/%d) had donor %s/%d. Moving old donor to PQ.\n", + donee->comm, donee->pid, old_donor->comm, old_donor->pid); + + binheap_delete(&old_wait->node, &sem->donors); + + // Remove donation from donee's inheritance heap. + binheap_delete(&old_wait->prio_donation.hp_binheap_node, + &tsk_rt(donee)->hp_blocked_tasks); + // WARNING: have not updated inh_prio! + + // Add old donor to PQ. + __ikglp_enqueue_on_pq(sem, old_wait); + + // Remove old donor from the global heap. + ikglp_del_global_list(sem, old_donor, &old_wait->global_heap_node); + } + + // Add back donee's node to the donees heap with increased prio + donee_node->donor_info = wait; + INIT_BINHEAP_NODE(&donee_node->node); + + + TRACE_CUR("Adding %s/%d back to donee list.\n", donee->comm, donee->pid); +// TRACE_CUR("donees Before:\n"); +// print_donees(sem, sem->donees.root, 1); + + binheap_add(&donee_node->node, &sem->donees, ikglp_donee_heap_node_t, node); + +// TRACE_CUR("donees After:\n"); +// print_donees(sem, sem->donees.root, 1); + + // Add an inheritance/donation to the donee's inheritance heap. + wait->prio_donation.lock = (struct litmus_lock*)sem; + wait->prio_donation.hp_waiter_eff_prio = t; + wait->prio_donation.hp_waiter_ptr = NULL; + INIT_BINHEAP_NODE(&wait->prio_donation.hp_binheap_node); + + binheap_add(&wait->prio_donation.hp_binheap_node, + &tsk_rt(donee)->hp_blocked_tasks, + struct nested_info, hp_binheap_node); + + new_max_eff_prio = top_priority(&tsk_rt(donee)->hp_blocked_tasks); + + if(new_max_eff_prio != old_max_eff_prio) { + if ((effective_priority(donee) == old_max_eff_prio) || + (litmus->__compare(new_max_eff_prio, BASE, donee, EFFECTIVE))){ + TRACE_TASK(t, "Donation increases %s/%d's effective priority\n", + donee->comm, donee->pid); + new_prio = new_max_eff_prio; + } +// else { +// // should be bug. donor would not be in top-m. +// TRACE_TASK(t, "Donation is not greater than base prio of %s/%d?\n", donee->comm, donee->pid); +// WARN_ON(1); +// } +// } +// else { +// // should be bug. donor would not be in top-m. +// TRACE_TASK(t, "No change in %s/%d's inheritance heap?\n", donee->comm, donee->pid); +// WARN_ON(1); + } + + if(new_prio) { + struct fifo_queue *donee_fq = donee_node->fq; + + if(donee != donee_fq->owner) { + TRACE_TASK(t, "%s/%d is not the owner. Propagating priority to owner %s/%d.\n", + donee->comm, donee->pid, + donee_fq->owner->comm, donee_fq->owner->pid); + + raw_spin_unlock(&tsk_rt(donee)->hp_blocked_tasks_lock); + ikglp_refresh_owners_prio_increase(donee, donee_fq, sem, flags); // unlocks sem->lock + } + else { + TRACE_TASK(t, "%s/%d is the owner. Progatating priority immediatly.\n", + donee->comm, donee->pid); + litmus->nested_increase_prio(donee, new_prio, &sem->lock, flags); // unlocks sem->lock and donee's heap lock + } + } + else { + TRACE_TASK(t, "No change in effective priority (it is %d/%s). BUG?\n", + new_max_eff_prio->comm, new_max_eff_prio->pid); + raw_spin_unlock(&tsk_rt(donee)->hp_blocked_tasks_lock); + unlock_fine_irqrestore(&sem->lock, flags); + } + + +// TRACE_CUR("donors After:\n"); +// print_donors(sem->donors.root, 1); +} + +int ikglp_lock(struct litmus_lock* l) +{ + struct task_struct* t = current; + struct ikglp_semaphore *sem = ikglp_from_lock(l); + unsigned long flags = 0, real_flags; + struct fifo_queue *fq = NULL; + int replica = -EINVAL; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + raw_spinlock_t *dgl_lock; +#endif + + ikglp_wait_state_t wait; + + if (!is_realtime(t)) + return -EPERM; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + dgl_lock = litmus->get_dgl_spinlock(t); +#endif + + raw_spin_lock_irqsave(&sem->real_lock, real_flags); + + lock_global_irqsave(dgl_lock, flags); + lock_fine_irqsave(&sem->lock, flags); + + if(sem->nr_in_fifos < sem->m) { + // enqueue somwhere +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + fq = (sem->aff_obs) ? + sem->aff_obs->ops->advise_enqueue(sem->aff_obs, t) : + sem->shortest_fifo_queue; +#else + fq = sem->shortest_fifo_queue; +#endif + if(fq->count == 0) { + // take available resource + replica = ikglp_get_idx(sem, fq); + + ikglp_get_immediate(t, fq, sem, flags); // unlocks sem->lock + + unlock_global_irqrestore(dgl_lock, flags); + raw_spin_unlock_irqrestore(&sem->real_lock, real_flags); + goto acquired; + } + else { + wait.task = t; // THIS IS CRITICALLY IMPORTANT!!! + + tsk_rt(t)->blocked_lock = (struct litmus_lock*)sem; // record where we are blocked + mb(); + + /* FIXME: interruptible would be nice some day */ + set_task_state(t, TASK_UNINTERRUPTIBLE); + + ikglp_enqueue_on_fq(sem, fq, &wait, flags); // unlocks sem->lock + } + } + else { + // donor! + wait.task = t; // THIS IS CRITICALLY IMPORTANT!!! + + tsk_rt(t)->blocked_lock = (struct litmus_lock*)sem; // record where we are blocked + mb(); + + /* FIXME: interruptible would be nice some day */ + set_task_state(t, TASK_UNINTERRUPTIBLE); + + if(litmus->__compare(ikglp_mth_highest(sem), BASE, t, BASE)) { + // enqueue on PQ + ikglp_enqueue_on_pq(sem, &wait); + unlock_fine_irqrestore(&sem->lock, flags); + } + else { + // enqueue as donor + ikglp_enqueue_on_donor(sem, &wait, flags); // unlocks sem->lock + } + } + + unlock_global_irqrestore(dgl_lock, flags); + raw_spin_unlock_irqrestore(&sem->real_lock, real_flags); + + TS_LOCK_SUSPEND; + + schedule(); + + TS_LOCK_RESUME; + + fq = ikglp_get_queue(sem, t); + BUG_ON(!fq); + + replica = ikglp_get_idx(sem, fq); + +acquired: + TRACE_CUR("Acquired lock %d, queue %d\n", + l->ident, replica); + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + return sem->aff_obs->ops->replica_to_resource(sem->aff_obs, fq); + } +#endif + + return replica; +} + +//int ikglp_lock(struct litmus_lock* l) +//{ +// struct task_struct* t = current; +// struct ikglp_semaphore *sem = ikglp_from_lock(l); +// unsigned long flags = 0, real_flags; +// struct fifo_queue *fq = NULL; +// int replica = -EINVAL; +// +//#ifdef CONFIG_LITMUS_DGL_SUPPORT +// raw_spinlock_t *dgl_lock; +//#endif +// +// ikglp_wait_state_t wait; +// +// if (!is_realtime(t)) +// return -EPERM; +// +//#ifdef CONFIG_LITMUS_DGL_SUPPORT +// dgl_lock = litmus->get_dgl_spinlock(t); +//#endif +// +// raw_spin_lock_irqsave(&sem->real_lock, real_flags); +// +// lock_global_irqsave(dgl_lock, flags); +// lock_fine_irqsave(&sem->lock, flags); +// +// +//#ifdef CONFIG_LITMUS_AFFINITY_LOCKING +// fq = (sem->aff_obs) ? +// sem->aff_obs->ops->advise_enqueue(sem->aff_obs, t) : +// sem->shortest_fifo_queue; +//#else +// fq = sem->shortest_fifo_queue; +//#endif +// +// if(fq->count == 0) { +// // take available resource +// replica = ikglp_get_idx(sem, fq); +// +// ikglp_get_immediate(t, fq, sem, flags); // unlocks sem->lock +// +// unlock_global_irqrestore(dgl_lock, flags); +// raw_spin_unlock_irqrestore(&sem->real_lock, real_flags); +// } +// else +// { +// // we have to suspend. +// +// wait.task = t; // THIS IS CRITICALLY IMPORTANT!!! +// +// tsk_rt(t)->blocked_lock = (struct litmus_lock*)sem; // record where we are blocked +// mb(); +// +// /* FIXME: interruptible would be nice some day */ +// set_task_state(t, TASK_UNINTERRUPTIBLE); +// +// if(fq->count < sem->max_fifo_len) { +// // enqueue on fq +// ikglp_enqueue_on_fq(sem, fq, &wait, flags); // unlocks sem->lock +// } +// else { +// +// TRACE_CUR("IKGLP fifo queues are full (at least they better be).\n"); +// +// // no room in fifos. Go to PQ or donors. +// +// if(litmus->__compare(ikglp_mth_highest(sem), BASE, t, BASE)) { +// // enqueue on PQ +// ikglp_enqueue_on_pq(sem, &wait); +// unlock_fine_irqrestore(&sem->lock, flags); +// } +// else { +// // enqueue as donor +// ikglp_enqueue_on_donor(sem, &wait, flags); // unlocks sem->lock +// } +// } +// +// unlock_global_irqrestore(dgl_lock, flags); +// raw_spin_unlock_irqrestore(&sem->real_lock, real_flags); +// +// TS_LOCK_SUSPEND; +// +// schedule(); +// +// TS_LOCK_RESUME; +// +// fq = ikglp_get_queue(sem, t); +// BUG_ON(!fq); +// +// replica = ikglp_get_idx(sem, fq); +// } +// +// TRACE_CUR("Acquired lock %d, queue %d\n", +// l->ident, replica); +// +//#ifdef CONFIG_LITMUS_AFFINITY_LOCKING +// if(sem->aff_obs) { +// return sem->aff_obs->ops->replica_to_resource(sem->aff_obs, fq); +// } +//#endif +// +// return replica; +//} + +static void ikglp_move_donor_to_fq(struct ikglp_semaphore *sem, + struct fifo_queue *fq, + ikglp_wait_state_t *donor_info) +{ + struct task_struct *t = donor_info->task; + + TRACE_CUR("Donor %s/%d being moved to fq %d\n", + t->comm, + t->pid, + ikglp_get_idx(sem, fq)); + + binheap_delete(&donor_info->node, &sem->donors); + + __ikglp_enqueue_on_fq(sem, fq, t, + &donor_info->fq_node, + NULL, // already in global_list, so pass null to prevent adding 2nd time. + &donor_info->donee_heap_node); + + // warning: + // ikglp_update_owners_prio(t, fq, sem, flags) has not been called. +} + +static void ikglp_move_pq_to_fq(struct ikglp_semaphore *sem, + struct fifo_queue *fq, + ikglp_wait_state_t *wait) +{ + struct task_struct *t = wait->task; + + TRACE_CUR("PQ request %s/%d being moved to fq %d\n", + t->comm, + t->pid, + ikglp_get_idx(sem, fq)); + + binheap_delete(&wait->pq_node.node, &sem->priority_queue); + + __ikglp_enqueue_on_fq(sem, fq, t, + &wait->fq_node, + &wait->global_heap_node, + &wait->donee_heap_node); + // warning: + // ikglp_update_owners_prio(t, fq, sem, flags) has not been called. +} + +static ikglp_wait_state_t* ikglp_find_hp_waiter_to_steal( + struct ikglp_semaphore* sem) +{ + /* must hold sem->lock */ + + struct fifo_queue *fq = NULL; + struct list_head *pos; + struct task_struct *queued; + int i; + + for(i = 0; i < sem->nr_replicas; ++i) { + if( (sem->fifo_queues[i].count > 1) && + (!fq || litmus->compare(sem->fifo_queues[i].hp_waiter, fq->hp_waiter)) ) { + + TRACE_CUR("hp_waiter on fq %d (%s/%d) has higher prio than hp_waiter on fq %d (%s/%d)\n", + ikglp_get_idx(sem, &sem->fifo_queues[i]), + sem->fifo_queues[i].hp_waiter->comm, + sem->fifo_queues[i].hp_waiter->pid, + (fq) ? ikglp_get_idx(sem, fq) : -1, + (fq) ? ((fq->hp_waiter) ? fq->hp_waiter->comm : "nil") : "nilXX", + (fq) ? ((fq->hp_waiter) ? fq->hp_waiter->pid : -1) : -2); + + fq = &sem->fifo_queues[i]; + + WARN_ON(!(fq->hp_waiter)); + } + } + + if(fq) { + struct task_struct *max_hp = fq->hp_waiter; + ikglp_wait_state_t* ret = NULL; + + TRACE_CUR("Searching for %s/%d on fq %d\n", + max_hp->comm, + max_hp->pid, + ikglp_get_idx(sem, fq)); + + BUG_ON(!max_hp); + + list_for_each(pos, &fq->wait.task_list) { + wait_queue_t *wait = list_entry(pos, wait_queue_t, task_list); + + queued = (struct task_struct*) wait->private; + + TRACE_CUR("fq %d entry: %s/%d\n", + ikglp_get_idx(sem, fq), + queued->comm, + queued->pid); + + /* Compare task prios, find high prio task. */ + if (queued == max_hp) { + TRACE_CUR("Found it!\n"); + ret = container_of(wait, ikglp_wait_state_t, fq_node); + } + } + + WARN_ON(!ret); + return ret; + } + + return(NULL); +} + +static void ikglp_steal_to_fq(struct ikglp_semaphore *sem, + struct fifo_queue *fq, + ikglp_wait_state_t *fq_wait) +{ + struct task_struct *t = fq_wait->task; + struct fifo_queue *fq_steal = fq_wait->donee_heap_node.fq; + + TRACE_CUR("FQ request %s/%d being moved to fq %d\n", + t->comm, + t->pid, + ikglp_get_idx(sem, fq)); + + fq_wait->donee_heap_node.fq = fq; // just to be safe + + + __remove_wait_queue(&fq_steal->wait, &fq_wait->fq_node); + --(fq_steal->count); + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_dequeue(sem->aff_obs, fq_steal, t); + } +#endif + + if(t == fq_steal->hp_waiter) { + fq_steal->hp_waiter = ikglp_find_hp_waiter(fq_steal, NULL); + TRACE_TASK(t, "New hp_waiter for fq %d is %s/%d!\n", + ikglp_get_idx(sem, fq_steal), + (fq_steal->hp_waiter) ? fq_steal->hp_waiter->comm : "nil", + (fq_steal->hp_waiter) ? fq_steal->hp_waiter->pid : -1); + } + + + // Update shortest. + if(fq_steal->count < sem->shortest_fifo_queue->count) { + sem->shortest_fifo_queue = fq_steal; + } + + __ikglp_enqueue_on_fq(sem, fq, t, + &fq_wait->fq_node, + NULL, + NULL); + + // warning: We have not checked the priority inheritance of fq's owner yet. +} + + +static void ikglp_migrate_fq_to_owner_heap_nodes(struct ikglp_semaphore *sem, + struct fifo_queue *fq, + ikglp_wait_state_t *old_wait) +{ + struct task_struct *t = old_wait->task; + + BUG_ON(old_wait->donee_heap_node.fq != fq); + + TRACE_TASK(t, "Migrating wait_state to memory of queue %d.\n", + ikglp_get_idx(sem, fq)); + + // need to migrate global_heap_node and donee_heap_node off of the stack + // to the nodes allocated for the owner of this fq. + + // TODO: Enhance binheap() to perform this operation in place. + + ikglp_del_global_list(sem, t, &old_wait->global_heap_node); // remove + fq->global_heap_node = old_wait->global_heap_node; // copy + ikglp_add_global_list(sem, t, &fq->global_heap_node); // re-add + + binheap_delete(&old_wait->donee_heap_node.node, &sem->donees); // remove + fq->donee_heap_node = old_wait->donee_heap_node; // copy + + if(fq->donee_heap_node.donor_info) { + // let donor know that our location has changed + BUG_ON(fq->donee_heap_node.donor_info->donee_info->task != t); // validate cross-link + fq->donee_heap_node.donor_info->donee_info = &fq->donee_heap_node; + } + INIT_BINHEAP_NODE(&fq->donee_heap_node.node); + binheap_add(&fq->donee_heap_node.node, &sem->donees, + ikglp_donee_heap_node_t, node); // re-add +} + +int ikglp_unlock(struct litmus_lock* l) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(l); + struct task_struct *t = current; + struct task_struct *donee = NULL; + struct task_struct *next = NULL; + struct task_struct *new_on_fq = NULL; + struct fifo_queue *fq_of_new_on_fq = NULL; + + ikglp_wait_state_t *other_donor_info = NULL; + struct fifo_queue *to_steal = NULL; + int need_steal_prio_reeval = 0; + struct fifo_queue *fq; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + raw_spinlock_t *dgl_lock; +#endif + + unsigned long flags = 0, real_flags; + + int err = 0; + + fq = ikglp_get_queue(sem, t); // returns NULL if 't' is not owner. + + if (!fq) { + err = -EINVAL; + goto out; + } + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + dgl_lock = litmus->get_dgl_spinlock(t); +#endif + raw_spin_lock_irqsave(&sem->real_lock, real_flags); + + lock_global_irqsave(dgl_lock, flags); // TODO: Push this deeper + lock_fine_irqsave(&sem->lock, flags); + + TRACE_TASK(t, "Freeing replica %d.\n", ikglp_get_idx(sem, fq)); + + + // Remove 't' from the heaps, but data in nodes will still be good. + ikglp_del_global_list(sem, t, &fq->global_heap_node); + binheap_delete(&fq->donee_heap_node.node, &sem->donees); + + fq->owner = NULL; // no longer owned!! + --(fq->count); + if(fq->count < sem->shortest_fifo_queue->count) { + sem->shortest_fifo_queue = fq; + } + --(sem->nr_in_fifos); + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_dequeue(sem->aff_obs, fq, t); + sem->aff_obs->ops->notify_freed(sem->aff_obs, fq, t); + } +#endif + + // Move the next request into the FQ and update heaps as needed. + // We defer re-evaluation of priorities to later in the function. + if(fq->donee_heap_node.donor_info) { // move my donor to FQ + ikglp_wait_state_t *donor_info = fq->donee_heap_node.donor_info; + + new_on_fq = donor_info->task; + + // donor moved to FQ + donee = t; + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs && sem->aff_obs->relax_max_fifo_len) { + fq_of_new_on_fq = sem->aff_obs->ops->advise_enqueue(sem->aff_obs, new_on_fq); + if(fq_of_new_on_fq->count == 0) { + // ignore it? +// fq_of_new_on_fq = fq; + } + } + else { + fq_of_new_on_fq = fq; + } +#else + fq_of_new_on_fq = fq; +#endif + + TRACE_TASK(t, "Moving MY donor (%s/%d) to fq %d (non-aff wanted fq %d).\n", + new_on_fq->comm, new_on_fq->pid, + ikglp_get_idx(sem, fq_of_new_on_fq), + ikglp_get_idx(sem, fq)); + + + ikglp_move_donor_to_fq(sem, fq_of_new_on_fq, donor_info); + } + else if(!binheap_empty(&sem->donors)) { // No donor, so move any donor to FQ + // move other donor to FQ + // Select a donor +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + other_donor_info = (sem->aff_obs) ? + sem->aff_obs->ops->advise_donor_to_fq(sem->aff_obs, fq) : + binheap_top_entry(&sem->donors, ikglp_wait_state_t, node); +#else + other_donor_info = binheap_top_entry(&sem->donors, ikglp_wait_state_t, node); +#endif + + new_on_fq = other_donor_info->task; + donee = other_donor_info->donee_info->task; + + // update the donee's heap position. + other_donor_info->donee_info->donor_info = NULL; // clear the cross-link + binheap_decrease(&other_donor_info->donee_info->node, &sem->donees); + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs && sem->aff_obs->relax_max_fifo_len) { + fq_of_new_on_fq = sem->aff_obs->ops->advise_enqueue(sem->aff_obs, new_on_fq); + if(fq_of_new_on_fq->count == 0) { + // ignore it? +// fq_of_new_on_fq = fq; + } + } + else { + fq_of_new_on_fq = fq; + } +#else + fq_of_new_on_fq = fq; +#endif + + TRACE_TASK(t, "Moving a donor (%s/%d) to fq %d (non-aff wanted fq %d).\n", + new_on_fq->comm, new_on_fq->pid, + ikglp_get_idx(sem, fq_of_new_on_fq), + ikglp_get_idx(sem, fq)); + + ikglp_move_donor_to_fq(sem, fq_of_new_on_fq, other_donor_info); + } + else if(!binheap_empty(&sem->priority_queue)) { // No donors, so move PQ + ikglp_heap_node_t *pq_node = binheap_top_entry(&sem->priority_queue, + ikglp_heap_node_t, node); + ikglp_wait_state_t *pq_wait = container_of(pq_node, ikglp_wait_state_t, + pq_node); + + new_on_fq = pq_wait->task; + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs && sem->aff_obs->relax_max_fifo_len) { + fq_of_new_on_fq = sem->aff_obs->ops->advise_enqueue(sem->aff_obs, new_on_fq); + if(fq_of_new_on_fq->count == 0) { + // ignore it? +// fq_of_new_on_fq = fq; + } + } + else { + fq_of_new_on_fq = fq; + } +#else + fq_of_new_on_fq = fq; +#endif + + TRACE_TASK(t, "Moving a pq waiter (%s/%d) to fq %d (non-aff wanted fq %d).\n", + new_on_fq->comm, new_on_fq->pid, + ikglp_get_idx(sem, fq_of_new_on_fq), + ikglp_get_idx(sem, fq)); + + ikglp_move_pq_to_fq(sem, fq_of_new_on_fq, pq_wait); + } + else if(fq->count == 0) { // No PQ and this queue is empty, so steal. + ikglp_wait_state_t *fq_wait; + + TRACE_TASK(t, "Looking to steal a request for fq %d...\n", + ikglp_get_idx(sem, fq)); + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + fq_wait = (sem->aff_obs) ? + sem->aff_obs->ops->advise_steal(sem->aff_obs, fq) : + ikglp_find_hp_waiter_to_steal(sem); +#else + fq_wait = ikglp_find_hp_waiter_to_steal(sem); +#endif + + if(fq_wait) { + to_steal = fq_wait->donee_heap_node.fq; + + new_on_fq = fq_wait->task; + fq_of_new_on_fq = fq; + need_steal_prio_reeval = (new_on_fq == to_steal->hp_waiter); + + TRACE_TASK(t, "Found %s/%d of fq %d to steal for fq %d...\n", + new_on_fq->comm, new_on_fq->pid, + ikglp_get_idx(sem, to_steal), + ikglp_get_idx(sem, fq)); + + ikglp_steal_to_fq(sem, fq, fq_wait); + } + else { + TRACE_TASK(t, "Found nothing to steal for fq %d.\n", + ikglp_get_idx(sem, fq)); + } + } + else { // move no one + } + + // 't' must drop all priority and clean up data structures before hand-off. + + // DROP ALL INHERITANCE. IKGLP MUST BE OUTER-MOST + raw_spin_lock(&tsk_rt(t)->hp_blocked_tasks_lock); + { + int count = 0; + while(!binheap_empty(&tsk_rt(t)->hp_blocked_tasks)) { + binheap_delete_root(&tsk_rt(t)->hp_blocked_tasks, + struct nested_info, hp_binheap_node); + ++count; + } + litmus->decrease_prio(t, NULL); + WARN_ON(count > 2); // should not be greater than 2. only local fq inh and donation can be possible. + } + raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); + + + + // Now patch up other priorities. + // + // At most one of the following: + // if(donee && donee != t), decrease prio, propagate to owner, or onward + // if(to_steal), update owner's prio (hp_waiter has already been set) + // + + BUG_ON((other_donor_info != NULL) && (to_steal != NULL)); + + if(other_donor_info) { + struct fifo_queue *other_fq = other_donor_info->donee_info->fq; + + BUG_ON(!donee); + BUG_ON(donee == t); + + TRACE_TASK(t, "Terminating donation relation of donor %s/%d to donee %s/%d!\n", + other_donor_info->task->comm, other_donor_info->task->pid, + donee->comm, donee->pid); + + // need to terminate donation relation. + if(donee == other_fq->owner) { + TRACE_TASK(t, "Donee %s/%d is an owner of fq %d.\n", + donee->comm, donee->pid, + ikglp_get_idx(sem, other_fq)); + + ikglp_remove_donation_from_owner(&other_donor_info->prio_donation.hp_binheap_node, other_fq, sem, flags); + lock_fine_irqsave(&sem->lock, flags); // there should be no contention!!!! + } + else { + TRACE_TASK(t, "Donee %s/%d is an blocked in of fq %d.\n", + donee->comm, donee->pid, + ikglp_get_idx(sem, other_fq)); + + ikglp_remove_donation_from_fq_waiter(donee, &other_donor_info->prio_donation.hp_binheap_node); + if(donee == other_fq->hp_waiter) { + TRACE_TASK(t, "Donee %s/%d was an hp_waiter of fq %d. Rechecking hp_waiter.\n", + donee->comm, donee->pid, + ikglp_get_idx(sem, other_fq)); + + other_fq->hp_waiter = ikglp_find_hp_waiter(other_fq, NULL); + TRACE_TASK(t, "New hp_waiter for fq %d is %s/%d!\n", + ikglp_get_idx(sem, other_fq), + (other_fq->hp_waiter) ? other_fq->hp_waiter->comm : "nil", + (other_fq->hp_waiter) ? other_fq->hp_waiter->pid : -1); + + ikglp_refresh_owners_prio_decrease(other_fq, sem, flags); // unlocks sem->lock. reacquire it. + lock_fine_irqsave(&sem->lock, flags); // there should be no contention!!!! + } + } + } + else if(to_steal) { + TRACE_TASK(t, "Rechecking priority inheritance of fq %d, triggered by stealing.\n", + ikglp_get_idx(sem, to_steal)); + + if(need_steal_prio_reeval) { + ikglp_refresh_owners_prio_decrease(to_steal, sem, flags); // unlocks sem->lock. reacquire it. + lock_fine_irqsave(&sem->lock, flags); // there should be no contention!!!! + } + } + + // check for new HP waiter. + if(new_on_fq) { + if(fq == fq_of_new_on_fq) { + // fq->owner is null, so just update the hp_waiter without locking. + if(new_on_fq == fq->hp_waiter) { + TRACE_TASK(t, "new_on_fq is already hp_waiter.\n", + fq->hp_waiter->comm, fq->hp_waiter->pid); + fq->nest.hp_waiter_eff_prio = effective_priority(fq->hp_waiter); // set this just to be sure... + } + else if(litmus->compare(new_on_fq, fq->hp_waiter)) { + if(fq->hp_waiter) + TRACE_TASK(t, "has higher prio than hp_waiter (%s/%d).\n", + fq->hp_waiter->comm, fq->hp_waiter->pid); + else + TRACE_TASK(t, "has higher prio than hp_waiter (NIL).\n"); + + fq->hp_waiter = new_on_fq; + fq->nest.hp_waiter_eff_prio = effective_priority(fq->hp_waiter); + + TRACE_TASK(t, "New hp_waiter for fq %d is %s/%d!\n", + ikglp_get_idx(sem, fq), + (fq->hp_waiter) ? fq->hp_waiter->comm : "nil", + (fq->hp_waiter) ? fq->hp_waiter->pid : -1); + } + } + else { + ikglp_refresh_owners_prio_increase(new_on_fq, fq_of_new_on_fq, sem, flags); // unlocks sem->lock. reacquire it. + lock_fine_irqsave(&sem->lock, flags); // there should be no contention!!!! + } + } + +wake_kludge: + if(waitqueue_active(&fq->wait)) + { + wait_queue_t *wait = list_entry(fq->wait.task_list.next, wait_queue_t, task_list); + ikglp_wait_state_t *fq_wait = container_of(wait, ikglp_wait_state_t, fq_node); + next = (struct task_struct*) wait->private; + + __remove_wait_queue(&fq->wait, wait); + + TRACE_CUR("queue %d: ASSIGNING %s/%d as owner - next\n", + ikglp_get_idx(sem, fq), + next->comm, next->pid); + + // migrate wait-state to fifo-memory. + ikglp_migrate_fq_to_owner_heap_nodes(sem, fq, fq_wait); + + /* next becomes the resouce holder */ + fq->owner = next; + tsk_rt(next)->blocked_lock = NULL; + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_acquired(sem->aff_obs, fq, next); + } +#endif + + /* determine new hp_waiter if necessary */ + if (next == fq->hp_waiter) { + + TRACE_TASK(next, "was highest-prio waiter\n"); + /* next has the highest priority --- it doesn't need to + * inherit. However, we need to make sure that the + * next-highest priority in the queue is reflected in + * hp_waiter. */ + fq->hp_waiter = ikglp_find_hp_waiter(fq, NULL); + TRACE_TASK(next, "New hp_waiter for fq %d is %s/%d!\n", + ikglp_get_idx(sem, fq), + (fq->hp_waiter) ? fq->hp_waiter->comm : "nil", + (fq->hp_waiter) ? fq->hp_waiter->pid : -1); + + fq->nest.hp_waiter_eff_prio = (fq->hp_waiter) ? + effective_priority(fq->hp_waiter) : NULL; + + if (fq->hp_waiter) + TRACE_TASK(fq->hp_waiter, "is new highest-prio waiter\n"); + else + TRACE("no further waiters\n"); + + raw_spin_lock(&tsk_rt(next)->hp_blocked_tasks_lock); + +// TRACE_TASK(next, "Heap Before:\n"); +// print_hp_waiters(tsk_rt(next)->hp_blocked_tasks.root, 0); + + binheap_add(&fq->nest.hp_binheap_node, + &tsk_rt(next)->hp_blocked_tasks, + struct nested_info, + hp_binheap_node); + +// TRACE_TASK(next, "Heap After:\n"); +// print_hp_waiters(tsk_rt(next)->hp_blocked_tasks.root, 0); + + raw_spin_unlock(&tsk_rt(next)->hp_blocked_tasks_lock); + } + else { + /* Well, if 'next' is not the highest-priority waiter, + * then it (probably) ought to inherit the highest-priority + * waiter's priority. */ + TRACE_TASK(next, "is not hp_waiter of replica %d. hp_waiter is %s/%d\n", + ikglp_get_idx(sem, fq), + (fq->hp_waiter) ? fq->hp_waiter->comm : "nil", + (fq->hp_waiter) ? fq->hp_waiter->pid : -1); + + raw_spin_lock(&tsk_rt(next)->hp_blocked_tasks_lock); + + binheap_add(&fq->nest.hp_binheap_node, + &tsk_rt(next)->hp_blocked_tasks, + struct nested_info, + hp_binheap_node); + + /* It is possible that 'next' *should* be the hp_waiter, but isn't + * because that update hasn't yet executed (update operation is + * probably blocked on mutex->lock). So only inherit if the top of + * 'next's top heap node is indeed the effective prio. of hp_waiter. + * (We use fq->hp_waiter_eff_prio instead of effective_priority(hp_waiter) + * since the effective priority of hp_waiter can change (and the + * update has not made it to this lock).) + */ + if(likely(top_priority(&tsk_rt(next)->hp_blocked_tasks) == + fq->nest.hp_waiter_eff_prio)) + { + if(fq->nest.hp_waiter_eff_prio) + litmus->increase_prio(next, fq->nest.hp_waiter_eff_prio); + else + WARN_ON(1); + } + + raw_spin_unlock(&tsk_rt(next)->hp_blocked_tasks_lock); + } + + + // wake up the new resource holder! + wake_up_process(next); + } + if(fq_of_new_on_fq && fq_of_new_on_fq != fq && fq_of_new_on_fq->count == 1) { + // The guy we promoted when to an empty FQ. (Why didn't stealing pick this up?) + // Wake up the new guy too. + + BUG_ON(fq_of_new_on_fq->owner != NULL); + + fq = fq_of_new_on_fq; + fq_of_new_on_fq = NULL; + goto wake_kludge; + } + + unlock_fine_irqrestore(&sem->lock, flags); + unlock_global_irqrestore(dgl_lock, flags); + + raw_spin_unlock_irqrestore(&sem->real_lock, real_flags); + +out: + return err; +} + + + +int ikglp_close(struct litmus_lock* l) +{ + struct task_struct *t = current; + struct ikglp_semaphore *sem = ikglp_from_lock(l); + unsigned long flags; + + int owner = 0; + int i; + + raw_spin_lock_irqsave(&sem->real_lock, flags); + + for(i = 0; i < sem->nr_replicas; ++i) { + if(sem->fifo_queues[i].owner == t) { + owner = 1; + break; + } + } + + raw_spin_unlock_irqrestore(&sem->real_lock, flags); + + if (owner) + ikglp_unlock(l); + + return 0; +} + +void ikglp_free(struct litmus_lock* l) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(l); + + kfree(sem->fifo_queues); + kfree(sem); +} + + + +struct litmus_lock* ikglp_new(int m, + struct litmus_lock_ops* ops, + void* __user arg) +{ + struct ikglp_semaphore* sem; + int nr_replicas = 0; + int i; + + if(!access_ok(VERIFY_READ, arg, sizeof(nr_replicas))) + { + return(NULL); + } + if(__copy_from_user(&nr_replicas, arg, sizeof(nr_replicas))) + { + return(NULL); + } + if(nr_replicas < 1) + { + return(NULL); + } + + sem = kmalloc(sizeof(*sem), GFP_KERNEL); + if(!sem) + { + return NULL; + } + + sem->fifo_queues = kmalloc(sizeof(struct fifo_queue)*nr_replicas, GFP_KERNEL); + if(!sem->fifo_queues) + { + kfree(sem); + return NULL; + } + + sem->litmus_lock.ops = ops; + +#ifdef CONFIG_DEBUG_SPINLOCK + { + __raw_spin_lock_init(&sem->lock, ((struct litmus_lock*)sem)->cheat_lockdep, &((struct litmus_lock*)sem)->key); + } +#else + raw_spin_lock_init(&sem->lock); +#endif + + raw_spin_lock_init(&sem->real_lock); + + sem->nr_replicas = nr_replicas; + sem->m = m; + sem->max_fifo_len = (sem->m/nr_replicas) + ((sem->m%nr_replicas) != 0); + sem->nr_in_fifos = 0; + + TRACE("New IKGLP Sem: m = %d, k = %d, max fifo_len = %d\n", + sem->m, + sem->nr_replicas, + sem->max_fifo_len); + + for(i = 0; i < nr_replicas; ++i) + { + struct fifo_queue* q = &(sem->fifo_queues[i]); + + q->owner = NULL; + q->hp_waiter = NULL; + init_waitqueue_head(&q->wait); + q->count = 0; + + q->global_heap_node.task = NULL; + INIT_BINHEAP_NODE(&q->global_heap_node.node); + + q->donee_heap_node.task = NULL; + q->donee_heap_node.donor_info = NULL; + q->donee_heap_node.fq = NULL; + INIT_BINHEAP_NODE(&q->donee_heap_node.node); + + q->nest.lock = (struct litmus_lock*)sem; + q->nest.hp_waiter_eff_prio = NULL; + q->nest.hp_waiter_ptr = &q->hp_waiter; + INIT_BINHEAP_NODE(&q->nest.hp_binheap_node); + } + + sem->shortest_fifo_queue = &sem->fifo_queues[0]; + + sem->top_m_size = 0; + + // init heaps + INIT_BINHEAP_HANDLE(&sem->top_m, ikglp_min_heap_base_priority_order); + INIT_BINHEAP_HANDLE(&sem->not_top_m, ikglp_max_heap_base_priority_order); + INIT_BINHEAP_HANDLE(&sem->donees, ikglp_min_heap_donee_order); + INIT_BINHEAP_HANDLE(&sem->priority_queue, ikglp_max_heap_base_priority_order); + INIT_BINHEAP_HANDLE(&sem->donors, ikglp_donor_max_heap_base_priority_order); + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + sem->aff_obs = NULL; +#endif + + return &sem->litmus_lock; +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) + +static inline int __replica_to_gpu(struct ikglp_affinity* aff, int replica) +{ + int gpu = replica % aff->nr_rsrc; + return gpu; +} + +static inline int replica_to_gpu(struct ikglp_affinity* aff, int replica) +{ + int gpu = __replica_to_gpu(aff, replica) + aff->offset; + return gpu; +} + +static inline int gpu_to_base_replica(struct ikglp_affinity* aff, int gpu) +{ + int replica = gpu - aff->offset; + return replica; +} + + +int ikglp_aff_obs_close(struct affinity_observer* obs) +{ + return 0; +} + +void ikglp_aff_obs_free(struct affinity_observer* obs) +{ + struct ikglp_affinity *ikglp_aff = ikglp_aff_obs_from_aff_obs(obs); + kfree(ikglp_aff->nr_cur_users_on_rsrc); + kfree(ikglp_aff->q_info); + kfree(ikglp_aff); +} + +static struct affinity_observer* ikglp_aff_obs_new(struct affinity_observer_ops* ops, + struct ikglp_affinity_ops* ikglp_ops, + void* __user args) +{ + struct ikglp_affinity* ikglp_aff; + struct gpu_affinity_observer_args aff_args; + struct ikglp_semaphore* sem; + int i; + unsigned long flags; + + if(!access_ok(VERIFY_READ, args, sizeof(aff_args))) { + return(NULL); + } + if(__copy_from_user(&aff_args, args, sizeof(aff_args))) { + return(NULL); + } + + sem = (struct ikglp_semaphore*) get_lock_from_od(aff_args.obs.lock_od); + + if(sem->litmus_lock.type != IKGLP_SEM) { + TRACE_CUR("Lock type not supported. Type = %d\n", sem->litmus_lock.type); + return(NULL); + } + + if((aff_args.nr_simult_users <= 0) || + (sem->nr_replicas%aff_args.nr_simult_users != 0)) { + TRACE_CUR("Lock %d does not support #replicas (%d) for #simult_users " + "(%d) per replica. #replicas should be evenly divisible " + "by #simult_users.\n", + sem->litmus_lock.ident, + sem->nr_replicas, + aff_args.nr_simult_users); + return(NULL); + } + + if(aff_args.nr_simult_users > NV_MAX_SIMULT_USERS) { + TRACE_CUR("System does not support #simult_users > %d. %d requested.\n", + NV_MAX_SIMULT_USERS, aff_args.nr_simult_users); +// return(NULL); + } + + ikglp_aff = kmalloc(sizeof(*ikglp_aff), GFP_KERNEL); + if(!ikglp_aff) { + return(NULL); + } + + ikglp_aff->q_info = kmalloc(sizeof(struct ikglp_queue_info)*sem->nr_replicas, GFP_KERNEL); + if(!ikglp_aff->q_info) { + kfree(ikglp_aff); + return(NULL); + } + + ikglp_aff->nr_cur_users_on_rsrc = kmalloc(sizeof(int)*(sem->nr_replicas / aff_args.nr_simult_users), GFP_KERNEL); + if(!ikglp_aff->nr_cur_users_on_rsrc) { + kfree(ikglp_aff->q_info); + kfree(ikglp_aff); + return(NULL); + } + + affinity_observer_new(&ikglp_aff->obs, ops, &aff_args.obs); + + ikglp_aff->ops = ikglp_ops; + ikglp_aff->offset = aff_args.replica_to_gpu_offset; + ikglp_aff->nr_simult = aff_args.nr_simult_users; + ikglp_aff->nr_rsrc = sem->nr_replicas / ikglp_aff->nr_simult; + ikglp_aff->relax_max_fifo_len = (aff_args.relaxed_rules) ? 1 : 0; + + TRACE_CUR("GPU affinity_observer: offset = %d, nr_simult = %d, " + "nr_rsrc = %d, relaxed_fifo_len = %d\n", + ikglp_aff->offset, ikglp_aff->nr_simult, ikglp_aff->nr_rsrc, + ikglp_aff->relax_max_fifo_len); + + memset(ikglp_aff->nr_cur_users_on_rsrc, 0, sizeof(int)*(ikglp_aff->nr_rsrc)); + + for(i = 0; i < sem->nr_replicas; ++i) { + ikglp_aff->q_info[i].q = &sem->fifo_queues[i]; + ikglp_aff->q_info[i].estimated_len = 0; + + // multiple q_info's will point to the same resource (aka GPU) if + // aff_args.nr_simult_users > 1 + ikglp_aff->q_info[i].nr_cur_users = &ikglp_aff->nr_cur_users_on_rsrc[__replica_to_gpu(ikglp_aff,i)]; + } + + // attach observer to the lock + raw_spin_lock_irqsave(&sem->real_lock, flags); + sem->aff_obs = ikglp_aff; + raw_spin_unlock_irqrestore(&sem->real_lock, flags); + + return &ikglp_aff->obs; +} + + + + +static int gpu_replica_to_resource(struct ikglp_affinity* aff, + struct fifo_queue* fq) { + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + return(replica_to_gpu(aff, ikglp_get_idx(sem, fq))); +} + + +// Smart IKGLP Affinity + +//static inline struct ikglp_queue_info* ikglp_aff_find_shortest(struct ikglp_affinity* aff) +//{ +// struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); +// struct ikglp_queue_info *shortest = &aff->q_info[0]; +// int i; +// +// for(i = 1; i < sem->nr_replicas; ++i) { +// if(aff->q_info[i].estimated_len < shortest->estimated_len) { +// shortest = &aff->q_info[i]; +// } +// } +// +// return(shortest); +//} + +struct fifo_queue* gpu_ikglp_advise_enqueue(struct ikglp_affinity* aff, struct task_struct* t) +{ + // advise_enqueue must be smart as not not break IKGLP rules: + // * No queue can be greater than ceil(m/k) in length. We may return + // such a queue, but IKGLP will be smart enough as to send requests + // to donors or PQ. + // * Cannot let a queue idle if there exist waiting PQ/donors + // -- needed to guarantee parallel progress of waiters. + // + // We may be able to relax some of these constraints, but this will have to + // be carefully evaluated. + // + // Huristic strategy: Find the shortest queue that is not full. + + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + lt_t min_len; + int min_nr_users; + struct ikglp_queue_info *shortest; + struct fifo_queue *to_enqueue; + int i; + int affinity_gpu; + + int max_fifo_len = (aff->relax_max_fifo_len) ? + sem->m : sem->max_fifo_len; + + // simply pick the shortest queue if, we have no affinity, or we have + // affinity with the shortest + if(unlikely(tsk_rt(t)->last_gpu < 0)) { + affinity_gpu = aff->offset; // first gpu + TRACE_CUR("no affinity\n"); + } + else { + affinity_gpu = tsk_rt(t)->last_gpu; + } + + // all things being equal, let's start with the queue with which we have + // affinity. this helps us maintain affinity even when we don't have + // an estiamte for local-affinity execution time (i.e., 2nd time on GPU) + shortest = &aff->q_info[gpu_to_base_replica(aff, affinity_gpu)]; + + // if(shortest == aff->shortest_queue) { + // TRACE_CUR("special case: have affinity with shortest queue\n"); + // goto out; + // } + + min_len = shortest->estimated_len + get_gpu_estimate(t, MIG_LOCAL); + min_nr_users = *(shortest->nr_cur_users); + + TRACE_CUR("cs is %llu on queue %d (count = %d): est len = %llu\n", + get_gpu_estimate(t, MIG_LOCAL), + ikglp_get_idx(sem, shortest->q), + shortest->q->count, + min_len); + + for(i = 0; i < sem->nr_replicas; ++i) { + if(&aff->q_info[i] != shortest) { + if(aff->q_info[i].q->count < max_fifo_len) { + + lt_t est_len = + aff->q_info[i].estimated_len + + get_gpu_estimate(t, + gpu_migration_distance(tsk_rt(t)->last_gpu, + replica_to_gpu(aff, i))); + + // queue is smaller, or they're equal and the other has a smaller number + // of total users. + // + // tie-break on the shortest number of simult users. this only kicks in + // when there are more than 1 empty queues. + if((shortest->q->count >= max_fifo_len) || /* 'shortest' is full and i-th queue is not */ + (est_len < min_len) || /* i-th queue has shortest length */ + ((est_len == min_len) && /* equal lengths, but one has fewer over-all users */ + (*(aff->q_info[i].nr_cur_users) < min_nr_users))) { + + shortest = &aff->q_info[i]; + min_len = est_len; + min_nr_users = *(aff->q_info[i].nr_cur_users); + } + + TRACE_CUR("cs is %llu on queue %d (count = %d): est len = %llu\n", + get_gpu_estimate(t, + gpu_migration_distance(tsk_rt(t)->last_gpu, + replica_to_gpu(aff, i))), + ikglp_get_idx(sem, aff->q_info[i].q), + aff->q_info[i].q->count, + est_len); + } + else { + TRACE_CUR("queue %d is too long. ineligible for enqueue.\n", + ikglp_get_idx(sem, aff->q_info[i].q)); + } + } + } + + if(shortest->q->count >= max_fifo_len) { + TRACE_CUR("selected fq %d is too long, but returning it anyway.\n", + ikglp_get_idx(sem, shortest->q)); + } + + to_enqueue = shortest->q; + TRACE_CUR("enqueue on fq %d (count = %d) (non-aff wanted fq %d)\n", + ikglp_get_idx(sem, to_enqueue), + to_enqueue->count, + ikglp_get_idx(sem, sem->shortest_fifo_queue)); + + return to_enqueue; + + //return(sem->shortest_fifo_queue); +} + + + + +static ikglp_wait_state_t* pick_steal(struct ikglp_affinity* aff, + int dest_gpu, + struct fifo_queue* fq) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + ikglp_wait_state_t *wait = NULL; + int max_improvement = -(MIG_NONE+1); + int replica = ikglp_get_idx(sem, fq); + + if(waitqueue_active(&fq->wait)) { + int this_gpu = replica_to_gpu(aff, replica); + struct list_head *pos; + + list_for_each(pos, &fq->wait.task_list) { + wait_queue_t *fq_wait = list_entry(pos, wait_queue_t, task_list); + ikglp_wait_state_t *tmp_wait = container_of(fq_wait, ikglp_wait_state_t, fq_node); + + int tmp_improvement = + gpu_migration_distance(this_gpu, tsk_rt(tmp_wait->task)->last_gpu) - + gpu_migration_distance(dest_gpu, tsk_rt(tmp_wait->task)->last_gpu); + + if(tmp_improvement > max_improvement) { + wait = tmp_wait; + max_improvement = tmp_improvement; + + if(max_improvement >= (MIG_NONE-1)) { + goto out; + } + } + } + + BUG_ON(!wait); + } + else { + TRACE_CUR("fq %d is empty!\n", replica); + } + +out: + + TRACE_CUR("Candidate victim from fq %d is %s/%d. aff improvement = %d.\n", + replica, + (wait) ? wait->task->comm : "nil", + (wait) ? wait->task->pid : -1, + max_improvement); + + return wait; +} + + +ikglp_wait_state_t* gpu_ikglp_advise_steal(struct ikglp_affinity* aff, + struct fifo_queue* dst) +{ + // Huristic strategy: Find task with greatest improvement in affinity. + // + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + ikglp_wait_state_t *to_steal_state = NULL; +// ikglp_wait_state_t *default_to_steal_state = ikglp_find_hp_waiter_to_steal(sem); + int max_improvement = -(MIG_NONE+1); + int replica, i; + int dest_gpu; + + replica = ikglp_get_idx(sem, dst); + dest_gpu = replica_to_gpu(aff, replica); + + for(i = 0; i < sem->nr_replicas; ++i) { + ikglp_wait_state_t *tmp_to_steal_state = + pick_steal(aff, dest_gpu, &sem->fifo_queues[i]); + + if(tmp_to_steal_state) { + int tmp_improvement = + gpu_migration_distance(replica_to_gpu(aff, i), tsk_rt(tmp_to_steal_state->task)->last_gpu) - + gpu_migration_distance(dest_gpu, tsk_rt(tmp_to_steal_state->task)->last_gpu); + + if(tmp_improvement > max_improvement) { + to_steal_state = tmp_to_steal_state; + max_improvement = tmp_improvement; + + if(max_improvement >= (MIG_NONE-1)) { + goto out; + } + } + } + } + +out: + if(!to_steal_state) { + TRACE_CUR("Could not find anyone to steal.\n"); + } + else { + TRACE_CUR("Selected victim %s/%d on fq %d (GPU %d) for fq %d (GPU %d): improvement = %d\n", + to_steal_state->task->comm, to_steal_state->task->pid, + ikglp_get_idx(sem, to_steal_state->donee_heap_node.fq), + replica_to_gpu(aff, ikglp_get_idx(sem, to_steal_state->donee_heap_node.fq)), + ikglp_get_idx(sem, dst), + dest_gpu, + max_improvement); + +// TRACE_CUR("Non-aff wanted to select victim %s/%d on fq %d (GPU %d) for fq %d (GPU %d): improvement = %d\n", +// default_to_steal_state->task->comm, default_to_steal_state->task->pid, +// ikglp_get_idx(sem, default_to_steal_state->donee_heap_node.fq), +// replica_to_gpu(aff, ikglp_get_idx(sem, default_to_steal_state->donee_heap_node.fq)), +// ikglp_get_idx(sem, dst), +// replica_to_gpu(aff, ikglp_get_idx(sem, dst)), +// +// gpu_migration_distance( +// replica_to_gpu(aff, ikglp_get_idx(sem, default_to_steal_state->donee_heap_node.fq)), +// tsk_rt(default_to_steal_state->task)->last_gpu) - +// gpu_migration_distance(dest_gpu, tsk_rt(default_to_steal_state->task)->last_gpu)); + } + + return(to_steal_state); +} + + +static inline int has_donor(wait_queue_t* fq_wait) +{ + ikglp_wait_state_t *wait = container_of(fq_wait, ikglp_wait_state_t, fq_node); + return(wait->donee_heap_node.donor_info != NULL); +} + +static ikglp_donee_heap_node_t* pick_donee(struct ikglp_affinity* aff, + struct fifo_queue* fq, + int* dist_from_head) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + struct task_struct *donee; + ikglp_donee_heap_node_t *donee_node; + struct task_struct *mth_highest = ikglp_mth_highest(sem); + +// lt_t now = litmus_clock(); +// +// TRACE_CUR("fq %d: mth_highest: %s/%d, deadline = %d: (donor) = ??? ", +// ikglp_get_idx(sem, fq), +// mth_highest->comm, mth_highest->pid, +// (int)get_deadline(mth_highest) - now); + + if(fq->owner && + fq->donee_heap_node.donor_info == NULL && + mth_highest != fq->owner && + litmus->__compare(mth_highest, BASE, fq->owner, BASE)) { + donee = fq->owner; + donee_node = &(fq->donee_heap_node); + *dist_from_head = 0; + + BUG_ON(donee != donee_node->task); + + TRACE_CUR("picked owner of fq %d as donee\n", + ikglp_get_idx(sem, fq)); + + goto out; + } + else if(waitqueue_active(&fq->wait)) { + struct list_head *pos; + + +// TRACE_CUR("fq %d: owner: %s/%d, deadline = %d: (donor) = %s/%d " +// "(mth_highest != fq->owner) = %d " +// "(mth_highest > fq->owner) = %d\n", +// ikglp_get_idx(sem, fq), +// (fq->owner) ? fq->owner->comm : "nil", +// (fq->owner) ? fq->owner->pid : -1, +// (fq->owner) ? (int)get_deadline(fq->owner) - now : -999, +// (fq->donee_heap_node.donor_info) ? fq->donee_heap_node.donor_info->task->comm : "nil", +// (fq->donee_heap_node.donor_info) ? fq->donee_heap_node.donor_info->task->pid : -1, +// (mth_highest != fq->owner), +// (litmus->__compare(mth_highest, BASE, fq->owner, BASE))); + + + *dist_from_head = 1; + + // iterating from the start of the queue is nice since this means + // the donee will be closer to obtaining a resource. + list_for_each(pos, &fq->wait.task_list) { + wait_queue_t *fq_wait = list_entry(pos, wait_queue_t, task_list); + ikglp_wait_state_t *wait = container_of(fq_wait, ikglp_wait_state_t, fq_node); + +// TRACE_CUR("fq %d: waiter %d: %s/%d, deadline = %d (donor) = %s/%d " +// "(mth_highest != wait->task) = %d " +// "(mth_highest > wait->task) = %d\n", +// ikglp_get_idx(sem, fq), +// dist_from_head, +// wait->task->comm, wait->task->pid, +// (int)get_deadline(wait->task) - now, +// (wait->donee_heap_node.donor_info) ? wait->donee_heap_node.donor_info->task->comm : "nil", +// (wait->donee_heap_node.donor_info) ? wait->donee_heap_node.donor_info->task->pid : -1, +// (mth_highest != wait->task), +// (litmus->__compare(mth_highest, BASE, wait->task, BASE))); + + + if(!has_donor(fq_wait) && + mth_highest != wait->task && + litmus->__compare(mth_highest, BASE, wait->task, BASE)) { + donee = (struct task_struct*) fq_wait->private; + donee_node = &wait->donee_heap_node; + + BUG_ON(donee != donee_node->task); + + TRACE_CUR("picked waiter in fq %d as donee\n", + ikglp_get_idx(sem, fq)); + + goto out; + } + ++(*dist_from_head); + } + } + + donee = NULL; + donee_node = NULL; + //*dist_from_head = sem->max_fifo_len + 1; + *dist_from_head = IKGLP_INVAL_DISTANCE; + + TRACE_CUR("Found no one to be donee in fq %d!\n", ikglp_get_idx(sem, fq)); + +out: + + TRACE_CUR("Candidate donee for fq %d is %s/%d (dist_from_head = %d)\n", + ikglp_get_idx(sem, fq), + (donee) ? (donee)->comm : "nil", + (donee) ? (donee)->pid : -1, + *dist_from_head); + + return donee_node; +} + +ikglp_donee_heap_node_t* gpu_ikglp_advise_donee_selection( + struct ikglp_affinity* aff, + struct task_struct* donor) +{ + // Huristic strategy: Find the highest-priority donee that is waiting on + // a queue closest to our affinity. (1) The donee CANNOT already have a + // donor (exception: donee is the lowest-prio task in the donee heap). + // (2) Requests in 'top_m' heap are ineligible. + // + // Further strategy: amongst elible donees waiting for the same GPU, pick + // the one closest to the head of the FIFO queue (including owners). + // + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + ikglp_donee_heap_node_t *donee_node; + gpu_migration_dist_t distance; + int start, i, j; + + ikglp_donee_heap_node_t *default_donee; + ikglp_wait_state_t *default_donee_donor_info; + + if(tsk_rt(donor)->last_gpu < 0) { + // no affinity. just return the min prio, like standard IKGLP + // TODO: Find something closer to the head of the queue?? + donee_node = binheap_top_entry(&sem->donees, + ikglp_donee_heap_node_t, + node); + goto out; + } + + + // Temporarily break any donation relation the default donee (the lowest + // prio task in the FIFO queues) to make it eligible for selection below. + // + // NOTE: The original donor relation *must* be restored, even if we select + // the default donee throug affinity-aware selection, before returning + // from this function so we don't screw up our heap ordering. + // The standard IKGLP algorithm will steal the donor relationship if needed. + default_donee = binheap_top_entry(&sem->donees, ikglp_donee_heap_node_t, node); + default_donee_donor_info = default_donee->donor_info; // back-up donor relation + default_donee->donor_info = NULL; // temporarily break any donor relation. + + // initialize our search + donee_node = NULL; + distance = MIG_NONE; + + // TODO: The below search logic may work well for locating nodes to steal + // when an FQ goes idle. Validate this code and apply it to stealing. + + // begin search with affinity GPU. + start = gpu_to_base_replica(aff, tsk_rt(donor)->last_gpu); + i = start; + do { // "for each gpu" / "for each aff->nr_rsrc" + gpu_migration_dist_t temp_distance = gpu_migration_distance(start, i); + + // only interested in queues that will improve our distance + if(temp_distance < distance || donee_node == NULL) { + int dist_from_head = IKGLP_INVAL_DISTANCE; + + TRACE_CUR("searching for donor on GPU %d", i); + + // visit each queue and pick a donee. bail as soon as we find + // one for this class. + + for(j = 0; j < aff->nr_simult; ++j) { + int temp_dist_from_head; + ikglp_donee_heap_node_t *temp_donee_node; + struct fifo_queue *fq; + + fq = &(sem->fifo_queues[i + j*aff->nr_rsrc]); + temp_donee_node = pick_donee(aff, fq, &temp_dist_from_head); + + if(temp_dist_from_head < dist_from_head) + { + // we check all the FQs for this GPU to spread priorities + // out across the queues. does this decrease jitter? + donee_node = temp_donee_node; + dist_from_head = temp_dist_from_head; + } + } + + if(dist_from_head != IKGLP_INVAL_DISTANCE) { + TRACE_CUR("found donee %s/%d and is the %d-th waiter.\n", + donee_node->task->comm, donee_node->task->pid, + dist_from_head); + } + else { + TRACE_CUR("found no eligible donors from GPU %d\n", i); + } + } + else { + TRACE_CUR("skipping GPU %d (distance = %d, best donor " + "distance = %d)\n", i, temp_distance, distance); + } + + i = (i+1 < aff->nr_rsrc) ? i+1 : 0; // increment with wrap-around + } while (i != start); + + + // restore old donor info state. + default_donee->donor_info = default_donee_donor_info; + + if(!donee_node) { + donee_node = default_donee; + + TRACE_CUR("Could not find a donee. We have to steal one.\n"); + WARN_ON(default_donee->donor_info == NULL); + } + +out: + + TRACE_CUR("Selected donee %s/%d on fq %d (GPU %d) for %s/%d with affinity for GPU %d\n", + donee_node->task->comm, donee_node->task->pid, + ikglp_get_idx(sem, donee_node->fq), + replica_to_gpu(aff, ikglp_get_idx(sem, donee_node->fq)), + donor->comm, donor->pid, tsk_rt(donor)->last_gpu); + + return(donee_node); +} + + + +static void __find_closest_donor(int target_gpu, + struct binheap_node* donor_node, + ikglp_wait_state_t** cur_closest, + int* cur_dist) +{ + ikglp_wait_state_t *this_donor = + binheap_entry(donor_node, ikglp_wait_state_t, node); + + int this_dist = + gpu_migration_distance(target_gpu, tsk_rt(this_donor->task)->last_gpu); + +// TRACE_CUR("%s/%d: dist from target = %d\n", +// this_donor->task->comm, +// this_donor->task->pid, +// this_dist); + + if(this_dist < *cur_dist) { + // take this donor + *cur_dist = this_dist; + *cur_closest = this_donor; + } + else if(this_dist == *cur_dist) { + // priority tie-break. Even though this is a pre-order traversal, + // this is a heap, not a binary tree, so we still need to do a priority + // comparision. + if(!(*cur_closest) || + litmus->compare(this_donor->task, (*cur_closest)->task)) { + *cur_dist = this_dist; + *cur_closest = this_donor; + } + } + + if(donor_node->left) __find_closest_donor(target_gpu, donor_node->left, cur_closest, cur_dist); + if(donor_node->right) __find_closest_donor(target_gpu, donor_node->right, cur_closest, cur_dist); +} + +ikglp_wait_state_t* gpu_ikglp_advise_donor_to_fq(struct ikglp_affinity* aff, struct fifo_queue* fq) +{ + // Huristic strategy: Find donor with the closest affinity to fq. + // Tie-break on priority. + + // We need to iterate over all the donors to do this. Unfortunatly, + // our donors are organized in a heap. We'll visit each node with a + // recurisve call. This is realitively safe since there are only sem->m + // donors, at most. We won't recurse too deeply to have to worry about + // our stack. (even with 128 CPUs, our nest depth is at most 7 deep). + + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + ikglp_wait_state_t *donor = NULL; + int distance = MIG_NONE; + int gpu = replica_to_gpu(aff, ikglp_get_idx(sem, fq)); + ikglp_wait_state_t* default_donor = binheap_top_entry(&sem->donors, ikglp_wait_state_t, node); + + __find_closest_donor(gpu, sem->donors.root, &donor, &distance); + + TRACE_CUR("Selected donor %s/%d (distance = %d) to move to fq %d " + "(non-aff wanted %s/%d). differs = %d\n", + donor->task->comm, donor->task->pid, + distance, + ikglp_get_idx(sem, fq), + default_donor->task->comm, default_donor->task->pid, + (donor->task != default_donor->task) + ); + + return(donor); +} + + + +void gpu_ikglp_notify_enqueue(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + int replica = ikglp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + struct ikglp_queue_info *info = &aff->q_info[replica]; + lt_t est_time; + lt_t est_len_before; + + if(current == t) { + tsk_rt(t)->suspend_gpu_tracker_on_block = 1; + } + + est_len_before = info->estimated_len; + est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); + info->estimated_len += est_time; + + TRACE_CUR("fq %d: q_len (%llu) + est_cs (%llu) = %llu\n", + ikglp_get_idx(sem, info->q), + est_len_before, est_time, + info->estimated_len); + + // if(aff->shortest_queue == info) { + // // we may no longer be the shortest + // aff->shortest_queue = ikglp_aff_find_shortest(aff); + // + // TRACE_CUR("shortest queue is fq %d (with %d in queue) has est len %llu\n", + // ikglp_get_idx(sem, aff->shortest_queue->q), + // aff->shortest_queue->q->count, + // aff->shortest_queue->estimated_len); + // } +} + +void gpu_ikglp_notify_dequeue(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + int replica = ikglp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + struct ikglp_queue_info *info = &aff->q_info[replica]; + lt_t est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); + + if(est_time > info->estimated_len) { + WARN_ON(1); + info->estimated_len = 0; + } + else { + info->estimated_len -= est_time; + } + + TRACE_CUR("fq %d est len is now %llu\n", + ikglp_get_idx(sem, info->q), + info->estimated_len); + + // check to see if we're the shortest queue now. + // if((aff->shortest_queue != info) && + // (aff->shortest_queue->estimated_len > info->estimated_len)) { + // + // aff->shortest_queue = info; + // + // TRACE_CUR("shortest queue is fq %d (with %d in queue) has est len %llu\n", + // ikglp_get_idx(sem, info->q), + // info->q->count, + // info->estimated_len); + // } +} + +void gpu_ikglp_notify_acquired(struct ikglp_affinity* aff, + struct fifo_queue* fq, + struct task_struct* t) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + int replica = ikglp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + + tsk_rt(t)->gpu_migration = gpu_migration_distance(tsk_rt(t)->last_gpu, gpu); // record the type of migration + + TRACE_CUR("%s/%d acquired gpu %d (prev = %d). migration type = %d\n", + t->comm, t->pid, gpu, tsk_rt(t)->last_gpu, tsk_rt(t)->gpu_migration); + + // count the number or resource holders + ++(*(aff->q_info[replica].nr_cur_users)); + + reg_nv_device(gpu, 1, t); // register + + tsk_rt(t)->suspend_gpu_tracker_on_block = 0; + reset_gpu_tracker(t); + start_gpu_tracker(t); +} + +void gpu_ikglp_notify_freed(struct ikglp_affinity* aff, + struct fifo_queue* fq, + struct task_struct* t) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + int replica = ikglp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + lt_t est_time; + + stop_gpu_tracker(t); // stop the tracker before we do anything else. + + est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); + + // count the number or resource holders + --(*(aff->q_info[replica].nr_cur_users)); + + reg_nv_device(gpu, 0, t); // unregister + + // update estimates + update_gpu_estimate(t, get_gpu_time(t)); + + TRACE_CUR("%s/%d freed gpu %d (prev = %d). mig type = %d. actual time was %llu. " + "estimated was %llu. diff is %d\n", + t->comm, t->pid, gpu, tsk_rt(t)->last_gpu, + tsk_rt(t)->gpu_migration, + get_gpu_time(t), + est_time, + (long long)get_gpu_time(t) - (long long)est_time); + + tsk_rt(t)->last_gpu = gpu; +} + +struct ikglp_affinity_ops gpu_ikglp_affinity = +{ + .advise_enqueue = gpu_ikglp_advise_enqueue, + .advise_steal = gpu_ikglp_advise_steal, + .advise_donee_selection = gpu_ikglp_advise_donee_selection, + .advise_donor_to_fq = gpu_ikglp_advise_donor_to_fq, + + .notify_enqueue = gpu_ikglp_notify_enqueue, + .notify_dequeue = gpu_ikglp_notify_dequeue, + .notify_acquired = gpu_ikglp_notify_acquired, + .notify_freed = gpu_ikglp_notify_freed, + + .replica_to_resource = gpu_replica_to_resource, +}; + +struct affinity_observer* ikglp_gpu_aff_obs_new(struct affinity_observer_ops* ops, + void* __user args) +{ + return ikglp_aff_obs_new(ops, &gpu_ikglp_affinity, args); +} + + + + + + + + +// Simple ikglp Affinity (standard ikglp with auto-gpu registration) + +struct fifo_queue* simple_gpu_ikglp_advise_enqueue(struct ikglp_affinity* aff, struct task_struct* t) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + int min_count; + int min_nr_users; + struct ikglp_queue_info *shortest; + struct fifo_queue *to_enqueue; + int i; + + // TRACE_CUR("Simple GPU ikglp advise_enqueue invoked\n"); + + shortest = &aff->q_info[0]; + min_count = shortest->q->count; + min_nr_users = *(shortest->nr_cur_users); + + TRACE_CUR("queue %d: waiters = %d, total holders = %d\n", + ikglp_get_idx(sem, shortest->q), + shortest->q->count, + min_nr_users); + + for(i = 1; i < sem->nr_replicas; ++i) { + int len = aff->q_info[i].q->count; + + // queue is smaller, or they're equal and the other has a smaller number + // of total users. + // + // tie-break on the shortest number of simult users. this only kicks in + // when there are more than 1 empty queues. + if((len < min_count) || + ((len == min_count) && (*(aff->q_info[i].nr_cur_users) < min_nr_users))) { + shortest = &aff->q_info[i]; + min_count = shortest->q->count; + min_nr_users = *(aff->q_info[i].nr_cur_users); + } + + TRACE_CUR("queue %d: waiters = %d, total holders = %d\n", + ikglp_get_idx(sem, aff->q_info[i].q), + aff->q_info[i].q->count, + *(aff->q_info[i].nr_cur_users)); + } + + to_enqueue = shortest->q; + TRACE_CUR("enqueue on fq %d (non-aff wanted fq %d)\n", + ikglp_get_idx(sem, to_enqueue), + ikglp_get_idx(sem, sem->shortest_fifo_queue)); + + return to_enqueue; +} + +ikglp_wait_state_t* simple_gpu_ikglp_advise_steal(struct ikglp_affinity* aff, + struct fifo_queue* dst) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + // TRACE_CUR("Simple GPU ikglp advise_steal invoked\n"); + return ikglp_find_hp_waiter_to_steal(sem); +} + +ikglp_donee_heap_node_t* simple_gpu_ikglp_advise_donee_selection(struct ikglp_affinity* aff, struct task_struct* donor) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + ikglp_donee_heap_node_t *donee = binheap_top_entry(&sem->donees, ikglp_donee_heap_node_t, node); + return(donee); +} + +ikglp_wait_state_t* simple_gpu_ikglp_advise_donor_to_fq(struct ikglp_affinity* aff, struct fifo_queue* fq) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + ikglp_wait_state_t* donor = binheap_top_entry(&sem->donors, ikglp_wait_state_t, node); + return(donor); +} + +void simple_gpu_ikglp_notify_enqueue(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t) +{ + // TRACE_CUR("Simple GPU ikglp notify_enqueue invoked\n"); +} + +void simple_gpu_ikglp_notify_dequeue(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t) +{ + // TRACE_CUR("Simple GPU ikglp notify_dequeue invoked\n"); +} + +void simple_gpu_ikglp_notify_acquired(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + int replica = ikglp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + + // TRACE_CUR("Simple GPU ikglp notify_acquired invoked\n"); + + // count the number or resource holders + ++(*(aff->q_info[replica].nr_cur_users)); + + reg_nv_device(gpu, 1, t); // register +} + +void simple_gpu_ikglp_notify_freed(struct ikglp_affinity* aff, struct fifo_queue* fq, struct task_struct* t) +{ + struct ikglp_semaphore *sem = ikglp_from_lock(aff->obs.lock); + int replica = ikglp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + + // TRACE_CUR("Simple GPU ikglp notify_freed invoked\n"); + // count the number or resource holders + --(*(aff->q_info[replica].nr_cur_users)); + + reg_nv_device(gpu, 0, t); // unregister +} + +struct ikglp_affinity_ops simple_gpu_ikglp_affinity = +{ + .advise_enqueue = simple_gpu_ikglp_advise_enqueue, + .advise_steal = simple_gpu_ikglp_advise_steal, + .advise_donee_selection = simple_gpu_ikglp_advise_donee_selection, + .advise_donor_to_fq = simple_gpu_ikglp_advise_donor_to_fq, + + .notify_enqueue = simple_gpu_ikglp_notify_enqueue, + .notify_dequeue = simple_gpu_ikglp_notify_dequeue, + .notify_acquired = simple_gpu_ikglp_notify_acquired, + .notify_freed = simple_gpu_ikglp_notify_freed, + + .replica_to_resource = gpu_replica_to_resource, +}; + +struct affinity_observer* ikglp_simple_gpu_aff_obs_new(struct affinity_observer_ops* ops, + void* __user args) +{ + return ikglp_aff_obs_new(ops, &simple_gpu_ikglp_affinity, args); +} + +#endif + + + + + + + + + diff --git a/litmus/jobs.c b/litmus/jobs.c new file mode 100644 index 0000000..1d97462 --- /dev/null +++ b/litmus/jobs.c @@ -0,0 +1,56 @@ +/* litmus/jobs.c - common job control code + */ + +#include + +#include +#include + +void prepare_for_next_period(struct task_struct *t) +{ + BUG_ON(!t); + /* prepare next release */ + + if(tsk_rt(t)->task_params.cls == RT_CLASS_SOFT_W_SLIP) { + /* allow the release point to slip if we've passed our deadline. */ + lt_t now = litmus_clock(); + t->rt_param.job_params.release = + (t->rt_param.job_params.deadline < now) ? + now : t->rt_param.job_params.deadline; + t->rt_param.job_params.deadline = + t->rt_param.job_params.release + get_rt_period(t); + } + else { + t->rt_param.job_params.release = t->rt_param.job_params.deadline; + t->rt_param.job_params.deadline += get_rt_period(t); + } + + t->rt_param.job_params.exec_time = 0; + /* update job sequence number */ + t->rt_param.job_params.job_no++; + + /* don't confuse Linux */ + t->rt.time_slice = 1; +} + +void release_at(struct task_struct *t, lt_t start) +{ + t->rt_param.job_params.deadline = start; + prepare_for_next_period(t); + set_rt_flags(t, RT_F_RUNNING); +} + + +/* + * Deactivate current task until the beginning of the next period. + */ +long complete_job(void) +{ + /* Mark that we do not excute anymore */ + set_rt_flags(current, RT_F_SLEEP); + /* call schedule, this will return when a new job arrives + * it also takes care of preparing for the next release + */ + schedule(); + return 0; +} diff --git a/litmus/kexclu_affinity.c b/litmus/kexclu_affinity.c new file mode 100644 index 0000000..5ef5e54 --- /dev/null +++ b/litmus/kexclu_affinity.c @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include + +#include + +static int create_generic_aff_obs(void** obj_ref, obj_type_t type, void* __user arg); +static int open_generic_aff_obs(struct od_table_entry* entry, void* __user arg); +static int close_generic_aff_obs(struct od_table_entry* entry); +static void destroy_generic_aff_obs(obj_type_t type, void* sem); + +struct fdso_ops generic_affinity_ops = { + .create = create_generic_aff_obs, + .open = open_generic_aff_obs, + .close = close_generic_aff_obs, + .destroy = destroy_generic_aff_obs +}; + +static atomic_t aff_obs_id_gen = ATOMIC_INIT(0); + +static inline bool is_affinity_observer(struct od_table_entry *entry) +{ + return (entry->class == &generic_affinity_ops); +} + +static inline struct affinity_observer* get_affinity_observer(struct od_table_entry* entry) +{ + BUG_ON(!is_affinity_observer(entry)); + return (struct affinity_observer*) entry->obj->obj; +} + +static int create_generic_aff_obs(void** obj_ref, obj_type_t type, void* __user arg) +{ + struct affinity_observer* aff_obs; + int err; + + err = litmus->allocate_aff_obs(&aff_obs, type, arg); + if (err == 0) { + BUG_ON(!aff_obs->lock); + aff_obs->type = type; + *obj_ref = aff_obs; + } + return err; +} + +static int open_generic_aff_obs(struct od_table_entry* entry, void* __user arg) +{ + struct affinity_observer* aff_obs = get_affinity_observer(entry); + if (aff_obs->ops->open) + return aff_obs->ops->open(aff_obs, arg); + else + return 0; /* default: any task can open it */ +} + +static int close_generic_aff_obs(struct od_table_entry* entry) +{ + struct affinity_observer* aff_obs = get_affinity_observer(entry); + if (aff_obs->ops->close) + return aff_obs->ops->close(aff_obs); + else + return 0; /* default: closing succeeds */ +} + +static void destroy_generic_aff_obs(obj_type_t type, void* obj) +{ + struct affinity_observer* aff_obs = (struct affinity_observer*) obj; + aff_obs->ops->deallocate(aff_obs); +} + + +struct litmus_lock* get_lock_from_od(int od) +{ + extern struct fdso_ops generic_lock_ops; + + struct od_table_entry *entry = get_entry_for_od(od); + + if(entry && entry->class == &generic_lock_ops) { + return (struct litmus_lock*) entry->obj->obj; + } + return NULL; +} + +void affinity_observer_new(struct affinity_observer* aff, + struct affinity_observer_ops* ops, + struct affinity_observer_args* args) +{ + aff->ops = ops; + aff->lock = get_lock_from_od(args->lock_od); + aff->ident = atomic_inc_return(&aff_obs_id_gen); +} \ No newline at end of file diff --git a/litmus/kfmlp_lock.c b/litmus/kfmlp_lock.c new file mode 100644 index 0000000..bff857e --- /dev/null +++ b/litmus/kfmlp_lock.c @@ -0,0 +1,1002 @@ +#include +#include + +#include +#include +#include + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) +#include +#include +#endif + +#include + +static inline int kfmlp_get_idx(struct kfmlp_semaphore* sem, + struct kfmlp_queue* queue) +{ + return (queue - &sem->queues[0]); +} + +static inline struct kfmlp_queue* kfmlp_get_queue(struct kfmlp_semaphore* sem, + struct task_struct* holder) +{ + int i; + for(i = 0; i < sem->num_resources; ++i) + if(sem->queues[i].owner == holder) + return(&sem->queues[i]); + return(NULL); +} + +/* caller is responsible for locking */ +static struct task_struct* kfmlp_find_hp_waiter(struct kfmlp_queue *kqueue, + struct task_struct *skip) +{ + struct list_head *pos; + struct task_struct *queued, *found = NULL; + + list_for_each(pos, &kqueue->wait.task_list) { + queued = (struct task_struct*) list_entry(pos, wait_queue_t, + task_list)->private; + + /* Compare task prios, find high prio task. */ + //if (queued != skip && edf_higher_prio(queued, found)) + if (queued != skip && litmus->compare(queued, found)) + found = queued; + } + return found; +} + +static inline struct kfmlp_queue* kfmlp_find_shortest(struct kfmlp_semaphore* sem, + struct kfmlp_queue* search_start) +{ + // we start our search at search_start instead of at the beginning of the + // queue list to load-balance across all resources. + struct kfmlp_queue* step = search_start; + struct kfmlp_queue* shortest = sem->shortest_queue; + + do + { + step = (step+1 != &sem->queues[sem->num_resources]) ? + step+1 : &sem->queues[0]; + + if(step->count < shortest->count) + { + shortest = step; + if(step->count == 0) + break; /* can't get any shorter */ + } + + }while(step != search_start); + + return(shortest); +} + + +static struct task_struct* kfmlp_select_hp_steal(struct kfmlp_semaphore* sem, + wait_queue_t** to_steal, + struct kfmlp_queue** to_steal_from) +{ + /* must hold sem->lock */ + + int i; + + *to_steal = NULL; + *to_steal_from = NULL; + + for(i = 0; i < sem->num_resources; ++i) + { + if( (sem->queues[i].count > 1) && + ((*to_steal_from == NULL) || + //(edf_higher_prio(sem->queues[i].hp_waiter, my_queue->hp_waiter))) ) + (litmus->compare(sem->queues[i].hp_waiter, (*to_steal_from)->hp_waiter))) ) + { + *to_steal_from = &sem->queues[i]; + } + } + + if(*to_steal_from) + { + struct list_head *pos; + struct task_struct *target = (*to_steal_from)->hp_waiter; + + TRACE_CUR("want to steal hp_waiter (%s/%d) from queue %d\n", + target->comm, + target->pid, + kfmlp_get_idx(sem, *to_steal_from)); + + list_for_each(pos, &(*to_steal_from)->wait.task_list) + { + wait_queue_t *node = list_entry(pos, wait_queue_t, task_list); + struct task_struct *queued = (struct task_struct*) node->private; + /* Compare task prios, find high prio task. */ + if (queued == target) + { + *to_steal = node; + + TRACE_CUR("steal: selected %s/%d from queue %d\n", + queued->comm, queued->pid, + kfmlp_get_idx(sem, *to_steal_from)); + + return queued; + } + } + + TRACE_CUR("Could not find %s/%d in queue %d!!! THIS IS A BUG!\n", + target->comm, + target->pid, + kfmlp_get_idx(sem, *to_steal_from)); + } + + return NULL; +} + +static void kfmlp_steal_node(struct kfmlp_semaphore *sem, + struct kfmlp_queue *dst, + wait_queue_t *wait, + struct kfmlp_queue *src) +{ + struct task_struct* t = (struct task_struct*) wait->private; + + __remove_wait_queue(&src->wait, wait); + --(src->count); + + if(t == src->hp_waiter) { + src->hp_waiter = kfmlp_find_hp_waiter(src, NULL); + + TRACE_CUR("queue %d: %s/%d is new hp_waiter\n", + kfmlp_get_idx(sem, src), + (src->hp_waiter) ? src->hp_waiter->comm : "nil", + (src->hp_waiter) ? src->hp_waiter->pid : -1); + + if(src->owner && tsk_rt(src->owner)->inh_task == t) { + litmus->decrease_prio(src->owner, src->hp_waiter); + } + } + + if(sem->shortest_queue->count > src->count) { + sem->shortest_queue = src; + TRACE_CUR("queue %d is the shortest\n", kfmlp_get_idx(sem, sem->shortest_queue)); + } + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_dequeue(sem->aff_obs, src, t); + } +#endif + + init_waitqueue_entry(wait, t); + __add_wait_queue_tail_exclusive(&dst->wait, wait); + ++(dst->count); + + if(litmus->compare(t, dst->hp_waiter)) { + dst->hp_waiter = t; + + TRACE_CUR("queue %d: %s/%d is new hp_waiter\n", + kfmlp_get_idx(sem, dst), + t->comm, t->pid); + + if(dst->owner && litmus->compare(t, dst->owner)) + { + litmus->increase_prio(dst->owner, t); + } + } + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_enqueue(sem->aff_obs, dst, t); + } +#endif +} + + +int kfmlp_lock(struct litmus_lock* l) +{ + struct task_struct* t = current; + struct kfmlp_semaphore *sem = kfmlp_from_lock(l); + struct kfmlp_queue* my_queue = NULL; + wait_queue_t wait; + unsigned long flags; + + if (!is_realtime(t)) + return -EPERM; + + spin_lock_irqsave(&sem->lock, flags); + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + my_queue = sem->aff_obs->ops->advise_enqueue(sem->aff_obs, t); + } + if(!my_queue) { + my_queue = sem->shortest_queue; + } +#else + my_queue = sem->shortest_queue; +#endif + + if (my_queue->owner) { + /* resource is not free => must suspend and wait */ + TRACE_CUR("queue %d: Resource is not free => must suspend and wait. (queue size = %d)\n", + kfmlp_get_idx(sem, my_queue), + my_queue->count); + + init_waitqueue_entry(&wait, t); + + /* FIXME: interruptible would be nice some day */ + set_task_state(t, TASK_UNINTERRUPTIBLE); + + __add_wait_queue_tail_exclusive(&my_queue->wait, &wait); + + TRACE_CUR("queue %d: hp_waiter is currently %s/%d\n", + kfmlp_get_idx(sem, my_queue), + (my_queue->hp_waiter) ? my_queue->hp_waiter->comm : "nil", + (my_queue->hp_waiter) ? my_queue->hp_waiter->pid : -1); + + /* check if we need to activate priority inheritance */ + //if (edf_higher_prio(t, my_queue->hp_waiter)) + if (litmus->compare(t, my_queue->hp_waiter)) { + my_queue->hp_waiter = t; + TRACE_CUR("queue %d: %s/%d is new hp_waiter\n", + kfmlp_get_idx(sem, my_queue), + t->comm, t->pid); + + //if (edf_higher_prio(t, my_queue->owner)) + if (litmus->compare(t, my_queue->owner)) { + litmus->increase_prio(my_queue->owner, my_queue->hp_waiter); + } + } + + ++(my_queue->count); + + if(my_queue == sem->shortest_queue) { + sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); + TRACE_CUR("queue %d is the shortest\n", + kfmlp_get_idx(sem, sem->shortest_queue)); + } + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_enqueue(sem->aff_obs, my_queue, t); + } +#endif + + /* release lock before sleeping */ + spin_unlock_irqrestore(&sem->lock, flags); + + /* We depend on the FIFO order. Thus, we don't need to recheck + * when we wake up; we are guaranteed to have the lock since + * there is only one wake up per release (or steal). + */ + schedule(); + + + if(my_queue->owner == t) { + TRACE_CUR("queue %d: acquired through waiting\n", + kfmlp_get_idx(sem, my_queue)); + } + else { + /* this case may happen if our wait entry was stolen + between queues. record where we went. */ + my_queue = kfmlp_get_queue(sem, t); + + BUG_ON(!my_queue); + TRACE_CUR("queue %d: acquired through stealing\n", + kfmlp_get_idx(sem, my_queue)); + } + } + else { + TRACE_CUR("queue %d: acquired immediately\n", + kfmlp_get_idx(sem, my_queue)); + + my_queue->owner = t; + + ++(my_queue->count); + + if(my_queue == sem->shortest_queue) { + sem->shortest_queue = kfmlp_find_shortest(sem, my_queue); + TRACE_CUR("queue %d is the shortest\n", + kfmlp_get_idx(sem, sem->shortest_queue)); + } + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_enqueue(sem->aff_obs, my_queue, t); + sem->aff_obs->ops->notify_acquired(sem->aff_obs, my_queue, t); + } +#endif + + spin_unlock_irqrestore(&sem->lock, flags); + } + + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + return sem->aff_obs->ops->replica_to_resource(sem->aff_obs, my_queue); + } +#endif + return kfmlp_get_idx(sem, my_queue); +} + + +int kfmlp_unlock(struct litmus_lock* l) +{ + struct task_struct *t = current, *next; + struct kfmlp_semaphore *sem = kfmlp_from_lock(l); + struct kfmlp_queue *my_queue, *to_steal_from; + unsigned long flags; + int err = 0; + + my_queue = kfmlp_get_queue(sem, t); + + if (!my_queue) { + err = -EINVAL; + goto out; + } + + spin_lock_irqsave(&sem->lock, flags); + + TRACE_CUR("queue %d: unlocking\n", kfmlp_get_idx(sem, my_queue)); + + my_queue->owner = NULL; // clear ownership + --(my_queue->count); + + if(my_queue->count < sem->shortest_queue->count) + { + sem->shortest_queue = my_queue; + TRACE_CUR("queue %d is the shortest\n", + kfmlp_get_idx(sem, sem->shortest_queue)); + } + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_dequeue(sem->aff_obs, my_queue, t); + sem->aff_obs->ops->notify_freed(sem->aff_obs, my_queue, t); + } +#endif + + /* we lose the benefit of priority inheritance (if any) */ + if (tsk_rt(t)->inh_task) + litmus->decrease_prio(t, NULL); + + + /* check if there are jobs waiting for this resource */ +RETRY: + next = __waitqueue_remove_first(&my_queue->wait); + if (next) { + /* next becomes the resouce holder */ + my_queue->owner = next; + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + if(sem->aff_obs) { + sem->aff_obs->ops->notify_acquired(sem->aff_obs, my_queue, next); + } +#endif + + TRACE_CUR("queue %d: lock ownership passed to %s/%d\n", + kfmlp_get_idx(sem, my_queue), next->comm, next->pid); + + /* determine new hp_waiter if necessary */ + if (next == my_queue->hp_waiter) { + TRACE_TASK(next, "was highest-prio waiter\n"); + my_queue->hp_waiter = kfmlp_find_hp_waiter(my_queue, next); + if (my_queue->hp_waiter) + TRACE_TASK(my_queue->hp_waiter, "queue %d: is new highest-prio waiter\n", kfmlp_get_idx(sem, my_queue)); + else + TRACE("queue %d: no further waiters\n", kfmlp_get_idx(sem, my_queue)); + } else { + /* Well, if next is not the highest-priority waiter, + * then it ought to inherit the highest-priority + * waiter's priority. */ + litmus->increase_prio(next, my_queue->hp_waiter); + } + + /* wake up next */ + wake_up_process(next); + } + else { + // TODO: put this stealing logic before we attempt to release + // our resource. (simplifies code and gets rid of ugly goto RETRY. + wait_queue_t *wait; + + TRACE_CUR("queue %d: looking to steal someone...\n", + kfmlp_get_idx(sem, my_queue)); + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + next = (sem->aff_obs) ? + sem->aff_obs->ops->advise_steal(sem->aff_obs, &wait, &to_steal_from) : + kfmlp_select_hp_steal(sem, &wait, &to_steal_from); +#else + next = kfmlp_select_hp_steal(sem, &wait, &to_steal_from); +#endif + + if(next) { + TRACE_CUR("queue %d: stealing %s/%d from queue %d\n", + kfmlp_get_idx(sem, my_queue), + next->comm, next->pid, + kfmlp_get_idx(sem, to_steal_from)); + + kfmlp_steal_node(sem, my_queue, wait, to_steal_from); + + goto RETRY; // will succeed this time. + } + else { + TRACE_CUR("queue %d: no one to steal.\n", + kfmlp_get_idx(sem, my_queue)); + } + } + + spin_unlock_irqrestore(&sem->lock, flags); + +out: + return err; +} + +int kfmlp_close(struct litmus_lock* l) +{ + struct task_struct *t = current; + struct kfmlp_semaphore *sem = kfmlp_from_lock(l); + struct kfmlp_queue *my_queue; + unsigned long flags; + + int owner; + + spin_lock_irqsave(&sem->lock, flags); + + my_queue = kfmlp_get_queue(sem, t); + owner = (my_queue) ? (my_queue->owner == t) : 0; + + spin_unlock_irqrestore(&sem->lock, flags); + + if (owner) + kfmlp_unlock(l); + + return 0; +} + +void kfmlp_free(struct litmus_lock* l) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(l); + kfree(sem->queues); + kfree(sem); +} + + + +struct litmus_lock* kfmlp_new(struct litmus_lock_ops* ops, void* __user args) +{ + struct kfmlp_semaphore* sem; + int num_resources = 0; + int i; + + if(!access_ok(VERIFY_READ, args, sizeof(num_resources))) + { + return(NULL); + } + if(__copy_from_user(&num_resources, args, sizeof(num_resources))) + { + return(NULL); + } + if(num_resources < 1) + { + return(NULL); + } + + sem = kmalloc(sizeof(*sem), GFP_KERNEL); + if(!sem) + { + return(NULL); + } + + sem->queues = kmalloc(sizeof(struct kfmlp_queue)*num_resources, GFP_KERNEL); + if(!sem->queues) + { + kfree(sem); + return(NULL); + } + + sem->litmus_lock.ops = ops; + spin_lock_init(&sem->lock); + sem->num_resources = num_resources; + + for(i = 0; i < num_resources; ++i) + { + sem->queues[i].owner = NULL; + sem->queues[i].hp_waiter = NULL; + init_waitqueue_head(&sem->queues[i].wait); + sem->queues[i].count = 0; + } + + sem->shortest_queue = &sem->queues[0]; + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + sem->aff_obs = NULL; +#endif + + return &sem->litmus_lock; +} + + + + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) + +static inline int __replica_to_gpu(struct kfmlp_affinity* aff, int replica) +{ + int gpu = replica % aff->nr_rsrc; + return gpu; +} + +static inline int replica_to_gpu(struct kfmlp_affinity* aff, int replica) +{ + int gpu = __replica_to_gpu(aff, replica) + aff->offset; + return gpu; +} + +static inline int gpu_to_base_replica(struct kfmlp_affinity* aff, int gpu) +{ + int replica = gpu - aff->offset; + return replica; +} + + +int kfmlp_aff_obs_close(struct affinity_observer* obs) +{ + return 0; +} + +void kfmlp_aff_obs_free(struct affinity_observer* obs) +{ + struct kfmlp_affinity *kfmlp_aff = kfmlp_aff_obs_from_aff_obs(obs); + kfree(kfmlp_aff->nr_cur_users_on_rsrc); + kfree(kfmlp_aff->q_info); + kfree(kfmlp_aff); +} + +static struct affinity_observer* kfmlp_aff_obs_new(struct affinity_observer_ops* ops, + struct kfmlp_affinity_ops* kfmlp_ops, + void* __user args) +{ + struct kfmlp_affinity* kfmlp_aff; + struct gpu_affinity_observer_args aff_args; + struct kfmlp_semaphore* sem; + int i; + unsigned long flags; + + if(!access_ok(VERIFY_READ, args, sizeof(aff_args))) { + return(NULL); + } + if(__copy_from_user(&aff_args, args, sizeof(aff_args))) { + return(NULL); + } + + sem = (struct kfmlp_semaphore*) get_lock_from_od(aff_args.obs.lock_od); + + if(sem->litmus_lock.type != KFMLP_SEM) { + TRACE_CUR("Lock type not supported. Type = %d\n", sem->litmus_lock.type); + return(NULL); + } + + if((aff_args.nr_simult_users <= 0) || + (sem->num_resources%aff_args.nr_simult_users != 0)) { + TRACE_CUR("Lock %d does not support #replicas (%d) for #simult_users " + "(%d) per replica. #replicas should be evenly divisible " + "by #simult_users.\n", + sem->litmus_lock.ident, + sem->num_resources, + aff_args.nr_simult_users); + return(NULL); + } + + if(aff_args.nr_simult_users > NV_MAX_SIMULT_USERS) { + TRACE_CUR("System does not support #simult_users > %d. %d requested.\n", + NV_MAX_SIMULT_USERS, aff_args.nr_simult_users); +// return(NULL); + } + + kfmlp_aff = kmalloc(sizeof(*kfmlp_aff), GFP_KERNEL); + if(!kfmlp_aff) { + return(NULL); + } + + kfmlp_aff->q_info = kmalloc(sizeof(struct kfmlp_queue_info)*sem->num_resources, GFP_KERNEL); + if(!kfmlp_aff->q_info) { + kfree(kfmlp_aff); + return(NULL); + } + + kfmlp_aff->nr_cur_users_on_rsrc = kmalloc(sizeof(int)*(sem->num_resources / aff_args.nr_simult_users), GFP_KERNEL); + if(!kfmlp_aff->nr_cur_users_on_rsrc) { + kfree(kfmlp_aff->q_info); + kfree(kfmlp_aff); + return(NULL); + } + + affinity_observer_new(&kfmlp_aff->obs, ops, &aff_args.obs); + + kfmlp_aff->ops = kfmlp_ops; + kfmlp_aff->offset = aff_args.replica_to_gpu_offset; + kfmlp_aff->nr_simult = aff_args.nr_simult_users; + kfmlp_aff->nr_rsrc = sem->num_resources / kfmlp_aff->nr_simult; + + memset(kfmlp_aff->nr_cur_users_on_rsrc, 0, sizeof(int)*(sem->num_resources / kfmlp_aff->nr_rsrc)); + + for(i = 0; i < sem->num_resources; ++i) { + kfmlp_aff->q_info[i].q = &sem->queues[i]; + kfmlp_aff->q_info[i].estimated_len = 0; + + // multiple q_info's will point to the same resource (aka GPU) if + // aff_args.nr_simult_users > 1 + kfmlp_aff->q_info[i].nr_cur_users = &kfmlp_aff->nr_cur_users_on_rsrc[__replica_to_gpu(kfmlp_aff,i)]; + } + + // attach observer to the lock + spin_lock_irqsave(&sem->lock, flags); + sem->aff_obs = kfmlp_aff; + spin_unlock_irqrestore(&sem->lock, flags); + + return &kfmlp_aff->obs; +} + + + + +static int gpu_replica_to_resource(struct kfmlp_affinity* aff, + struct kfmlp_queue* fq) { + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + return(replica_to_gpu(aff, kfmlp_get_idx(sem, fq))); +} + + +// Smart KFMLP Affinity + +//static inline struct kfmlp_queue_info* kfmlp_aff_find_shortest(struct kfmlp_affinity* aff) +//{ +// struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); +// struct kfmlp_queue_info *shortest = &aff->q_info[0]; +// int i; +// +// for(i = 1; i < sem->num_resources; ++i) { +// if(aff->q_info[i].estimated_len < shortest->estimated_len) { +// shortest = &aff->q_info[i]; +// } +// } +// +// return(shortest); +//} + +struct kfmlp_queue* gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, struct task_struct* t) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + lt_t min_len; + int min_nr_users; + struct kfmlp_queue_info *shortest; + struct kfmlp_queue *to_enqueue; + int i; + int affinity_gpu; + + // simply pick the shortest queue if, we have no affinity, or we have + // affinity with the shortest + if(unlikely(tsk_rt(t)->last_gpu < 0)) { + affinity_gpu = aff->offset; // first gpu + TRACE_CUR("no affinity\n"); + } + else { + affinity_gpu = tsk_rt(t)->last_gpu; + } + + // all things being equal, let's start with the queue with which we have + // affinity. this helps us maintain affinity even when we don't have + // an estiamte for local-affinity execution time (i.e., 2nd time on GPU) + shortest = &aff->q_info[gpu_to_base_replica(aff, affinity_gpu)]; + +// if(shortest == aff->shortest_queue) { +// TRACE_CUR("special case: have affinity with shortest queue\n"); +// goto out; +// } + + min_len = shortest->estimated_len + get_gpu_estimate(t, MIG_LOCAL); + min_nr_users = *(shortest->nr_cur_users); + + TRACE_CUR("cs is %llu on queue %d: est len = %llu\n", + get_gpu_estimate(t, MIG_LOCAL), + kfmlp_get_idx(sem, shortest->q), + min_len); + + for(i = 0; i < sem->num_resources; ++i) { + if(&aff->q_info[i] != shortest) { + + lt_t est_len = + aff->q_info[i].estimated_len + + get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, replica_to_gpu(aff, i))); + + // queue is smaller, or they're equal and the other has a smaller number + // of total users. + // + // tie-break on the shortest number of simult users. this only kicks in + // when there are more than 1 empty queues. + if((est_len < min_len) || + ((est_len == min_len) && (*(aff->q_info[i].nr_cur_users) < min_nr_users))) { + shortest = &aff->q_info[i]; + min_len = est_len; + min_nr_users = *(aff->q_info[i].nr_cur_users); + } + + TRACE_CUR("cs is %llu on queue %d: est len = %llu\n", + get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, replica_to_gpu(aff, i))), + kfmlp_get_idx(sem, aff->q_info[i].q), + est_len); + } + } + + to_enqueue = shortest->q; + TRACE_CUR("enqueue on fq %d (non-aff wanted fq %d)\n", + kfmlp_get_idx(sem, to_enqueue), + kfmlp_get_idx(sem, sem->shortest_queue)); + + return to_enqueue; +} + +struct task_struct* gpu_kfmlp_advise_steal(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + + // For now, just steal highest priority waiter + // TODO: Implement affinity-aware stealing. + + return kfmlp_select_hp_steal(sem, to_steal, to_steal_from); +} + + +void gpu_kfmlp_notify_enqueue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + int replica = kfmlp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + struct kfmlp_queue_info *info = &aff->q_info[replica]; + lt_t est_time; + lt_t est_len_before; + + if(current == t) { + tsk_rt(t)->suspend_gpu_tracker_on_block = 1; + } + + est_len_before = info->estimated_len; + est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); + info->estimated_len += est_time; + + TRACE_CUR("fq %d: q_len (%llu) + est_cs (%llu) = %llu\n", + kfmlp_get_idx(sem, info->q), + est_len_before, est_time, + info->estimated_len); + +// if(aff->shortest_queue == info) { +// // we may no longer be the shortest +// aff->shortest_queue = kfmlp_aff_find_shortest(aff); +// +// TRACE_CUR("shortest queue is fq %d (with %d in queue) has est len %llu\n", +// kfmlp_get_idx(sem, aff->shortest_queue->q), +// aff->shortest_queue->q->count, +// aff->shortest_queue->estimated_len); +// } +} + +void gpu_kfmlp_notify_dequeue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + int replica = kfmlp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + struct kfmlp_queue_info *info = &aff->q_info[replica]; + lt_t est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); + + if(est_time > info->estimated_len) { + WARN_ON(1); + info->estimated_len = 0; + } + else { + info->estimated_len -= est_time; + } + + TRACE_CUR("fq %d est len is now %llu\n", + kfmlp_get_idx(sem, info->q), + info->estimated_len); + + // check to see if we're the shortest queue now. +// if((aff->shortest_queue != info) && +// (aff->shortest_queue->estimated_len > info->estimated_len)) { +// +// aff->shortest_queue = info; +// +// TRACE_CUR("shortest queue is fq %d (with %d in queue) has est len %llu\n", +// kfmlp_get_idx(sem, info->q), +// info->q->count, +// info->estimated_len); +// } +} + +void gpu_kfmlp_notify_acquired(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + int replica = kfmlp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + + tsk_rt(t)->gpu_migration = gpu_migration_distance(tsk_rt(t)->last_gpu, gpu); // record the type of migration + + TRACE_CUR("%s/%d acquired gpu %d. migration type = %d\n", + t->comm, t->pid, gpu, tsk_rt(t)->gpu_migration); + + // count the number or resource holders + ++(*(aff->q_info[replica].nr_cur_users)); + + reg_nv_device(gpu, 1, t); // register + + tsk_rt(t)->suspend_gpu_tracker_on_block = 0; + reset_gpu_tracker(t); + start_gpu_tracker(t); +} + +void gpu_kfmlp_notify_freed(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + int replica = kfmlp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + lt_t est_time; + + stop_gpu_tracker(t); // stop the tracker before we do anything else. + + est_time = get_gpu_estimate(t, gpu_migration_distance(tsk_rt(t)->last_gpu, gpu)); + + tsk_rt(t)->last_gpu = gpu; + + // count the number or resource holders + --(*(aff->q_info[replica].nr_cur_users)); + + reg_nv_device(gpu, 0, t); // unregister + + // update estimates + update_gpu_estimate(t, get_gpu_time(t)); + + TRACE_CUR("%s/%d freed gpu %d. actual time was %llu. estimated was %llu. diff is %d\n", + t->comm, t->pid, gpu, + get_gpu_time(t), + est_time, + (long long)get_gpu_time(t) - (long long)est_time); +} + +struct kfmlp_affinity_ops gpu_kfmlp_affinity = +{ + .advise_enqueue = gpu_kfmlp_advise_enqueue, + .advise_steal = gpu_kfmlp_advise_steal, + .notify_enqueue = gpu_kfmlp_notify_enqueue, + .notify_dequeue = gpu_kfmlp_notify_dequeue, + .notify_acquired = gpu_kfmlp_notify_acquired, + .notify_freed = gpu_kfmlp_notify_freed, + .replica_to_resource = gpu_replica_to_resource, +}; + +struct affinity_observer* kfmlp_gpu_aff_obs_new(struct affinity_observer_ops* ops, + void* __user args) +{ + return kfmlp_aff_obs_new(ops, &gpu_kfmlp_affinity, args); +} + + + + + + + + +// Simple KFMLP Affinity (standard KFMLP with auto-gpu registration) + +struct kfmlp_queue* simple_gpu_kfmlp_advise_enqueue(struct kfmlp_affinity* aff, struct task_struct* t) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + int min_count; + int min_nr_users; + struct kfmlp_queue_info *shortest; + struct kfmlp_queue *to_enqueue; + int i; + +// TRACE_CUR("Simple GPU KFMLP advise_enqueue invoked\n"); + + shortest = &aff->q_info[0]; + min_count = shortest->q->count; + min_nr_users = *(shortest->nr_cur_users); + + TRACE_CUR("queue %d: waiters = %d, total holders = %d\n", + kfmlp_get_idx(sem, shortest->q), + shortest->q->count, + min_nr_users); + + for(i = 1; i < sem->num_resources; ++i) { + int len = aff->q_info[i].q->count; + + // queue is smaller, or they're equal and the other has a smaller number + // of total users. + // + // tie-break on the shortest number of simult users. this only kicks in + // when there are more than 1 empty queues. + if((len < min_count) || + ((len == min_count) && (*(aff->q_info[i].nr_cur_users) < min_nr_users))) { + shortest = &aff->q_info[i]; + min_count = shortest->q->count; + min_nr_users = *(aff->q_info[i].nr_cur_users); + } + + TRACE_CUR("queue %d: waiters = %d, total holders = %d\n", + kfmlp_get_idx(sem, aff->q_info[i].q), + aff->q_info[i].q->count, + *(aff->q_info[i].nr_cur_users)); + } + + to_enqueue = shortest->q; + TRACE_CUR("enqueue on fq %d (non-aff wanted fq %d)\n", + kfmlp_get_idx(sem, to_enqueue), + kfmlp_get_idx(sem, sem->shortest_queue)); + + return to_enqueue; +} + +struct task_struct* simple_gpu_kfmlp_advise_steal(struct kfmlp_affinity* aff, wait_queue_t** to_steal, struct kfmlp_queue** to_steal_from) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); +// TRACE_CUR("Simple GPU KFMLP advise_steal invoked\n"); + return kfmlp_select_hp_steal(sem, to_steal, to_steal_from); +} + +void simple_gpu_kfmlp_notify_enqueue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t) +{ +// TRACE_CUR("Simple GPU KFMLP notify_enqueue invoked\n"); +} + +void simple_gpu_kfmlp_notify_dequeue(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t) +{ +// TRACE_CUR("Simple GPU KFMLP notify_dequeue invoked\n"); +} + +void simple_gpu_kfmlp_notify_acquired(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + int replica = kfmlp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + +// TRACE_CUR("Simple GPU KFMLP notify_acquired invoked\n"); + + // count the number or resource holders + ++(*(aff->q_info[replica].nr_cur_users)); + + reg_nv_device(gpu, 1, t); // register +} + +void simple_gpu_kfmlp_notify_freed(struct kfmlp_affinity* aff, struct kfmlp_queue* fq, struct task_struct* t) +{ + struct kfmlp_semaphore *sem = kfmlp_from_lock(aff->obs.lock); + int replica = kfmlp_get_idx(sem, fq); + int gpu = replica_to_gpu(aff, replica); + +// TRACE_CUR("Simple GPU KFMLP notify_freed invoked\n"); + // count the number or resource holders + --(*(aff->q_info[replica].nr_cur_users)); + + reg_nv_device(gpu, 0, t); // unregister +} + +struct kfmlp_affinity_ops simple_gpu_kfmlp_affinity = +{ + .advise_enqueue = simple_gpu_kfmlp_advise_enqueue, + .advise_steal = simple_gpu_kfmlp_advise_steal, + .notify_enqueue = simple_gpu_kfmlp_notify_enqueue, + .notify_dequeue = simple_gpu_kfmlp_notify_dequeue, + .notify_acquired = simple_gpu_kfmlp_notify_acquired, + .notify_freed = simple_gpu_kfmlp_notify_freed, + .replica_to_resource = gpu_replica_to_resource, +}; + +struct affinity_observer* kfmlp_simple_gpu_aff_obs_new(struct affinity_observer_ops* ops, + void* __user args) +{ + return kfmlp_aff_obs_new(ops, &simple_gpu_kfmlp_affinity, args); +} + +#endif + diff --git a/litmus/litmus.c b/litmus/litmus.c new file mode 100644 index 0000000..d1f836c --- /dev/null +++ b/litmus/litmus.c @@ -0,0 +1,684 @@ +/* + * litmus.c -- Implementation of the LITMUS syscalls, + * the LITMUS intialization code, + * and the procfs interface.. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_SCHED_CPU_AFFINITY +#include +#endif + +#ifdef CONFIG_LITMUS_NVIDIA +#include +#endif + +/* Number of RT tasks that exist in the system */ +atomic_t rt_task_count = ATOMIC_INIT(0); +static DEFINE_RAW_SPINLOCK(task_transition_lock); +/* synchronize plugin switching */ +atomic_t cannot_use_plugin = ATOMIC_INIT(0); + +/* Give log messages sequential IDs. */ +atomic_t __log_seq_no = ATOMIC_INIT(0); + +#ifdef CONFIG_RELEASE_MASTER +/* current master CPU for handling timer IRQs */ +atomic_t release_master_cpu = ATOMIC_INIT(NO_CPU); +#endif + +static struct kmem_cache * bheap_node_cache; +extern struct kmem_cache * release_heap_cache; + +struct bheap_node* bheap_node_alloc(int gfp_flags) +{ + return kmem_cache_alloc(bheap_node_cache, gfp_flags); +} + +void bheap_node_free(struct bheap_node* hn) +{ + kmem_cache_free(bheap_node_cache, hn); +} + +struct release_heap* release_heap_alloc(int gfp_flags); +void release_heap_free(struct release_heap* rh); + +#ifdef CONFIG_LITMUS_NVIDIA +/* + * sys_register_nv_device + * @nv_device_id: The Nvidia device id that the task want to register + * @reg_action: set to '1' to register the specified device. zero otherwise. + * Syscall for register task's designated nvidia device into NV_DEVICE_REG array + * Returns EFAULT if nv_device_id is out of range. + * 0 if success + */ +asmlinkage long sys_register_nv_device(int nv_device_id, int reg_action) +{ + /* register the device to caller (aka 'current') */ + return(reg_nv_device(nv_device_id, reg_action, current)); +} +#else +asmlinkage long sys_register_nv_device(int nv_device_id, int reg_action) +{ + return(-EINVAL); +} +#endif + + +/* + * sys_set_task_rt_param + * @pid: Pid of the task which scheduling parameters must be changed + * @param: New real-time extension parameters such as the execution cost and + * period + * Syscall for manipulating with task rt extension params + * Returns EFAULT if param is NULL. + * ESRCH if pid is not corrsponding + * to a valid task. + * EINVAL if either period or execution cost is <=0 + * EPERM if pid is a real-time task + * 0 if success + * + * Only non-real-time tasks may be configured with this system call + * to avoid races with the scheduler. In practice, this means that a + * task's parameters must be set _before_ calling sys_prepare_rt_task() + * + * find_task_by_vpid() assumes that we are in the same namespace of the + * target. + */ +asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param) +{ + struct rt_task tp; + struct task_struct *target; + int retval = -EINVAL; + + printk("Setting up rt task parameters for process %d.\n", pid); + + if (pid < 0 || param == 0) { + goto out; + } + if (copy_from_user(&tp, param, sizeof(tp))) { + retval = -EFAULT; + goto out; + } + + /* Task search and manipulation must be protected */ + read_lock_irq(&tasklist_lock); + if (!(target = find_task_by_vpid(pid))) { + retval = -ESRCH; + goto out_unlock; + } + + if (is_realtime(target)) { + /* The task is already a real-time task. + * We cannot not allow parameter changes at this point. + */ + retval = -EBUSY; + goto out_unlock; + } + + if (tp.exec_cost <= 0) + goto out_unlock; + if (tp.period <= 0) + goto out_unlock; + if (!cpu_online(tp.cpu)) + goto out_unlock; + if (tp.period < tp.exec_cost) + { + printk(KERN_INFO "litmus: real-time task %d rejected " + "because wcet > period\n", pid); + goto out_unlock; + } + if ( tp.cls != RT_CLASS_HARD && + tp.cls != RT_CLASS_SOFT && + tp.cls != RT_CLASS_BEST_EFFORT) + { + printk(KERN_INFO "litmus: real-time task %d rejected " + "because its class is invalid\n", pid); + goto out_unlock; + } + if (tp.budget_policy != NO_ENFORCEMENT && + tp.budget_policy != QUANTUM_ENFORCEMENT && + tp.budget_policy != PRECISE_ENFORCEMENT) + { + printk(KERN_INFO "litmus: real-time task %d rejected " + "because unsupported budget enforcement policy " + "specified (%d)\n", + pid, tp.budget_policy); + goto out_unlock; + } + + target->rt_param.task_params = tp; + + retval = 0; + out_unlock: + read_unlock_irq(&tasklist_lock); + out: + return retval; +} + +/* + * Getter of task's RT params + * returns EINVAL if param or pid is NULL + * returns ESRCH if pid does not correspond to a valid task + * returns EFAULT if copying of parameters has failed. + * + * find_task_by_vpid() assumes that we are in the same namespace of the + * target. + */ +asmlinkage long sys_get_rt_task_param(pid_t pid, struct rt_task __user * param) +{ + int retval = -EINVAL; + struct task_struct *source; + struct rt_task lp; + if (param == 0 || pid < 0) + goto out; + read_lock(&tasklist_lock); + if (!(source = find_task_by_vpid(pid))) { + retval = -ESRCH; + goto out_unlock; + } + lp = source->rt_param.task_params; + read_unlock(&tasklist_lock); + /* Do copying outside the lock */ + retval = + copy_to_user(param, &lp, sizeof(lp)) ? -EFAULT : 0; + return retval; + out_unlock: + read_unlock(&tasklist_lock); + out: + return retval; + +} + +/* + * This is the crucial function for periodic task implementation, + * It checks if a task is periodic, checks if such kind of sleep + * is permitted and calls plugin-specific sleep, which puts the + * task into a wait array. + * returns 0 on successful wakeup + * returns EPERM if current conditions do not permit such sleep + * returns EINVAL if current task is not able to go to sleep + */ +asmlinkage long sys_complete_job(void) +{ + int retval = -EPERM; + if (!is_realtime(current)) { + retval = -EINVAL; + goto out; + } + /* Task with negative or zero period cannot sleep */ + if (get_rt_period(current) <= 0) { + retval = -EINVAL; + goto out; + } + /* The plugin has to put the task into an + * appropriate queue and call schedule + */ + retval = litmus->complete_job(); + out: + return retval; +} + +/* This is an "improved" version of sys_complete_job that + * addresses the problem of unintentionally missing a job after + * an overrun. + * + * returns 0 on successful wakeup + * returns EPERM if current conditions do not permit such sleep + * returns EINVAL if current task is not able to go to sleep + */ +asmlinkage long sys_wait_for_job_release(unsigned int job) +{ + int retval = -EPERM; + if (!is_realtime(current)) { + retval = -EINVAL; + goto out; + } + + /* Task with negative or zero period cannot sleep */ + if (get_rt_period(current) <= 0) { + retval = -EINVAL; + goto out; + } + + retval = 0; + + /* first wait until we have "reached" the desired job + * + * This implementation has at least two problems: + * + * 1) It doesn't gracefully handle the wrap around of + * job_no. Since LITMUS is a prototype, this is not much + * of a problem right now. + * + * 2) It is theoretically racy if a job release occurs + * between checking job_no and calling sleep_next_period(). + * A proper solution would requiring adding another callback + * in the plugin structure and testing the condition with + * interrupts disabled. + * + * FIXME: At least problem 2 should be taken care of eventually. + */ + while (!retval && job > current->rt_param.job_params.job_no) + /* If the last job overran then job <= job_no and we + * don't send the task to sleep. + */ + retval = litmus->complete_job(); + out: + return retval; +} + +/* This is a helper syscall to query the current job sequence number. + * + * returns 0 on successful query + * returns EPERM if task is not a real-time task. + * returns EFAULT if &job is not a valid pointer. + */ +asmlinkage long sys_query_job_no(unsigned int __user *job) +{ + int retval = -EPERM; + if (is_realtime(current)) + retval = put_user(current->rt_param.job_params.job_no, job); + + return retval; +} + + +/* sys_null_call() is only used for determining raw system call + * overheads (kernel entry, kernel exit). It has no useful side effects. + * If ts is non-NULL, then the current Feather-Trace time is recorded. + */ +asmlinkage long sys_null_call(cycles_t __user *ts) +{ + long ret = 0; + cycles_t now; + + if (ts) { + now = get_cycles(); + ret = put_user(now, ts); + } + + return ret; +} + + +#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) +void init_gpu_affinity_state(struct task_struct* p) +{ + // under-damped + //p->rt_param.gpu_fb_param_a = _frac(14008, 10000); + //p->rt_param.gpu_fb_param_b = _frac(16024, 10000); + + // emperical; + p->rt_param.gpu_fb_param_a[0] = _frac(7550, 10000); + p->rt_param.gpu_fb_param_b[0] = _frac(45800, 10000); + + p->rt_param.gpu_fb_param_a[1] = _frac(8600, 10000); + p->rt_param.gpu_fb_param_b[1] = _frac(40000, 10000); + + p->rt_param.gpu_fb_param_a[2] = _frac(6890, 10000); + p->rt_param.gpu_fb_param_b[2] = _frac(40000, 10000); + + p->rt_param.gpu_fb_param_a[3] = _frac(7580, 10000); + p->rt_param.gpu_fb_param_b[3] = _frac(34590, 10000); + + p->rt_param.gpu_migration = MIG_NONE; + p->rt_param.last_gpu = -1; +} +#endif + +/* p is a real-time task. Re-init its state as a best-effort task. */ +static void reinit_litmus_state(struct task_struct* p, int restore) +{ + struct rt_task user_config = {}; + void* ctrl_page = NULL; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + binheap_order_t prio_order = NULL; +#endif + + if (restore) { + /* Safe user-space provided configuration data. + * and allocated page. */ + user_config = p->rt_param.task_params; + ctrl_page = p->rt_param.ctrl_page; + } + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + prio_order = p->rt_param.hp_blocked_tasks.compare; +#endif + + /* We probably should not be inheriting any task's priority + * at this point in time. + */ + WARN_ON(p->rt_param.inh_task); + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + WARN_ON(p->rt_param.blocked_lock); + WARN_ON(!binheap_empty(&p->rt_param.hp_blocked_tasks)); +#endif + +#ifdef CONFIG_LITMUS_SOFTIRQD + /* We probably should not have any tasklets executing for + * us at this time. + */ + WARN_ON(p->rt_param.cur_klitirqd); + WARN_ON(atomic_read(&p->rt_param.klitirqd_sem_stat) == HELD); + + if(p->rt_param.cur_klitirqd) + flush_pending(p->rt_param.cur_klitirqd, p); + + if(atomic_read(&p->rt_param.klitirqd_sem_stat) == HELD) + up_and_set_stat(p, NOT_HELD, &p->rt_param.klitirqd_sem); +#endif + +#ifdef CONFIG_LITMUS_NVIDIA + WARN_ON(p->rt_param.held_gpus != 0); +#endif + + /* Cleanup everything else. */ + memset(&p->rt_param, 0, sizeof(p->rt_param)); + + /* Restore preserved fields. */ + if (restore) { + p->rt_param.task_params = user_config; + p->rt_param.ctrl_page = ctrl_page; + } + +#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) + init_gpu_affinity_state(p); +#endif + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + INIT_BINHEAP_HANDLE(&p->rt_param.hp_blocked_tasks, prio_order); + raw_spin_lock_init(&p->rt_param.hp_blocked_tasks_lock); +#endif +} + +long litmus_admit_task(struct task_struct* tsk) +{ + long retval = 0; + unsigned long flags; + + BUG_ON(is_realtime(tsk)); + + if (get_rt_period(tsk) == 0 || + get_exec_cost(tsk) > get_rt_period(tsk)) { + TRACE_TASK(tsk, "litmus admit: invalid task parameters " + "(%lu, %lu)\n", + get_exec_cost(tsk), get_rt_period(tsk)); + retval = -EINVAL; + goto out; + } + + if (!cpu_online(get_partition(tsk))) { + TRACE_TASK(tsk, "litmus admit: cpu %d is not online\n", + get_partition(tsk)); + retval = -EINVAL; + goto out; + } + + INIT_LIST_HEAD(&tsk_rt(tsk)->list); + + /* avoid scheduler plugin changing underneath us */ + raw_spin_lock_irqsave(&task_transition_lock, flags); + + /* allocate heap node for this task */ + tsk_rt(tsk)->heap_node = bheap_node_alloc(GFP_ATOMIC); + tsk_rt(tsk)->rel_heap = release_heap_alloc(GFP_ATOMIC); + + if (!tsk_rt(tsk)->heap_node || !tsk_rt(tsk)->rel_heap) { + printk(KERN_WARNING "litmus: no more heap node memory!?\n"); + + bheap_node_free(tsk_rt(tsk)->heap_node); + release_heap_free(tsk_rt(tsk)->rel_heap); + + retval = -ENOMEM; + goto out_unlock; + } else { + bheap_node_init(&tsk_rt(tsk)->heap_node, tsk); + } + + +#ifdef CONFIG_LITMUS_NVIDIA + atomic_set(&tsk_rt(tsk)->nv_int_count, 0); +#endif +#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) + init_gpu_affinity_state(tsk); +#endif +#ifdef CONFIG_LITMUS_NESTED_LOCKING + tsk_rt(tsk)->blocked_lock = NULL; + raw_spin_lock_init(&tsk_rt(tsk)->hp_blocked_tasks_lock); + //INIT_BINHEAP_HANDLE(&tsk_rt(tsk)->hp_blocked_tasks, prio_order); // done by scheduler +#endif +#ifdef CONFIG_LITMUS_SOFTIRQD + /* proxy thread off by default */ + tsk_rt(tsk)is_proxy_thread = 0; + tsk_rt(tsk)cur_klitirqd = NULL; + mutex_init(&tsk_rt(tsk)->klitirqd_sem); + atomic_set(&tsk_rt(tsk)->klitirqd_sem_stat, NOT_HELD); +#endif + + retval = litmus->admit_task(tsk); + + if (!retval) { + sched_trace_task_name(tsk); + sched_trace_task_param(tsk); + atomic_inc(&rt_task_count); + } + +out_unlock: + raw_spin_unlock_irqrestore(&task_transition_lock, flags); +out: + return retval; +} + +void litmus_exit_task(struct task_struct* tsk) +{ + if (is_realtime(tsk)) { + sched_trace_task_completion(tsk, 1); + + litmus->task_exit(tsk); + + BUG_ON(bheap_node_in_heap(tsk_rt(tsk)->heap_node)); + bheap_node_free(tsk_rt(tsk)->heap_node); + release_heap_free(tsk_rt(tsk)->rel_heap); + + atomic_dec(&rt_task_count); + reinit_litmus_state(tsk, 1); + } +} + +/* IPI callback to synchronize plugin switching */ +static void synch_on_plugin_switch(void* info) +{ + atomic_inc(&cannot_use_plugin); + while (atomic_read(&cannot_use_plugin) > 0) + cpu_relax(); +} + +/* Switching a plugin in use is tricky. + * We must watch out that no real-time tasks exists + * (and that none is created in parallel) and that the plugin is not + * currently in use on any processor (in theory). + */ +int switch_sched_plugin(struct sched_plugin* plugin) +{ + //unsigned long flags; + int ret = 0; + + BUG_ON(!plugin); + + /* forbid other cpus to use the plugin */ + atomic_set(&cannot_use_plugin, 1); + /* send IPI to force other CPUs to synch with us */ + smp_call_function(synch_on_plugin_switch, NULL, 0); + + /* wait until all other CPUs have started synch */ + while (atomic_read(&cannot_use_plugin) < num_online_cpus()) + cpu_relax(); + +#ifdef CONFIG_LITMUS_SOFTIRQD + if(!klitirqd_is_dead()) + { + kill_klitirqd(); + } +#endif + + /* stop task transitions */ + //raw_spin_lock_irqsave(&task_transition_lock, flags); + + /* don't switch if there are active real-time tasks */ + if (atomic_read(&rt_task_count) == 0) { + ret = litmus->deactivate_plugin(); + if (0 != ret) + goto out; + ret = plugin->activate_plugin(); + if (0 != ret) { + printk(KERN_INFO "Can't activate %s (%d).\n", + plugin->plugin_name, ret); + plugin = &linux_sched_plugin; + } + printk(KERN_INFO "Switching to LITMUS^RT plugin %s.\n", plugin->plugin_name); + litmus = plugin; + } else + ret = -EBUSY; +out: + //raw_spin_unlock_irqrestore(&task_transition_lock, flags); + atomic_set(&cannot_use_plugin, 0); + return ret; +} + +/* Called upon fork. + * p is the newly forked task. + */ +void litmus_fork(struct task_struct* p) +{ + if (is_realtime(p)) { + /* clean out any litmus related state, don't preserve anything */ + reinit_litmus_state(p, 0); + /* Don't let the child be a real-time task. */ + p->sched_reset_on_fork = 1; + } else + /* non-rt tasks might have ctrl_page set */ + tsk_rt(p)->ctrl_page = NULL; + + /* od tables are never inherited across a fork */ + p->od_table = NULL; +} + +/* Called upon execve(). + * current is doing the exec. + * Don't let address space specific stuff leak. + */ +void litmus_exec(void) +{ + struct task_struct* p = current; + + if (is_realtime(p)) { + WARN_ON(p->rt_param.inh_task); + if (tsk_rt(p)->ctrl_page) { + free_page((unsigned long) tsk_rt(p)->ctrl_page); + tsk_rt(p)->ctrl_page = NULL; + } + } +} + +void exit_litmus(struct task_struct *dead_tsk) +{ + /* We also allow non-RT tasks to + * allocate control pages to allow + * measurements with non-RT tasks. + * So check if we need to free the page + * in any case. + */ + if (tsk_rt(dead_tsk)->ctrl_page) { + TRACE_TASK(dead_tsk, + "freeing ctrl_page %p\n", + tsk_rt(dead_tsk)->ctrl_page); + free_page((unsigned long) tsk_rt(dead_tsk)->ctrl_page); + } + + /* main cleanup only for RT tasks */ + if (is_realtime(dead_tsk)) + litmus_exit_task(dead_tsk); +} + + +#ifdef CONFIG_MAGIC_SYSRQ +int sys_kill(int pid, int sig); + +static void sysrq_handle_kill_rt_tasks(int key) +{ + struct task_struct *t; + read_lock(&tasklist_lock); + for_each_process(t) { + if (is_realtime(t)) { + sys_kill(t->pid, SIGKILL); + } + } + read_unlock(&tasklist_lock); +} + +static struct sysrq_key_op sysrq_kill_rt_tasks_op = { + .handler = sysrq_handle_kill_rt_tasks, + .help_msg = "quit-rt-tasks(X)", + .action_msg = "sent SIGKILL to all LITMUS^RT real-time tasks", +}; +#endif + +extern struct sched_plugin linux_sched_plugin; + +static int __init _init_litmus(void) +{ + /* Common initializers, + * mode change lock is used to enforce single mode change + * operation. + */ + printk("Starting LITMUS^RT kernel\n"); + + BUILD_BUG_ON(sizeof(union np_flag) != sizeof(uint32_t)); + + register_sched_plugin(&linux_sched_plugin); + + bheap_node_cache = KMEM_CACHE(bheap_node, SLAB_PANIC); + release_heap_cache = KMEM_CACHE(release_heap, SLAB_PANIC); + +#ifdef CONFIG_MAGIC_SYSRQ + /* offer some debugging help */ + if (!register_sysrq_key('x', &sysrq_kill_rt_tasks_op)) + printk("Registered kill rt tasks magic sysrq.\n"); + else + printk("Could not register kill rt tasks magic sysrq.\n"); +#endif + + init_litmus_proc(); + +#ifdef CONFIG_SCHED_CPU_AFFINITY + init_topology(); +#endif + + return 0; +} + +static void _exit_litmus(void) +{ + exit_litmus_proc(); + kmem_cache_destroy(bheap_node_cache); + kmem_cache_destroy(release_heap_cache); +} + +module_init(_init_litmus); +module_exit(_exit_litmus); diff --git a/litmus/litmus_pai_softirq.c b/litmus/litmus_pai_softirq.c new file mode 100644 index 0000000..300571a --- /dev/null +++ b/litmus/litmus_pai_softirq.c @@ -0,0 +1,64 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + + + +int __litmus_tasklet_schedule(struct tasklet_struct *t, unsigned int k_id) +{ + int ret = 0; /* assume failure */ + if(unlikely((t->owner == NULL) || !is_realtime(t->owner))) + { + TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); + BUG(); + } + + ret = litmus->enqueue_pai_tasklet(t); + + return(ret); +} + +EXPORT_SYMBOL(__litmus_tasklet_schedule); + + + +// failure causes default Linux handling. +int __litmus_tasklet_hi_schedule(struct tasklet_struct *t, unsigned int k_id) +{ + int ret = 0; /* assume failure */ + return(ret); +} +EXPORT_SYMBOL(__litmus_tasklet_hi_schedule); + + +// failure causes default Linux handling. +int __litmus_tasklet_hi_schedule_first(struct tasklet_struct *t, unsigned int k_id) +{ + int ret = 0; /* assume failure */ + return(ret); +} +EXPORT_SYMBOL(__litmus_tasklet_hi_schedule_first); + + +// failure causes default Linux handling. +int __litmus_schedule_work(struct work_struct *w, unsigned int k_id) +{ + int ret = 0; /* assume failure */ + return(ret); +} +EXPORT_SYMBOL(__litmus_schedule_work); + diff --git a/litmus/litmus_proc.c b/litmus/litmus_proc.c new file mode 100644 index 0000000..9ab7e01 --- /dev/null +++ b/litmus/litmus_proc.c @@ -0,0 +1,364 @@ +/* + * litmus_proc.c -- Implementation of the /proc/litmus directory tree. + */ + +#include +#include + +#include +#include + +#include + +/* in litmus/litmus.c */ +extern atomic_t rt_task_count; + +static struct proc_dir_entry *litmus_dir = NULL, + *curr_file = NULL, + *stat_file = NULL, + *plugs_dir = NULL, +#ifdef CONFIG_RELEASE_MASTER + *release_master_file = NULL, +#endif +#ifdef CONFIG_LITMUS_SOFTIRQD + *klitirqd_file = NULL, +#endif + *plugs_file = NULL; + +/* in litmus/sync.c */ +int count_tasks_waiting_for_release(void); + +extern int proc_read_klitirqd_stats(char *page, char **start, + off_t off, int count, + int *eof, void *data); + +static int proc_read_stats(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + int len; + + len = snprintf(page, PAGE_SIZE, + "real-time tasks = %d\n" + "ready for release = %d\n", + atomic_read(&rt_task_count), + count_tasks_waiting_for_release()); + return len; +} + +static int proc_read_plugins(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + int len; + + len = print_sched_plugins(page, PAGE_SIZE); + return len; +} + +static int proc_read_curr(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + int len; + + len = snprintf(page, PAGE_SIZE, "%s\n", litmus->plugin_name); + return len; +} + +/* in litmus/litmus.c */ +int switch_sched_plugin(struct sched_plugin*); + +static int proc_write_curr(struct file *file, + const char *buffer, + unsigned long count, + void *data) +{ + int len, ret; + char name[65]; + struct sched_plugin* found; + + len = copy_and_chomp(name, sizeof(name), buffer, count); + if (len < 0) + return len; + + found = find_sched_plugin(name); + + if (found) { + ret = switch_sched_plugin(found); + if (ret != 0) + printk(KERN_INFO "Could not switch plugin: %d\n", ret); + } else + printk(KERN_INFO "Plugin '%s' is unknown.\n", name); + + return len; +} + +#ifdef CONFIG_RELEASE_MASTER +static int proc_read_release_master(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + int len, master; + master = atomic_read(&release_master_cpu); + if (master == NO_CPU) + len = snprintf(page, PAGE_SIZE, "NO_CPU\n"); + else + len = snprintf(page, PAGE_SIZE, "%d\n", master); + return len; +} + +static int proc_write_release_master(struct file *file, + const char *buffer, + unsigned long count, + void *data) +{ + int cpu, err, len, online = 0; + char msg[64]; + + len = copy_and_chomp(msg, sizeof(msg), buffer, count); + + if (len < 0) + return len; + + if (strcmp(msg, "NO_CPU") == 0) + atomic_set(&release_master_cpu, NO_CPU); + else { + err = sscanf(msg, "%d", &cpu); + if (err == 1 && cpu >= 0 && (online = cpu_online(cpu))) { + atomic_set(&release_master_cpu, cpu); + } else { + TRACE("invalid release master: '%s' " + "(err:%d cpu:%d online:%d)\n", + msg, err, cpu, online); + len = -EINVAL; + } + } + return len; +} +#endif + +int __init init_litmus_proc(void) +{ + litmus_dir = proc_mkdir("litmus", NULL); + if (!litmus_dir) { + printk(KERN_ERR "Could not allocate LITMUS^RT procfs entry.\n"); + return -ENOMEM; + } + + curr_file = create_proc_entry("active_plugin", + 0644, litmus_dir); + if (!curr_file) { + printk(KERN_ERR "Could not allocate active_plugin " + "procfs entry.\n"); + return -ENOMEM; + } + curr_file->read_proc = proc_read_curr; + curr_file->write_proc = proc_write_curr; + +#ifdef CONFIG_RELEASE_MASTER + release_master_file = create_proc_entry("release_master", + 0644, litmus_dir); + if (!release_master_file) { + printk(KERN_ERR "Could not allocate release_master " + "procfs entry.\n"); + return -ENOMEM; + } + release_master_file->read_proc = proc_read_release_master; + release_master_file->write_proc = proc_write_release_master; +#endif + +#ifdef CONFIG_LITMUS_SOFTIRQD + klitirqd_file = + create_proc_read_entry("klitirqd_stats", 0444, litmus_dir, + proc_read_klitirqd_stats, NULL); +#endif + + stat_file = create_proc_read_entry("stats", 0444, litmus_dir, + proc_read_stats, NULL); + + plugs_dir = proc_mkdir("plugins", litmus_dir); + if (!plugs_dir){ + printk(KERN_ERR "Could not allocate plugins directory " + "procfs entry.\n"); + return -ENOMEM; + } + + plugs_file = create_proc_read_entry("loaded", 0444, plugs_dir, + proc_read_plugins, NULL); + + return 0; +} + +void exit_litmus_proc(void) +{ + if (plugs_file) + remove_proc_entry("loaded", plugs_dir); + if (plugs_dir) + remove_proc_entry("plugins", litmus_dir); + if (stat_file) + remove_proc_entry("stats", litmus_dir); + if (curr_file) + remove_proc_entry("active_plugin", litmus_dir); +#ifdef CONFIG_LITMUS_SOFTIRQD + if (klitirqd_file) + remove_proc_entry("klitirqd_stats", litmus_dir); +#endif +#ifdef CONFIG_RELEASE_MASTER + if (release_master_file) + remove_proc_entry("release_master", litmus_dir); +#endif + if (litmus_dir) + remove_proc_entry("litmus", NULL); +} + +long make_plugin_proc_dir(struct sched_plugin* plugin, + struct proc_dir_entry** pde_in) +{ + struct proc_dir_entry *pde_new = NULL; + long rv; + + if (!plugin || !plugin->plugin_name){ + printk(KERN_ERR "Invalid plugin struct passed to %s.\n", + __func__); + rv = -EINVAL; + goto out_no_pde; + } + + if (!plugs_dir){ + printk(KERN_ERR "Could not make plugin sub-directory, because " + "/proc/litmus/plugins does not exist.\n"); + rv = -ENOENT; + goto out_no_pde; + } + + pde_new = proc_mkdir(plugin->plugin_name, plugs_dir); + if (!pde_new){ + printk(KERN_ERR "Could not make plugin sub-directory: " + "out of memory?.\n"); + rv = -ENOMEM; + goto out_no_pde; + } + + rv = 0; + *pde_in = pde_new; + goto out_ok; + +out_no_pde: + *pde_in = NULL; +out_ok: + return rv; +} + +void remove_plugin_proc_dir(struct sched_plugin* plugin) +{ + if (!plugin || !plugin->plugin_name){ + printk(KERN_ERR "Invalid plugin struct passed to %s.\n", + __func__); + return; + } + remove_proc_entry(plugin->plugin_name, plugs_dir); +} + + + +/* misc. I/O helper functions */ + +int copy_and_chomp(char *kbuf, unsigned long ksize, + __user const char* ubuf, unsigned long ulength) +{ + /* caller must provide buffer space */ + BUG_ON(!ksize); + + ksize--; /* leave space for null byte */ + + if (ksize > ulength) + ksize = ulength; + + if(copy_from_user(kbuf, ubuf, ksize)) + return -EFAULT; + + kbuf[ksize] = '\0'; + + /* chomp kbuf */ + if (ksize > 0 && kbuf[ksize - 1] == '\n') + kbuf[ksize - 1] = '\0'; + + return ksize; +} + +/* helper functions for clustered plugins */ +static const char* cache_level_names[] = { + "ALL", + "L1", + "L2", + "L3", +}; + +int parse_cache_level(const char *cache_name, enum cache_level *level) +{ + int err = -EINVAL; + int i; + /* do a quick and dirty comparison to find the cluster size */ + for (i = GLOBAL_CLUSTER; i <= L3_CLUSTER; i++) + if (!strcmp(cache_name, cache_level_names[i])) { + *level = (enum cache_level) i; + err = 0; + break; + } + return err; +} + +const char* cache_level_name(enum cache_level level) +{ + int idx = level; + + if (idx >= GLOBAL_CLUSTER && idx <= L3_CLUSTER) + return cache_level_names[idx]; + else + return "INVALID"; +} + + +/* proc file interface to configure the cluster size */ +static int proc_read_cluster_size(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + return snprintf(page, PAGE_SIZE, "%s\n", + cache_level_name(*((enum cache_level*) data)));; +} + +static int proc_write_cluster_size(struct file *file, + const char *buffer, + unsigned long count, + void *data) +{ + int len; + char cache_name[8]; + + len = copy_and_chomp(cache_name, sizeof(cache_name), buffer, count); + + if (len > 0 && parse_cache_level(cache_name, (enum cache_level*) data)) + printk(KERN_INFO "Cluster '%s' is unknown.\n", cache_name); + + return len; +} + +struct proc_dir_entry* create_cluster_file(struct proc_dir_entry* parent, + enum cache_level* level) +{ + struct proc_dir_entry* cluster_file; + + cluster_file = create_proc_entry("cluster", 0644, parent); + if (!cluster_file) { + printk(KERN_ERR "Could not allocate %s/cluster " + "procfs entry.\n", parent->name); + } else { + cluster_file->read_proc = proc_read_cluster_size; + cluster_file->write_proc = proc_write_cluster_size; + cluster_file->data = level; + } + return cluster_file; +} + diff --git a/litmus/litmus_softirq.c b/litmus/litmus_softirq.c new file mode 100644 index 0000000..9f7d9da --- /dev/null +++ b/litmus/litmus_softirq.c @@ -0,0 +1,1582 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +/* TODO: Remove unneeded mb() and other barriers. */ + + +/* counts number of daemons ready to handle litmus irqs. */ +static atomic_t num_ready_klitirqds = ATOMIC_INIT(0); + +enum pending_flags +{ + LIT_TASKLET_LOW = 0x1, + LIT_TASKLET_HI = LIT_TASKLET_LOW<<1, + LIT_WORK = LIT_TASKLET_HI<<1 +}; + +/* only support tasklet processing for now. */ +struct tasklet_head +{ + struct tasklet_struct *head; + struct tasklet_struct **tail; +}; + +struct klitirqd_info +{ + struct task_struct* klitirqd; + struct task_struct* current_owner; + int terminating; + + + raw_spinlock_t lock; + + u32 pending; + atomic_t num_hi_pending; + atomic_t num_low_pending; + atomic_t num_work_pending; + + /* in order of priority */ + struct tasklet_head pending_tasklets_hi; + struct tasklet_head pending_tasklets; + struct list_head worklist; +}; + +/* one list for each klitirqd */ +static struct klitirqd_info klitirqds[NR_LITMUS_SOFTIRQD]; + + + + + +int proc_read_klitirqd_stats(char *page, char **start, + off_t off, int count, + int *eof, void *data) +{ + int len = snprintf(page, PAGE_SIZE, + "num ready klitirqds: %d\n\n", + atomic_read(&num_ready_klitirqds)); + + if(klitirqd_is_ready()) + { + int i; + for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) + { + len += + snprintf(page + len - 1, PAGE_SIZE, /* -1 to strip off \0 */ + "klitirqd_th%d: %s/%d\n" + "\tcurrent_owner: %s/%d\n" + "\tpending: %x\n" + "\tnum hi: %d\n" + "\tnum low: %d\n" + "\tnum work: %d\n\n", + i, + klitirqds[i].klitirqd->comm, klitirqds[i].klitirqd->pid, + (klitirqds[i].current_owner != NULL) ? + klitirqds[i].current_owner->comm : "(null)", + (klitirqds[i].current_owner != NULL) ? + klitirqds[i].current_owner->pid : 0, + klitirqds[i].pending, + atomic_read(&klitirqds[i].num_hi_pending), + atomic_read(&klitirqds[i].num_low_pending), + atomic_read(&klitirqds[i].num_work_pending)); + } + } + + return(len); +} + + + + + +#if 0 +static atomic_t dump_id = ATOMIC_INIT(0); + +static void __dump_state(struct klitirqd_info* which, const char* caller) +{ + struct tasklet_struct* list; + + int id = atomic_inc_return(&dump_id); + + //if(in_interrupt()) + { + if(which->current_owner) + { + TRACE("(id: %d caller: %s)\n" + "klitirqd: %s/%d\n" + "current owner: %s/%d\n" + "pending: %x\n", + id, caller, + which->klitirqd->comm, which->klitirqd->pid, + which->current_owner->comm, which->current_owner->pid, + which->pending); + } + else + { + TRACE("(id: %d caller: %s)\n" + "klitirqd: %s/%d\n" + "current owner: %p\n" + "pending: %x\n", + id, caller, + which->klitirqd->comm, which->klitirqd->pid, + NULL, + which->pending); + } + + list = which->pending_tasklets.head; + while(list) + { + struct tasklet_struct *t = list; + list = list->next; /* advance */ + if(t->owner) + TRACE("(id: %d caller: %s) Tasklet: %x, Owner = %s/%d\n", id, caller, t, t->owner->comm, t->owner->pid); + else + TRACE("(id: %d caller: %s) Tasklet: %x, Owner = %p\n", id, caller, t, NULL); + } + } +} + +static void dump_state(struct klitirqd_info* which, const char* caller) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&which->lock, flags); + __dump_state(which, caller); + raw_spin_unlock_irqrestore(&which->lock, flags); +} +#endif + + +/* forward declarations */ +static void ___litmus_tasklet_schedule(struct tasklet_struct *t, + struct klitirqd_info *which, + int wakeup); +static void ___litmus_tasklet_hi_schedule(struct tasklet_struct *t, + struct klitirqd_info *which, + int wakeup); +static void ___litmus_schedule_work(struct work_struct *w, + struct klitirqd_info *which, + int wakeup); + + + +inline unsigned int klitirqd_id(struct task_struct* tsk) +{ + int i; + for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) + { + if(klitirqds[i].klitirqd == tsk) + { + return i; + } + } + + BUG(); + + return 0; +} + + +inline static u32 litirq_pending_hi_irqoff(struct klitirqd_info* which) +{ + return (which->pending & LIT_TASKLET_HI); +} + +inline static u32 litirq_pending_low_irqoff(struct klitirqd_info* which) +{ + return (which->pending & LIT_TASKLET_LOW); +} + +inline static u32 litirq_pending_work_irqoff(struct klitirqd_info* which) +{ + return (which->pending & LIT_WORK); +} + +inline static u32 litirq_pending_irqoff(struct klitirqd_info* which) +{ + return(which->pending); +} + + +inline static u32 litirq_pending(struct klitirqd_info* which) +{ + unsigned long flags; + u32 pending; + + raw_spin_lock_irqsave(&which->lock, flags); + pending = litirq_pending_irqoff(which); + raw_spin_unlock_irqrestore(&which->lock, flags); + + return pending; +}; + +inline static u32 litirq_pending_with_owner(struct klitirqd_info* which, struct task_struct* owner) +{ + unsigned long flags; + u32 pending; + + raw_spin_lock_irqsave(&which->lock, flags); + pending = litirq_pending_irqoff(which); + if(pending) + { + if(which->current_owner != owner) + { + pending = 0; // owner switch! + } + } + raw_spin_unlock_irqrestore(&which->lock, flags); + + return pending; +} + + +inline static u32 litirq_pending_and_sem_and_owner(struct klitirqd_info* which, + struct mutex** sem, + struct task_struct** t) +{ + unsigned long flags; + u32 pending; + + /* init values */ + *sem = NULL; + *t = NULL; + + raw_spin_lock_irqsave(&which->lock, flags); + + pending = litirq_pending_irqoff(which); + if(pending) + { + if(which->current_owner != NULL) + { + *t = which->current_owner; + *sem = &tsk_rt(which->current_owner)->klitirqd_sem; + } + else + { + BUG(); + } + } + raw_spin_unlock_irqrestore(&which->lock, flags); + + if(likely(*sem)) + { + return pending; + } + else + { + return 0; + } +} + +/* returns true if the next piece of work to do is from a different owner. + */ +static int tasklet_ownership_change( + struct klitirqd_info* which, + enum pending_flags taskletQ) +{ + /* this function doesn't have to look at work objects since they have + priority below tasklets. */ + + unsigned long flags; + int ret = 0; + + raw_spin_lock_irqsave(&which->lock, flags); + + switch(taskletQ) + { + case LIT_TASKLET_HI: + if(litirq_pending_hi_irqoff(which)) + { + ret = (which->pending_tasklets_hi.head->owner != + which->current_owner); + } + break; + case LIT_TASKLET_LOW: + if(litirq_pending_low_irqoff(which)) + { + ret = (which->pending_tasklets.head->owner != + which->current_owner); + } + break; + default: + break; + } + + raw_spin_unlock_irqrestore(&which->lock, flags); + + TRACE_TASK(which->klitirqd, "ownership change needed: %d\n", ret); + + return ret; +} + + +static void __reeval_prio(struct klitirqd_info* which) +{ + struct task_struct* next_owner = NULL; + struct task_struct* klitirqd = which->klitirqd; + + /* Check in prio-order */ + u32 pending = litirq_pending_irqoff(which); + + //__dump_state(which, "__reeval_prio: before"); + + if(pending) + { + if(pending & LIT_TASKLET_HI) + { + next_owner = which->pending_tasklets_hi.head->owner; + } + else if(pending & LIT_TASKLET_LOW) + { + next_owner = which->pending_tasklets.head->owner; + } + else if(pending & LIT_WORK) + { + struct work_struct* work = + list_first_entry(&which->worklist, struct work_struct, entry); + next_owner = work->owner; + } + } + + if(next_owner != which->current_owner) + { + struct task_struct* old_owner = which->current_owner; + + /* bind the next owner. */ + which->current_owner = next_owner; + mb(); + + if(next_owner != NULL) + { + if(!in_interrupt()) + { + TRACE_CUR("%s: Ownership change: %s/%d to %s/%d\n", __FUNCTION__, + ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->comm, + ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->pid, + next_owner->comm, next_owner->pid); + } + else + { + TRACE("%s: Ownership change: %s/%d to %s/%d\n", __FUNCTION__, + ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->comm, + ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->pid, + next_owner->comm, next_owner->pid); + } + + litmus->increase_prio_inheritance_klitirqd(klitirqd, old_owner, next_owner); + } + else + { + if(likely(!in_interrupt())) + { + TRACE_CUR("%s: Ownership change: %s/%d to NULL (reverting)\n", + __FUNCTION__, klitirqd->comm, klitirqd->pid); + } + else + { + // is this a bug? + TRACE("%s: Ownership change: %s/%d to NULL (reverting)\n", + __FUNCTION__, klitirqd->comm, klitirqd->pid); + } + + BUG_ON(pending != 0); + litmus->decrease_prio_inheritance_klitirqd(klitirqd, old_owner, NULL); + } + } + + //__dump_state(which, "__reeval_prio: after"); +} + +static void reeval_prio(struct klitirqd_info* which) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&which->lock, flags); + __reeval_prio(which); + raw_spin_unlock_irqrestore(&which->lock, flags); +} + + +static void wakeup_litirqd_locked(struct klitirqd_info* which) +{ + /* Interrupts are disabled: no need to stop preemption */ + if (which && which->klitirqd) + { + __reeval_prio(which); /* configure the proper priority */ + + if(which->klitirqd->state != TASK_RUNNING) + { + TRACE("%s: Waking up klitirqd: %s/%d\n", __FUNCTION__, + which->klitirqd->comm, which->klitirqd->pid); + + wake_up_process(which->klitirqd); + } + } +} + + +static void do_lit_tasklet(struct klitirqd_info* which, + struct tasklet_head* pending_tasklets) +{ + unsigned long flags; + struct tasklet_struct *list; + atomic_t* count; + + raw_spin_lock_irqsave(&which->lock, flags); + + //__dump_state(which, "do_lit_tasklet: before steal"); + + /* copy out the tasklets for our private use. */ + list = pending_tasklets->head; + pending_tasklets->head = NULL; + pending_tasklets->tail = &pending_tasklets->head; + + /* remove pending flag */ + which->pending &= (pending_tasklets == &which->pending_tasklets) ? + ~LIT_TASKLET_LOW : + ~LIT_TASKLET_HI; + + count = (pending_tasklets == &which->pending_tasklets) ? + &which->num_low_pending: + &which->num_hi_pending; + + //__dump_state(which, "do_lit_tasklet: after steal"); + + raw_spin_unlock_irqrestore(&which->lock, flags); + + + while(list) + { + struct tasklet_struct *t = list; + + /* advance, lest we forget */ + list = list->next; + + /* execute tasklet if it has my priority and is free */ + if ((t->owner == which->current_owner) && tasklet_trylock(t)) { + if (!atomic_read(&t->count)) { + + sched_trace_tasklet_begin(t->owner); + + if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) + { + BUG(); + } + TRACE_CUR("%s: Invoking tasklet.\n", __FUNCTION__); + t->func(t->data); + tasklet_unlock(t); + + atomic_dec(count); + + sched_trace_tasklet_end(t->owner, 0ul); + + continue; /* process more tasklets */ + } + tasklet_unlock(t); + } + + TRACE_CUR("%s: Could not invoke tasklet. Requeuing.\n", __FUNCTION__); + + /* couldn't process tasklet. put it back at the end of the queue. */ + if(pending_tasklets == &which->pending_tasklets) + ___litmus_tasklet_schedule(t, which, 0); + else + ___litmus_tasklet_hi_schedule(t, which, 0); + } +} + + +// returns 1 if priorities need to be changed to continue processing +// pending tasklets. +static int do_litirq(struct klitirqd_info* which) +{ + u32 pending; + int resched = 0; + + if(in_interrupt()) + { + TRACE("%s: exiting early: in interrupt context!\n", __FUNCTION__); + return(0); + } + + if(which->klitirqd != current) + { + TRACE_CUR("%s: exiting early: thread/info mismatch! Running %s/%d but given %s/%d.\n", + __FUNCTION__, current->comm, current->pid, + which->klitirqd->comm, which->klitirqd->pid); + return(0); + } + + if(!is_realtime(current)) + { + TRACE_CUR("%s: exiting early: klitirqd is not real-time. Sched Policy = %d\n", + __FUNCTION__, current->policy); + return(0); + } + + + /* We only handle tasklets & work objects, no need for RCU triggers? */ + + pending = litirq_pending(which); + if(pending) + { + /* extract the work to do and do it! */ + if(pending & LIT_TASKLET_HI) + { + TRACE_CUR("%s: Invoking HI tasklets.\n", __FUNCTION__); + do_lit_tasklet(which, &which->pending_tasklets_hi); + resched = tasklet_ownership_change(which, LIT_TASKLET_HI); + + if(resched) + { + TRACE_CUR("%s: HI tasklets of another owner remain. " + "Skipping any LOW tasklets.\n", __FUNCTION__); + } + } + + if(!resched && (pending & LIT_TASKLET_LOW)) + { + TRACE_CUR("%s: Invoking LOW tasklets.\n", __FUNCTION__); + do_lit_tasklet(which, &which->pending_tasklets); + resched = tasklet_ownership_change(which, LIT_TASKLET_LOW); + + if(resched) + { + TRACE_CUR("%s: LOW tasklets of another owner remain. " + "Skipping any work objects.\n", __FUNCTION__); + } + } + } + + return(resched); +} + + +static void do_work(struct klitirqd_info* which) +{ + unsigned long flags; + work_func_t f; + struct work_struct* work; + + // only execute one work-queue item to yield to tasklets. + // ...is this a good idea, or should we just batch them? + raw_spin_lock_irqsave(&which->lock, flags); + + if(!litirq_pending_work_irqoff(which)) + { + raw_spin_unlock_irqrestore(&which->lock, flags); + goto no_work; + } + + work = list_first_entry(&which->worklist, struct work_struct, entry); + list_del_init(&work->entry); + + if(list_empty(&which->worklist)) + { + which->pending &= ~LIT_WORK; + } + + raw_spin_unlock_irqrestore(&which->lock, flags); + + + + /* safe to read current_owner outside of lock since only this thread + may write to the pointer. */ + if(work->owner == which->current_owner) + { + TRACE_CUR("%s: Invoking work object.\n", __FUNCTION__); + // do the work! + work_clear_pending(work); + f = work->func; + f(work); /* can't touch 'work' after this point, + the user may have freed it. */ + + atomic_dec(&which->num_work_pending); + } + else + { + TRACE_CUR("%s: Could not invoke work object. Requeuing.\n", + __FUNCTION__); + ___litmus_schedule_work(work, which, 0); + } + +no_work: + return; +} + + +static int set_litmus_daemon_sched(void) +{ + /* set up a daemon job that will never complete. + it should only ever run on behalf of another + real-time task. + + TODO: Transition to a new job whenever a + new tasklet is handled */ + + int ret = 0; + + struct rt_task tp = { + .exec_cost = 0, + .period = 1000000000, /* dummy 1 second period */ + .phase = 0, + .cpu = task_cpu(current), + .budget_policy = NO_ENFORCEMENT, + .cls = RT_CLASS_BEST_EFFORT + }; + + struct sched_param param = { .sched_priority = 0}; + + + /* set task params, mark as proxy thread, and init other data */ + tsk_rt(current)->task_params = tp; + tsk_rt(current)->is_proxy_thread = 1; + tsk_rt(current)->cur_klitirqd = NULL; + mutex_init(&tsk_rt(current)->klitirqd_sem); + atomic_set(&tsk_rt(current)->klitirqd_sem_stat, NOT_HELD); + + /* inform the OS we're SCHED_LITMUS -- + sched_setscheduler_nocheck() calls litmus_admit_task(). */ + sched_setscheduler_nocheck(current, SCHED_LITMUS, ¶m); + + return ret; +} + +static void enter_execution_phase(struct klitirqd_info* which, + struct mutex* sem, + struct task_struct* t) +{ + TRACE_CUR("%s: Trying to enter execution phase. " + "Acquiring semaphore of %s/%d\n", __FUNCTION__, + t->comm, t->pid); + down_and_set_stat(current, HELD, sem); + TRACE_CUR("%s: Execution phase entered! " + "Acquired semaphore of %s/%d\n", __FUNCTION__, + t->comm, t->pid); +} + +static void exit_execution_phase(struct klitirqd_info* which, + struct mutex* sem, + struct task_struct* t) +{ + TRACE_CUR("%s: Exiting execution phase. " + "Releasing semaphore of %s/%d\n", __FUNCTION__, + t->comm, t->pid); + if(atomic_read(&tsk_rt(current)->klitirqd_sem_stat) == HELD) + { + up_and_set_stat(current, NOT_HELD, sem); + TRACE_CUR("%s: Execution phase exited! " + "Released semaphore of %s/%d\n", __FUNCTION__, + t->comm, t->pid); + } + else + { + TRACE_CUR("%s: COULDN'T RELEASE SEMAPHORE BECAUSE ONE IS NOT HELD!\n", __FUNCTION__); + } +} + +/* main loop for klitsoftirqd */ +static int run_klitirqd(void* unused) +{ + struct klitirqd_info* which = &klitirqds[klitirqd_id(current)]; + struct mutex* sem; + struct task_struct* owner; + + int rt_status = set_litmus_daemon_sched(); + + if(rt_status != 0) + { + TRACE_CUR("%s: Failed to transition to rt-task.\n", __FUNCTION__); + goto rt_failed; + } + + atomic_inc(&num_ready_klitirqds); + + set_current_state(TASK_INTERRUPTIBLE); + + while (!kthread_should_stop()) + { + preempt_disable(); + if (!litirq_pending(which)) + { + /* sleep for work */ + TRACE_CUR("%s: No more tasklets or work objects. Going to sleep.\n", + __FUNCTION__); + preempt_enable_no_resched(); + schedule(); + + if(kthread_should_stop()) /* bail out */ + { + TRACE_CUR("%s:%d: Signaled to terminate.\n", __FUNCTION__, __LINE__); + continue; + } + + preempt_disable(); + } + + __set_current_state(TASK_RUNNING); + + while (litirq_pending_and_sem_and_owner(which, &sem, &owner)) + { + int needs_resched = 0; + + preempt_enable_no_resched(); + + BUG_ON(sem == NULL); + + // wait to enter execution phase; wait for 'current_owner' to block. + enter_execution_phase(which, sem, owner); + + if(kthread_should_stop()) + { + TRACE_CUR("%s:%d: Signaled to terminate.\n", __FUNCTION__, __LINE__); + break; + } + + preempt_disable(); + + /* Double check that there's still pending work and the owner hasn't + * changed. Pending items may have been flushed while we were sleeping. + */ + if(litirq_pending_with_owner(which, owner)) + { + TRACE_CUR("%s: Executing tasklets and/or work objects.\n", + __FUNCTION__); + + needs_resched = do_litirq(which); + + preempt_enable_no_resched(); + + // work objects are preemptible. + if(!needs_resched) + { + do_work(which); + } + + // exit execution phase. + exit_execution_phase(which, sem, owner); + + TRACE_CUR("%s: Setting up next priority.\n", __FUNCTION__); + reeval_prio(which); /* check if we need to change priority here */ + } + else + { + TRACE_CUR("%s: Pending work was flushed! Prev owner was %s/%d\n", + __FUNCTION__, + owner->comm, owner->pid); + preempt_enable_no_resched(); + + // exit execution phase. + exit_execution_phase(which, sem, owner); + } + + cond_resched(); + preempt_disable(); + } + preempt_enable(); + set_current_state(TASK_INTERRUPTIBLE); + } + __set_current_state(TASK_RUNNING); + + atomic_dec(&num_ready_klitirqds); + +rt_failed: + litmus_exit_task(current); + + return rt_status; +} + + +struct klitirqd_launch_data +{ + int* cpu_affinity; + struct work_struct work; +}; + +/* executed by a kworker from workqueues */ +static void launch_klitirqd(struct work_struct *work) +{ + int i; + + struct klitirqd_launch_data* launch_data = + container_of(work, struct klitirqd_launch_data, work); + + TRACE("%s: Creating %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD); + + /* create the daemon threads */ + for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) + { + if(launch_data->cpu_affinity) + { + klitirqds[i].klitirqd = + kthread_create( + run_klitirqd, + /* treat the affinity as a pointer, we'll cast it back later */ + (void*)(long long)launch_data->cpu_affinity[i], + "klitirqd_th%d/%d", + i, + launch_data->cpu_affinity[i]); + + /* litmus will put is in the right cluster. */ + kthread_bind(klitirqds[i].klitirqd, launch_data->cpu_affinity[i]); + } + else + { + klitirqds[i].klitirqd = + kthread_create( + run_klitirqd, + /* treat the affinity as a pointer, we'll cast it back later */ + (void*)(long long)(-1), + "klitirqd_th%d", + i); + } + } + + TRACE("%s: Launching %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD); + + /* unleash the daemons */ + for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) + { + wake_up_process(klitirqds[i].klitirqd); + } + + if(launch_data->cpu_affinity) + kfree(launch_data->cpu_affinity); + kfree(launch_data); +} + + +void spawn_klitirqd(int* affinity) +{ + int i; + struct klitirqd_launch_data* delayed_launch; + + if(atomic_read(&num_ready_klitirqds) != 0) + { + TRACE("%s: At least one klitirqd is already running! Need to call kill_klitirqd()?\n"); + return; + } + + /* init the tasklet & work queues */ + for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) + { + klitirqds[i].terminating = 0; + klitirqds[i].pending = 0; + + klitirqds[i].num_hi_pending.counter = 0; + klitirqds[i].num_low_pending.counter = 0; + klitirqds[i].num_work_pending.counter = 0; + + klitirqds[i].pending_tasklets_hi.head = NULL; + klitirqds[i].pending_tasklets_hi.tail = &klitirqds[i].pending_tasklets_hi.head; + + klitirqds[i].pending_tasklets.head = NULL; + klitirqds[i].pending_tasklets.tail = &klitirqds[i].pending_tasklets.head; + + INIT_LIST_HEAD(&klitirqds[i].worklist); + + raw_spin_lock_init(&klitirqds[i].lock); + } + + /* wait to flush the initializations to memory since other threads + will access it. */ + mb(); + + /* tell a work queue to launch the threads. we can't make scheduling + calls since we're in an atomic state. */ + TRACE("%s: Setting callback up to launch klitirqds\n", __FUNCTION__); + delayed_launch = kmalloc(sizeof(struct klitirqd_launch_data), GFP_ATOMIC); + if(affinity) + { + delayed_launch->cpu_affinity = + kmalloc(sizeof(int)*NR_LITMUS_SOFTIRQD, GFP_ATOMIC); + + memcpy(delayed_launch->cpu_affinity, affinity, + sizeof(int)*NR_LITMUS_SOFTIRQD); + } + else + { + delayed_launch->cpu_affinity = NULL; + } + INIT_WORK(&delayed_launch->work, launch_klitirqd); + schedule_work(&delayed_launch->work); +} + + +void kill_klitirqd(void) +{ + if(!klitirqd_is_dead()) + { + int i; + + TRACE("%s: Killing %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD); + + for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i) + { + if(klitirqds[i].terminating != 1) + { + klitirqds[i].terminating = 1; + mb(); /* just to be sure? */ + flush_pending(klitirqds[i].klitirqd, NULL); + + /* signal termination */ + kthread_stop(klitirqds[i].klitirqd); + } + } + } +} + + +int klitirqd_is_ready(void) +{ + return(atomic_read(&num_ready_klitirqds) == NR_LITMUS_SOFTIRQD); +} + +int klitirqd_is_dead(void) +{ + return(atomic_read(&num_ready_klitirqds) == 0); +} + + +struct task_struct* get_klitirqd(unsigned int k_id) +{ + return(klitirqds[k_id].klitirqd); +} + + +void flush_pending(struct task_struct* klitirqd_thread, + struct task_struct* owner) +{ + unsigned int k_id = klitirqd_id(klitirqd_thread); + struct klitirqd_info *which = &klitirqds[k_id]; + + unsigned long flags; + struct tasklet_struct *list; + + u32 work_flushed = 0; + + raw_spin_lock_irqsave(&which->lock, flags); + + //__dump_state(which, "flush_pending: before"); + + // flush hi tasklets. + if(litirq_pending_hi_irqoff(which)) + { + which->pending &= ~LIT_TASKLET_HI; + + list = which->pending_tasklets_hi.head; + which->pending_tasklets_hi.head = NULL; + which->pending_tasklets_hi.tail = &which->pending_tasklets_hi.head; + + TRACE("%s: Handing HI tasklets back to Linux.\n", __FUNCTION__); + + while(list) + { + struct tasklet_struct *t = list; + list = list->next; + + if(likely((t->owner == owner) || (owner == NULL))) + { + if(unlikely(!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))) + { + BUG(); + } + + work_flushed |= LIT_TASKLET_HI; + + t->owner = NULL; + + // WTF? + if(!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) + { + atomic_dec(&which->num_hi_pending); + ___tasklet_hi_schedule(t); + } + else + { + TRACE("%s: dropped hi tasklet??\n", __FUNCTION__); + BUG(); + } + } + else + { + TRACE("%s: Could not flush a HI tasklet.\n", __FUNCTION__); + // put back on queue. + ___litmus_tasklet_hi_schedule(t, which, 0); + } + } + } + + // flush low tasklets. + if(litirq_pending_low_irqoff(which)) + { + which->pending &= ~LIT_TASKLET_LOW; + + list = which->pending_tasklets.head; + which->pending_tasklets.head = NULL; + which->pending_tasklets.tail = &which->pending_tasklets.head; + + TRACE("%s: Handing LOW tasklets back to Linux.\n", __FUNCTION__); + + while(list) + { + struct tasklet_struct *t = list; + list = list->next; + + if(likely((t->owner == owner) || (owner == NULL))) + { + if(unlikely(!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))) + { + BUG(); + } + + work_flushed |= LIT_TASKLET_LOW; + + t->owner = NULL; + sched_trace_tasklet_end(owner, 1ul); + + if(!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) + { + atomic_dec(&which->num_low_pending); + ___tasklet_schedule(t); + } + else + { + TRACE("%s: dropped tasklet??\n", __FUNCTION__); + BUG(); + } + } + else + { + TRACE("%s: Could not flush a LOW tasklet.\n", __FUNCTION__); + // put back on queue + ___litmus_tasklet_schedule(t, which, 0); + } + } + } + + // flush work objects + if(litirq_pending_work_irqoff(which)) + { + which->pending &= ~LIT_WORK; + + TRACE("%s: Handing work objects back to Linux.\n", __FUNCTION__); + + while(!list_empty(&which->worklist)) + { + struct work_struct* work = + list_first_entry(&which->worklist, struct work_struct, entry); + list_del_init(&work->entry); + + if(likely((work->owner == owner) || (owner == NULL))) + { + work_flushed |= LIT_WORK; + atomic_dec(&which->num_work_pending); + + work->owner = NULL; + sched_trace_work_end(owner, current, 1ul); + __schedule_work(work); + } + else + { + TRACE("%s: Could not flush a work object.\n", __FUNCTION__); + // put back on queue + ___litmus_schedule_work(work, which, 0); + } + } + } + + //__dump_state(which, "flush_pending: after (before reeval prio)"); + + + mb(); /* commit changes to pending flags */ + + /* reset the scheduling priority */ + if(work_flushed) + { + __reeval_prio(which); + + /* Try to offload flushed tasklets to Linux's ksoftirqd. */ + if(work_flushed & (LIT_TASKLET_LOW | LIT_TASKLET_HI)) + { + wakeup_softirqd(); + } + } + else + { + TRACE_CUR("%s: no work flushed, so __reeval_prio() skipped\n", __FUNCTION__); + } + + raw_spin_unlock_irqrestore(&which->lock, flags); +} + + + + +static void ___litmus_tasklet_schedule(struct tasklet_struct *t, + struct klitirqd_info *which, + int wakeup) +{ + unsigned long flags; + u32 old_pending; + + t->next = NULL; + + raw_spin_lock_irqsave(&which->lock, flags); + + //__dump_state(which, "___litmus_tasklet_schedule: before queuing"); + + *(which->pending_tasklets.tail) = t; + which->pending_tasklets.tail = &t->next; + + old_pending = which->pending; + which->pending |= LIT_TASKLET_LOW; + + atomic_inc(&which->num_low_pending); + + mb(); + + if(!old_pending && wakeup) + { + wakeup_litirqd_locked(which); /* wake up the klitirqd */ + } + + //__dump_state(which, "___litmus_tasklet_schedule: after queuing"); + + raw_spin_unlock_irqrestore(&which->lock, flags); +} + +int __litmus_tasklet_schedule(struct tasklet_struct *t, unsigned int k_id) +{ + int ret = 0; /* assume failure */ + if(unlikely((t->owner == NULL) || !is_realtime(t->owner))) + { + TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); + BUG(); + } + + if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) + { + TRACE("%s: No klitirqd_th%d!\n", __FUNCTION__, k_id); + BUG(); + } + + if(likely(!klitirqds[k_id].terminating)) + { + /* Can't accept tasklets while we're processing a workqueue + because they're handled by the same thread. This case is + very RARE. + + TODO: Use a separate thread for work objects!!!!!! + */ + if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0)) + { + ret = 1; + ___litmus_tasklet_schedule(t, &klitirqds[k_id], 1); + } + else + { + TRACE("%s: rejected tasklet because of pending work.\n", + __FUNCTION__); + } + } + return(ret); +} + +EXPORT_SYMBOL(__litmus_tasklet_schedule); + + +static void ___litmus_tasklet_hi_schedule(struct tasklet_struct *t, + struct klitirqd_info *which, + int wakeup) +{ + unsigned long flags; + u32 old_pending; + + t->next = NULL; + + raw_spin_lock_irqsave(&which->lock, flags); + + *(which->pending_tasklets_hi.tail) = t; + which->pending_tasklets_hi.tail = &t->next; + + old_pending = which->pending; + which->pending |= LIT_TASKLET_HI; + + atomic_inc(&which->num_hi_pending); + + mb(); + + if(!old_pending && wakeup) + { + wakeup_litirqd_locked(which); /* wake up the klitirqd */ + } + + raw_spin_unlock_irqrestore(&which->lock, flags); +} + +int __litmus_tasklet_hi_schedule(struct tasklet_struct *t, unsigned int k_id) +{ + int ret = 0; /* assume failure */ + if(unlikely((t->owner == NULL) || !is_realtime(t->owner))) + { + TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); + BUG(); + } + + if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) + { + TRACE("%s: No klitirqd_th%d!\n", __FUNCTION__, k_id); + BUG(); + } + + if(unlikely(!klitirqd_is_ready())) + { + TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id); + BUG(); + } + + if(likely(!klitirqds[k_id].terminating)) + { + if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0)) + { + ret = 1; + ___litmus_tasklet_hi_schedule(t, &klitirqds[k_id], 1); + } + else + { + TRACE("%s: rejected tasklet because of pending work.\n", + __FUNCTION__); + } + } + return(ret); +} + +EXPORT_SYMBOL(__litmus_tasklet_hi_schedule); + + +int __litmus_tasklet_hi_schedule_first(struct tasklet_struct *t, unsigned int k_id) +{ + int ret = 0; /* assume failure */ + u32 old_pending; + + BUG_ON(!irqs_disabled()); + + if(unlikely((t->owner == NULL) || !is_realtime(t->owner))) + { + TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); + BUG(); + } + + if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) + { + TRACE("%s: No klitirqd_th%u!\n", __FUNCTION__, k_id); + BUG(); + } + + if(unlikely(!klitirqd_is_ready())) + { + TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id); + BUG(); + } + + if(likely(!klitirqds[k_id].terminating)) + { + raw_spin_lock(&klitirqds[k_id].lock); + + if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0)) + { + ret = 1; // success! + + t->next = klitirqds[k_id].pending_tasklets_hi.head; + klitirqds[k_id].pending_tasklets_hi.head = t; + + old_pending = klitirqds[k_id].pending; + klitirqds[k_id].pending |= LIT_TASKLET_HI; + + atomic_inc(&klitirqds[k_id].num_hi_pending); + + mb(); + + if(!old_pending) + wakeup_litirqd_locked(&klitirqds[k_id]); /* wake up the klitirqd */ + } + else + { + TRACE("%s: rejected tasklet because of pending work.\n", + __FUNCTION__); + } + + raw_spin_unlock(&klitirqds[k_id].lock); + } + return(ret); +} + +EXPORT_SYMBOL(__litmus_tasklet_hi_schedule_first); + + + +static void ___litmus_schedule_work(struct work_struct *w, + struct klitirqd_info *which, + int wakeup) +{ + unsigned long flags; + u32 old_pending; + + raw_spin_lock_irqsave(&which->lock, flags); + + work_pending(w); + list_add_tail(&w->entry, &which->worklist); + + old_pending = which->pending; + which->pending |= LIT_WORK; + + atomic_inc(&which->num_work_pending); + + mb(); + + if(!old_pending && wakeup) + { + wakeup_litirqd_locked(which); /* wakeup the klitirqd */ + } + + raw_spin_unlock_irqrestore(&which->lock, flags); +} + +int __litmus_schedule_work(struct work_struct *w, unsigned int k_id) +{ + int ret = 1; /* assume success */ + if(unlikely(w->owner == NULL) || !is_realtime(w->owner)) + { + TRACE("%s: No owner associated with this work object!\n", __FUNCTION__); + BUG(); + } + + if(unlikely(k_id >= NR_LITMUS_SOFTIRQD)) + { + TRACE("%s: No klitirqd_th%u!\n", k_id); + BUG(); + } + + if(unlikely(!klitirqd_is_ready())) + { + TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id); + BUG(); + } + + if(likely(!klitirqds[k_id].terminating)) + ___litmus_schedule_work(w, &klitirqds[k_id], 1); + else + ret = 0; + return(ret); +} +EXPORT_SYMBOL(__litmus_schedule_work); + + +static int set_klitirqd_sem_status(unsigned long stat) +{ + TRACE_CUR("SETTING STATUS FROM %d TO %d\n", + atomic_read(&tsk_rt(current)->klitirqd_sem_stat), + stat); + atomic_set(&tsk_rt(current)->klitirqd_sem_stat, stat); + //mb(); + + return(0); +} + +static int set_klitirqd_sem_status_if_not_held(unsigned long stat) +{ + if(atomic_read(&tsk_rt(current)->klitirqd_sem_stat) != HELD) + { + return(set_klitirqd_sem_status(stat)); + } + return(-1); +} + + +void __down_and_reset_and_set_stat(struct task_struct* t, + enum klitirqd_sem_status to_reset, + enum klitirqd_sem_status to_set, + struct mutex* sem) +{ +#if 0 + struct rt_param* param = container_of(sem, struct rt_param, klitirqd_sem); + struct task_struct* task = container_of(param, struct task_struct, rt_param); + + TRACE_CUR("%s: entered. Locking semaphore of %s/%d\n", + __FUNCTION__, task->comm, task->pid); +#endif + + mutex_lock_sfx(sem, + set_klitirqd_sem_status_if_not_held, to_reset, + set_klitirqd_sem_status, to_set); +#if 0 + TRACE_CUR("%s: exiting. Have semaphore of %s/%d\n", + __FUNCTION__, task->comm, task->pid); +#endif +} + +void down_and_set_stat(struct task_struct* t, + enum klitirqd_sem_status to_set, + struct mutex* sem) +{ +#if 0 + struct rt_param* param = container_of(sem, struct rt_param, klitirqd_sem); + struct task_struct* task = container_of(param, struct task_struct, rt_param); + + TRACE_CUR("%s: entered. Locking semaphore of %s/%d\n", + __FUNCTION__, task->comm, task->pid); +#endif + + mutex_lock_sfx(sem, + NULL, 0, + set_klitirqd_sem_status, to_set); + +#if 0 + TRACE_CUR("%s: exiting. Have semaphore of %s/%d\n", + __FUNCTION__, task->comm, task->pid); +#endif +} + + +void up_and_set_stat(struct task_struct* t, + enum klitirqd_sem_status to_set, + struct mutex* sem) +{ +#if 0 + struct rt_param* param = container_of(sem, struct rt_param, klitirqd_sem); + struct task_struct* task = container_of(param, struct task_struct, rt_param); + + TRACE_CUR("%s: entered. Unlocking semaphore of %s/%d\n", + __FUNCTION__, + task->comm, task->pid); +#endif + + mutex_unlock_sfx(sem, NULL, 0, + set_klitirqd_sem_status, to_set); + +#if 0 + TRACE_CUR("%s: exiting. Unlocked semaphore of %s/%d\n", + __FUNCTION__, + task->comm, task->pid); +#endif +} + + + +void release_klitirqd_lock(struct task_struct* t) +{ + if(is_realtime(t) && (atomic_read(&tsk_rt(t)->klitirqd_sem_stat) == HELD)) + { + struct mutex* sem; + struct task_struct* owner = t; + + if(t->state == TASK_RUNNING) + { + TRACE_TASK(t, "NOT giving up klitirqd_sem because we're not blocked!\n"); + return; + } + + if(likely(!tsk_rt(t)->is_proxy_thread)) + { + sem = &tsk_rt(t)->klitirqd_sem; + } + else + { + unsigned int k_id = klitirqd_id(t); + owner = klitirqds[k_id].current_owner; + + BUG_ON(t != klitirqds[k_id].klitirqd); + + if(likely(owner)) + { + sem = &tsk_rt(owner)->klitirqd_sem; + } + else + { + BUG(); + + // We had the rug pulled out from under us. Abort attempt + // to reacquire the lock since our client no longer needs us. + TRACE_CUR("HUH?! How did this happen?\n"); + atomic_set(&tsk_rt(t)->klitirqd_sem_stat, NOT_HELD); + return; + } + } + + //TRACE_CUR("Releasing semaphore of %s/%d...\n", owner->comm, owner->pid); + up_and_set_stat(t, NEED_TO_REACQUIRE, sem); + //TRACE_CUR("Semaphore of %s/%d released!\n", owner->comm, owner->pid); + } + /* + else if(is_realtime(t)) + { + TRACE_CUR("%s: Nothing to do. Stat = %d\n", __FUNCTION__, tsk_rt(t)->klitirqd_sem_stat); + } + */ +} + +int reacquire_klitirqd_lock(struct task_struct* t) +{ + int ret = 0; + + if(is_realtime(t) && (atomic_read(&tsk_rt(t)->klitirqd_sem_stat) == NEED_TO_REACQUIRE)) + { + struct mutex* sem; + struct task_struct* owner = t; + + if(likely(!tsk_rt(t)->is_proxy_thread)) + { + sem = &tsk_rt(t)->klitirqd_sem; + } + else + { + unsigned int k_id = klitirqd_id(t); + //struct task_struct* owner = klitirqds[k_id].current_owner; + owner = klitirqds[k_id].current_owner; + + BUG_ON(t != klitirqds[k_id].klitirqd); + + if(likely(owner)) + { + sem = &tsk_rt(owner)->klitirqd_sem; + } + else + { + // We had the rug pulled out from under us. Abort attempt + // to reacquire the lock since our client no longer needs us. + TRACE_CUR("No longer needs to reacquire klitirqd_sem!\n"); + atomic_set(&tsk_rt(t)->klitirqd_sem_stat, NOT_HELD); + return(0); + } + } + + //TRACE_CUR("Trying to reacquire semaphore of %s/%d\n", owner->comm, owner->pid); + __down_and_reset_and_set_stat(t, REACQUIRING, HELD, sem); + //TRACE_CUR("Reacquired semaphore %s/%d\n", owner->comm, owner->pid); + } + /* + else if(is_realtime(t)) + { + TRACE_CUR("%s: Nothing to do. Stat = %d\n", __FUNCTION__, tsk_rt(t)->klitirqd_sem_stat); + } + */ + + return(ret); +} + diff --git a/litmus/locking.c b/litmus/locking.c new file mode 100644 index 0000000..718a5a3 --- /dev/null +++ b/litmus/locking.c @@ -0,0 +1,524 @@ +#include + +#ifdef CONFIG_LITMUS_LOCKING + +#include +#include +#include + +#ifdef CONFIG_LITMUS_DGL_SUPPORT +#include +#endif + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) +#include +#endif + +static int create_generic_lock(void** obj_ref, obj_type_t type, void* __user arg); +static int open_generic_lock(struct od_table_entry* entry, void* __user arg); +static int close_generic_lock(struct od_table_entry* entry); +static void destroy_generic_lock(obj_type_t type, void* sem); + +struct fdso_ops generic_lock_ops = { + .create = create_generic_lock, + .open = open_generic_lock, + .close = close_generic_lock, + .destroy = destroy_generic_lock +}; + +static atomic_t lock_id_gen = ATOMIC_INIT(0); + + +static inline bool is_lock(struct od_table_entry* entry) +{ + return entry->class == &generic_lock_ops; +} + +static inline struct litmus_lock* get_lock(struct od_table_entry* entry) +{ + BUG_ON(!is_lock(entry)); + return (struct litmus_lock*) entry->obj->obj; +} + +static int create_generic_lock(void** obj_ref, obj_type_t type, void* __user arg) +{ + struct litmus_lock* lock; + int err; + + err = litmus->allocate_lock(&lock, type, arg); + if (err == 0) { +#ifdef CONFIG_LITMUS_NESTED_LOCKING + lock->nest.lock = lock; + lock->nest.hp_waiter_eff_prio = NULL; + + INIT_BINHEAP_NODE(&lock->nest.hp_binheap_node); + if(!lock->nest.hp_waiter_ptr) { + TRACE_CUR("BEWARE: hp_waiter_ptr should probably not be NULL in " + "most uses. (exception: IKGLP donors)\n"); + } +#endif + lock->type = type; + lock->ident = atomic_inc_return(&lock_id_gen); + *obj_ref = lock; + } + return err; +} + +static int open_generic_lock(struct od_table_entry* entry, void* __user arg) +{ + struct litmus_lock* lock = get_lock(entry); + if (lock->ops->open) + return lock->ops->open(lock, arg); + else + return 0; /* default: any task can open it */ +} + +static int close_generic_lock(struct od_table_entry* entry) +{ + struct litmus_lock* lock = get_lock(entry); + if (lock->ops->close) + return lock->ops->close(lock); + else + return 0; /* default: closing succeeds */ +} + +static void destroy_generic_lock(obj_type_t type, void* obj) +{ + struct litmus_lock* lock = (struct litmus_lock*) obj; + lock->ops->deallocate(lock); +} + +asmlinkage long sys_litmus_lock(int lock_od) +{ + long err = -EINVAL; + struct od_table_entry* entry; + struct litmus_lock* l; + + TS_LOCK_START; + + entry = get_entry_for_od(lock_od); + if (entry && is_lock(entry)) { + l = get_lock(entry); + //TRACE_CUR("attempts to lock 0x%p\n", l); + TRACE_CUR("attempts to lock %d\n", l->ident); + err = l->ops->lock(l); + } + + /* Note: task my have been suspended or preempted in between! Take + * this into account when computing overheads. */ + TS_LOCK_END; + + return err; +} + +asmlinkage long sys_litmus_unlock(int lock_od) +{ + long err = -EINVAL; + struct od_table_entry* entry; + struct litmus_lock* l; + + TS_UNLOCK_START; + + entry = get_entry_for_od(lock_od); + if (entry && is_lock(entry)) { + l = get_lock(entry); + //TRACE_CUR("attempts to unlock 0x%p\n", l); + TRACE_CUR("attempts to unlock %d\n", l->ident); + err = l->ops->unlock(l); + } + + /* Note: task my have been preempted in between! Take this into + * account when computing overheads. */ + TS_UNLOCK_END; + + return err; +} + +struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq) +{ + wait_queue_t* q; + struct task_struct* t = NULL; + + if (waitqueue_active(wq)) { + q = list_entry(wq->task_list.next, + wait_queue_t, task_list); + t = (struct task_struct*) q->private; + __remove_wait_queue(wq, q); + } + return(t); +} + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + +void print_hp_waiters(struct binheap_node* n, int depth) +{ + struct litmus_lock *l; + struct nested_info *nest; + char padding[81] = " "; + struct task_struct *hp = NULL; + struct task_struct *hp_eff = NULL; + struct task_struct *node_prio = NULL; + + + if(n == NULL) { + TRACE("+-> %p\n", NULL); + return; + } + + nest = binheap_entry(n, struct nested_info, hp_binheap_node); + l = nest->lock; + + if(depth*2 <= 80) + padding[depth*2] = '\0'; + + if(nest->hp_waiter_ptr && *(nest->hp_waiter_ptr)) { + hp = *(nest->hp_waiter_ptr); + + if(tsk_rt(hp)->inh_task) { + hp_eff = tsk_rt(hp)->inh_task; + } + } + + node_prio = nest->hp_waiter_eff_prio; + + TRACE("%s+-> %s/%d [waiter = %s/%d] [waiter's inh = %s/%d] (lock = %d)\n", + padding, + (node_prio) ? node_prio->comm : "nil", + (node_prio) ? node_prio->pid : -1, + (hp) ? hp->comm : "nil", + (hp) ? hp->pid : -1, + (hp_eff) ? hp_eff->comm : "nil", + (hp_eff) ? hp_eff->pid : -1, + l->ident); + + if(n->left) print_hp_waiters(n->left, depth+1); + if(n->right) print_hp_waiters(n->right, depth+1); +} +#endif + + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + +void select_next_lock(dgl_wait_state_t* dgl_wait /*, struct litmus_lock* prev_lock*/) +{ + /* + We pick the next lock in reverse order. This causes inheritance propagation + from locks received earlier to flow in the same direction as regular nested + locking. This might make fine-grain DGL easier in the future. + */ + + BUG_ON(tsk_rt(dgl_wait->task)->blocked_lock); + + //WARN_ON(dgl_wait->locks[dgl_wait->last_primary] != prev_lock); + + // note reverse order + for(dgl_wait->last_primary = dgl_wait->last_primary - 1; + dgl_wait->last_primary >= 0; + --(dgl_wait->last_primary)){ + if(!dgl_wait->locks[dgl_wait->last_primary]->ops->is_owner( + dgl_wait->locks[dgl_wait->last_primary], dgl_wait->task)) { + + tsk_rt(dgl_wait->task)->blocked_lock = + dgl_wait->locks[dgl_wait->last_primary]; + mb(); + + TRACE_CUR("New blocked lock is %d\n", + dgl_wait->locks[dgl_wait->last_primary]->ident); + + break; + } + } +} + +int dgl_wake_up(wait_queue_t *wq_node, unsigned mode, int sync, void *key) +{ + // should never be called. + BUG(); + return 1; +} + +void __waitqueue_dgl_remove_first(wait_queue_head_t *wq, + dgl_wait_state_t** dgl_wait, + struct task_struct **task) +{ + wait_queue_t *q; + + *dgl_wait = NULL; + *task = NULL; + + if (waitqueue_active(wq)) { + q = list_entry(wq->task_list.next, + wait_queue_t, task_list); + + if(q->func == dgl_wake_up) { + *dgl_wait = (dgl_wait_state_t*) q->private; + } + else { + *task = (struct task_struct*) q->private; + } + + __remove_wait_queue(wq, q); + } +} + +void init_dgl_waitqueue_entry(wait_queue_t *wq_node, dgl_wait_state_t* dgl_wait) +{ + init_waitqueue_entry(wq_node, dgl_wait->task); + wq_node->private = dgl_wait; + wq_node->func = dgl_wake_up; +} + + +static long do_litmus_dgl_lock(dgl_wait_state_t *dgl_wait) +{ + int i; + unsigned long irqflags; //, dummyflags; + raw_spinlock_t *dgl_lock = litmus->get_dgl_spinlock(dgl_wait->task); + + BUG_ON(dgl_wait->task != current); + + raw_spin_lock_irqsave(dgl_lock, irqflags); + + + dgl_wait->nr_remaining = dgl_wait->size; + + TRACE_CUR("Locking DGL with size %d\n", dgl_wait->size); + + // try to acquire each lock. enqueue (non-blocking) if it is unavailable. + for(i = 0; i < dgl_wait->size; ++i) { + struct litmus_lock *l = dgl_wait->locks[i]; + + // dgl_lock() must set task state to TASK_UNINTERRUPTIBLE if task blocks. + + if(l->ops->dgl_lock(l, dgl_wait, &dgl_wait->wq_nodes[i])) { + --(dgl_wait->nr_remaining); + TRACE_CUR("Acquired lock %d immediatly.\n", l->ident); + } + } + + if(dgl_wait->nr_remaining == 0) { + // acquired entire group immediatly + TRACE_CUR("Acquired all locks in DGL immediatly!\n"); + } + else { + + TRACE_CUR("As many as %d locks in DGL are pending. Suspending.\n", + dgl_wait->nr_remaining); + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) + // KLUDGE: don't count this suspension as time in the critical gpu + // critical section + if(tsk_rt(dgl_wait->task)->held_gpus) { + tsk_rt(dgl_wait->task)->suspend_gpu_tracker_on_block = 1; + } +#endif + + // note reverse order. see comments in select_next_lock for reason. + for(i = dgl_wait->size - 1; i >= 0; --i) { + struct litmus_lock *l = dgl_wait->locks[i]; + if(!l->ops->is_owner(l, dgl_wait->task)) { // double-check to be thread safe + + TRACE_CUR("Activating priority inheritance on lock %d\n", + l->ident); + + TS_DGL_LOCK_SUSPEND; + + l->ops->enable_priority(l, dgl_wait); + dgl_wait->last_primary = i; + + TRACE_CUR("Suspending for lock %d\n", l->ident); + + raw_spin_unlock_irqrestore(dgl_lock, irqflags); // free dgl_lock before suspending + + schedule(); // suspend!!! + + TS_DGL_LOCK_RESUME; + + TRACE_CUR("Woken up from DGL suspension.\n"); + + goto all_acquired; // we should hold all locks when we wake up. + } + } + + TRACE_CUR("Didn't have to suspend after all, but calling schedule() anyway.\n"); + //BUG(); + } + + raw_spin_unlock_irqrestore(dgl_lock, irqflags); + +all_acquired: + + // FOR SANITY CHECK FOR TESTING +// for(i = 0; i < dgl_wait->size; ++i) { +// struct litmus_lock *l = dgl_wait->locks[i]; +// BUG_ON(!l->ops->is_owner(l, dgl_wait->task)); +// } + + TRACE_CUR("Acquired entire DGL\n"); + + return 0; +} + +static int supports_dgl(struct litmus_lock *l) +{ + struct litmus_lock_ops* ops = l->ops; + + return (ops->dgl_lock && + ops->is_owner && + ops->enable_priority); +} + +asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size) +{ + struct task_struct *t = current; + long err = -EINVAL; + int dgl_ods[MAX_DGL_SIZE]; + int i; + + dgl_wait_state_t dgl_wait_state; // lives on the stack until all resources in DGL are held. + + if(dgl_size > MAX_DGL_SIZE || dgl_size < 1) + goto out; + + if(!access_ok(VERIFY_READ, usr_dgl_ods, dgl_size*(sizeof(int)))) + goto out; + + if(__copy_from_user(&dgl_ods, usr_dgl_ods, dgl_size*(sizeof(int)))) + goto out; + + if (!is_realtime(t)) { + err = -EPERM; + goto out; + } + + for(i = 0; i < dgl_size; ++i) { + struct od_table_entry *entry = get_entry_for_od(dgl_ods[i]); + if(entry && is_lock(entry)) { + dgl_wait_state.locks[i] = get_lock(entry); + if(!supports_dgl(dgl_wait_state.locks[i])) { + TRACE_CUR("Lock %d does not support all required DGL operations.\n", + dgl_wait_state.locks[i]->ident); + goto out; + } + } + else { + TRACE_CUR("Invalid lock identifier\n"); + goto out; + } + } + + dgl_wait_state.task = t; + dgl_wait_state.size = dgl_size; + + TS_DGL_LOCK_START; + err = do_litmus_dgl_lock(&dgl_wait_state); + + /* Note: task my have been suspended or preempted in between! Take + * this into account when computing overheads. */ + TS_DGL_LOCK_END; + +out: + return err; +} + +static long do_litmus_dgl_unlock(struct litmus_lock* dgl_locks[], int dgl_size) +{ + int i; + long err = 0; + + TRACE_CUR("Unlocking a DGL of %d size\n", dgl_size); + + for(i = dgl_size - 1; i >= 0; --i) { // unlock in reverse order + + struct litmus_lock *l = dgl_locks[i]; + long tmp_err; + + TRACE_CUR("Unlocking lock %d of DGL.\n", l->ident); + + tmp_err = l->ops->unlock(l); + + if(tmp_err) { + TRACE_CUR("There was an error unlocking %d: %d.\n", l->ident, tmp_err); + err = tmp_err; + } + } + + TRACE_CUR("DGL unlocked. err = %d\n", err); + + return err; +} + +asmlinkage long sys_litmus_dgl_unlock(void* __user usr_dgl_ods, int dgl_size) +{ + long err = -EINVAL; + int dgl_ods[MAX_DGL_SIZE]; + struct od_table_entry* entry; + int i; + + struct litmus_lock* dgl_locks[MAX_DGL_SIZE]; + + if(dgl_size > MAX_DGL_SIZE || dgl_size < 1) + goto out; + + if(!access_ok(VERIFY_READ, usr_dgl_ods, dgl_size*(sizeof(int)))) + goto out; + + if(__copy_from_user(&dgl_ods, usr_dgl_ods, dgl_size*(sizeof(int)))) + goto out; + + for(i = 0; i < dgl_size; ++i) { + entry = get_entry_for_od(dgl_ods[i]); + if(entry && is_lock(entry)) { + dgl_locks[i] = get_lock(entry); + if(!supports_dgl(dgl_locks[i])) { + TRACE_CUR("Lock %d does not support all required DGL operations.\n", + dgl_locks[i]->ident); + goto out; + } + } + else { + TRACE_CUR("Invalid lock identifier\n"); + goto out; + } + } + + TS_DGL_UNLOCK_START; + err = do_litmus_dgl_unlock(dgl_locks, dgl_size); + + /* Note: task my have been suspended or preempted in between! Take + * this into account when computing overheads. */ + TS_DGL_UNLOCK_END; + +out: + return err; +} + +#else // CONFIG_LITMUS_DGL_SUPPORT + +asmlinkage long sys_litmus_dgl_lock(void* __user usr_dgl_ods, int dgl_size) +{ + return -ENOSYS; +} + +asmlinkage long sys_litmus_dgl_unlock(void* __user usr_dgl_ods, int dgl_size) +{ + return -ENOSYS; +} + +#endif + +#else // CONFIG_LITMUS_LOCKING + +struct fdso_ops generic_lock_ops = {}; + +asmlinkage long sys_litmus_lock(int sem_od) +{ + return -ENOSYS; +} + +asmlinkage long sys_litmus_unlock(int sem_od) +{ + return -ENOSYS; +} + +#endif diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c new file mode 100644 index 0000000..4b86a50 --- /dev/null +++ b/litmus/nvidia_info.c @@ -0,0 +1,597 @@ +#include +#include +#include + +#include +#include +#include + +#include + +#include + +typedef unsigned char NvV8; /* "void": enumerated or multiple fields */ +typedef unsigned short NvV16; /* "void": enumerated or multiple fields */ +typedef unsigned char NvU8; /* 0 to 255 */ +typedef unsigned short NvU16; /* 0 to 65535 */ +typedef signed char NvS8; /* -128 to 127 */ +typedef signed short NvS16; /* -32768 to 32767 */ +typedef float NvF32; /* IEEE Single Precision (S1E8M23) */ +typedef double NvF64; /* IEEE Double Precision (S1E11M52) */ +typedef unsigned int NvV32; /* "void": enumerated or multiple fields */ +typedef unsigned int NvU32; /* 0 to 4294967295 */ +typedef unsigned long long NvU64; /* 0 to 18446744073709551615 */ +typedef union +{ + volatile NvV8 Reg008[1]; + volatile NvV16 Reg016[1]; + volatile NvV32 Reg032[1]; +} litmus_nv_hwreg_t, * litmus_nv_phwreg_t; + +typedef struct +{ + NvU64 address; + NvU64 size; + NvU32 offset; + NvU32 *map; + litmus_nv_phwreg_t map_u; +} litmus_nv_aperture_t; + +typedef struct +{ + void *priv; /* private data */ + void *os_state; /* os-specific device state */ + + int rmInitialized; + int flags; + + /* PCI config info */ + NvU32 domain; + NvU16 bus; + NvU16 slot; + NvU16 vendor_id; + NvU16 device_id; + NvU16 subsystem_id; + NvU32 gpu_id; + void *handle; + + NvU32 pci_cfg_space[16]; + + /* physical characteristics */ + litmus_nv_aperture_t bars[3]; + litmus_nv_aperture_t *regs; + litmus_nv_aperture_t *fb, ud; + litmus_nv_aperture_t agp; + + NvU32 interrupt_line; + + NvU32 agp_config; + NvU32 agp_status; + + NvU32 primary_vga; + + NvU32 sim_env; + + NvU32 rc_timer_enabled; + + /* list of events allocated for this device */ + void *event_list; + + void *kern_mappings; + +} litmus_nv_state_t; + +typedef struct work_struct litmus_nv_task_t; + +typedef struct litmus_nv_work_s { + litmus_nv_task_t task; + void *data; +} litmus_nv_work_t; + +typedef struct litmus_nv_linux_state_s { + litmus_nv_state_t nv_state; + atomic_t usage_count; + + struct pci_dev *dev; + void *agp_bridge; + void *alloc_queue; + + void *timer_sp; + void *isr_sp; + void *pci_cfgchk_sp; + void *isr_bh_sp; + +#ifdef CONFIG_CUDA_4_0 + char registry_keys[512]; +#endif + + /* keep track of any pending bottom halfes */ + struct tasklet_struct tasklet; + litmus_nv_work_t work; + + /* get a timer callback every second */ + struct timer_list rc_timer; + + /* lock for linux-specific data, not used by core rm */ + struct semaphore ldata_lock; + + /* lock for linux-specific alloc queue */ + struct semaphore at_lock; + +#if 0 +#if defined(NV_USER_MAP) + /* list of user mappings */ + struct nv_usermap_s *usermap_list; + + /* lock for VMware-specific mapping list */ + struct semaphore mt_lock; +#endif /* defined(NV_USER_MAP) */ +#if defined(NV_PM_SUPPORT_OLD_STYLE_APM) + void *apm_nv_dev; +#endif +#endif + + NvU32 device_num; + struct litmus_nv_linux_state_s *next; +} litmus_nv_linux_state_t; + +void dump_nvidia_info(const struct tasklet_struct *t) +{ + litmus_nv_state_t* nvstate = NULL; + litmus_nv_linux_state_t* linuxstate = NULL; + struct pci_dev* pci = NULL; + + nvstate = (litmus_nv_state_t*)(t->data); + + if(nvstate) + { + TRACE("NV State:\n" + "\ttasklet ptr = %p\n" + "\tstate ptr = %p\n" + "\tprivate data ptr = %p\n" + "\tos state ptr = %p\n" + "\tdomain = %u\n" + "\tbus = %u\n" + "\tslot = %u\n" + "\tvender_id = %u\n" + "\tdevice_id = %u\n" + "\tsubsystem_id = %u\n" + "\tgpu_id = %u\n" + "\tinterrupt_line = %u\n", + t, + nvstate, + nvstate->priv, + nvstate->os_state, + nvstate->domain, + nvstate->bus, + nvstate->slot, + nvstate->vendor_id, + nvstate->device_id, + nvstate->subsystem_id, + nvstate->gpu_id, + nvstate->interrupt_line); + + linuxstate = container_of(nvstate, litmus_nv_linux_state_t, nv_state); + } + else + { + TRACE("INVALID NVSTATE????\n"); + } + + if(linuxstate) + { + int ls_offset = (void*)(&(linuxstate->device_num)) - (void*)(linuxstate); + int ns_offset_raw = (void*)(&(linuxstate->device_num)) - (void*)(&(linuxstate->nv_state)); + int ns_offset_desired = (void*)(&(linuxstate->device_num)) - (void*)(nvstate); + + + TRACE("LINUX NV State:\n" + "\tlinux nv state ptr: %p\n" + "\taddress of tasklet: %p\n" + "\taddress of work: %p\n" + "\tusage_count: %d\n" + "\tdevice_num: %u\n" + "\ttasklet addr == this tasklet: %d\n" + "\tpci: %p\n", + linuxstate, + &(linuxstate->tasklet), + &(linuxstate->work), + atomic_read(&(linuxstate->usage_count)), + linuxstate->device_num, + (t == &(linuxstate->tasklet)), + linuxstate->dev); + + pci = linuxstate->dev; + + TRACE("Offsets:\n" + "\tOffset from LinuxState: %d, %x\n" + "\tOffset from NVState: %d, %x\n" + "\tOffset from parameter: %d, %x\n" + "\tdevice_num: %u\n", + ls_offset, ls_offset, + ns_offset_raw, ns_offset_raw, + ns_offset_desired, ns_offset_desired, + *((u32*)((void*)nvstate + ns_offset_desired))); + } + else + { + TRACE("INVALID LINUXNVSTATE?????\n"); + } + +#if 0 + if(pci) + { + TRACE("PCI DEV Info:\n" + "pci device ptr: %p\n" + "\tdevfn = %d\n" + "\tvendor = %d\n" + "\tdevice = %d\n" + "\tsubsystem_vendor = %d\n" + "\tsubsystem_device = %d\n" + "\tslot # = %d\n", + pci, + pci->devfn, + pci->vendor, + pci->device, + pci->subsystem_vendor, + pci->subsystem_device, + pci->slot->number); + } + else + { + TRACE("INVALID PCIDEV PTR?????\n"); + } +#endif +} + +static struct module* nvidia_mod = NULL; +int init_nvidia_info(void) +{ + mutex_lock(&module_mutex); + nvidia_mod = find_module("nvidia"); + mutex_unlock(&module_mutex); + if(nvidia_mod != NULL) + { + TRACE("%s : Found NVIDIA module. Core Code: %p to %p\n", __FUNCTION__, + (void*)(nvidia_mod->module_core), + (void*)(nvidia_mod->module_core) + nvidia_mod->core_size); + init_nv_device_reg(); + return(0); + } + else + { + TRACE("%s : Could not find NVIDIA module! Loaded?\n", __FUNCTION__); + return(-1); + } +} + +void shutdown_nvidia_info(void) +{ + nvidia_mod = NULL; + mb(); +} + +/* works with pointers to static data inside the module too. */ +int is_nvidia_func(void* func_addr) +{ + int ret = 0; + if(nvidia_mod) + { + ret = within_module_core((long unsigned int)func_addr, nvidia_mod); + /* + if(ret) + { + TRACE("%s : %p is in NVIDIA module: %d\n", + __FUNCTION__, func_addr, ret); + }*/ + } + + return(ret); +} + +u32 get_tasklet_nv_device_num(const struct tasklet_struct *t) +{ + // life is too short to use hard-coded offsets. update this later. + litmus_nv_state_t* nvstate = (litmus_nv_state_t*)(t->data); + litmus_nv_linux_state_t* linuxstate = container_of(nvstate, litmus_nv_linux_state_t, nv_state); + + BUG_ON(linuxstate->device_num >= NV_DEVICE_NUM); + + return(linuxstate->device_num); + + //int DEVICE_NUM_OFFSET = (void*)(&(linuxstate->device_num)) - (void*)(nvstate); + +#if 0 + // offset determined though observed behavior of the NV driver. + //const int DEVICE_NUM_OFFSET = 0x480; // CUDA 4.0 RC1 + //const int DEVICE_NUM_OFFSET = 0x510; // CUDA 4.0 RC2 + + void* state = (void*)(t->data); + void* device_num_ptr = state + DEVICE_NUM_OFFSET; + + //dump_nvidia_info(t); + return(*((u32*)device_num_ptr)); +#endif +} + +u32 get_work_nv_device_num(const struct work_struct *t) +{ + // offset determined though observed behavior of the NV driver. + const int DEVICE_NUM_OFFSET = sizeof(struct work_struct); + void* state = (void*)(t); + void** device_num_ptr = state + DEVICE_NUM_OFFSET; + return(*((u32*)(*device_num_ptr))); +} + + +typedef struct { + raw_spinlock_t lock; + int nr_owners; + struct task_struct* max_prio_owner; + struct task_struct* owners[NV_MAX_SIMULT_USERS]; +}nv_device_registry_t; + +static nv_device_registry_t NV_DEVICE_REG[NV_DEVICE_NUM]; + +int init_nv_device_reg(void) +{ + int i; + + memset(NV_DEVICE_REG, 0, sizeof(NV_DEVICE_REG)); + + for(i = 0; i < NV_DEVICE_NUM; ++i) + { + raw_spin_lock_init(&NV_DEVICE_REG[i].lock); + } + + return(1); +} + +/* use to get nv_device_id by given owner. + (if return -1, can't get the assocaite device id)*/ +/* +int get_nv_device_id(struct task_struct* owner) +{ + int i; + if(!owner) + { + return(-1); + } + for(i = 0; i < NV_DEVICE_NUM; ++i) + { + if(NV_DEVICE_REG[i].device_owner == owner) + return(i); + } + return(-1); +} +*/ + +static struct task_struct* find_hp_owner(nv_device_registry_t *reg, struct task_struct *skip) { + int i; + struct task_struct *found = NULL; + for(i = 0; i < reg->nr_owners; ++i) { + if(reg->owners[i] && reg->owners[i] != skip && litmus->compare(reg->owners[i], found)) { + found = reg->owners[i]; + } + } + return found; +} + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD +void pai_check_priority_increase(struct task_struct *t, int reg_device_id) +{ + unsigned long flags; + nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id]; + + if(reg->max_prio_owner != t) { + + raw_spin_lock_irqsave(®->lock, flags); + + if(reg->max_prio_owner != t) { + if(litmus->compare(t, reg->max_prio_owner)) { + litmus->change_prio_pai_tasklet(reg->max_prio_owner, t); + reg->max_prio_owner = t; + } + } + + raw_spin_unlock_irqrestore(®->lock, flags); + } +} + + +void pai_check_priority_decrease(struct task_struct *t, int reg_device_id) +{ + unsigned long flags; + nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id]; + + if(reg->max_prio_owner == t) { + + raw_spin_lock_irqsave(®->lock, flags); + + if(reg->max_prio_owner == t) { + reg->max_prio_owner = find_hp_owner(reg, NULL); + if(reg->max_prio_owner != t) { + litmus->change_prio_pai_tasklet(t, reg->max_prio_owner); + } + } + + raw_spin_unlock_irqrestore(®->lock, flags); + } +} +#endif + +static int __reg_nv_device(int reg_device_id, struct task_struct *t) +{ + int ret = 0; + int i; + struct task_struct *old_max = NULL; + unsigned long flags; + nv_device_registry_t *reg = &NV_DEVICE_REG[reg_device_id]; + + if(test_bit(reg_device_id, &tsk_rt(t)->held_gpus)) { + // TODO: check if taks is already registered. + return ret; // assume already registered. + } + + + raw_spin_lock_irqsave(®->lock, flags); + + if(reg->nr_owners < NV_MAX_SIMULT_USERS) { + TRACE_TASK(t, "registers GPU %d\n", reg_device_id); + for(i = 0; i < NV_MAX_SIMULT_USERS; ++i) { + if(reg->owners[i] == NULL) { + reg->owners[i] = t; + + //if(edf_higher_prio(t, reg->max_prio_owner)) { + if(litmus->compare(t, reg->max_prio_owner)) { + old_max = reg->max_prio_owner; + reg->max_prio_owner = t; + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + litmus->change_prio_pai_tasklet(old_max, t); +#endif + } + +#ifdef CONFIG_LITMUS_SOFTIRQD + down_and_set_stat(t, HELD, &tsk_rt(t)->klitirqd_sem); +#endif + ++(reg->nr_owners); + + break; + } + } + } + else + { + TRACE_CUR("%s: device %d is already in use!\n", __FUNCTION__, reg_device_id); + //ret = -EBUSY; + } + + raw_spin_unlock_irqrestore(®->lock, flags); + + __set_bit(reg_device_id, &tsk_rt(t)->held_gpus); + + return(ret); +} + +static int __clear_reg_nv_device(int de_reg_device_id, struct task_struct *t) +{ + int ret = 0; + int i; + unsigned long flags; + nv_device_registry_t *reg = &NV_DEVICE_REG[de_reg_device_id]; + +#ifdef CONFIG_LITMUS_SOFTIRQD + struct task_struct* klitirqd_th = get_klitirqd(de_reg_device_id); +#endif + + if(!test_bit(de_reg_device_id, &tsk_rt(t)->held_gpus)) { + return ret; + } + + raw_spin_lock_irqsave(®->lock, flags); + + TRACE_TASK(t, "unregisters GPU %d\n", de_reg_device_id); + + for(i = 0; i < NV_MAX_SIMULT_USERS; ++i) { + if(reg->owners[i] == t) { +#ifdef CONFIG_LITMUS_SOFTIRQD + flush_pending(klitirqd_th, t); +#endif + if(reg->max_prio_owner == t) { + reg->max_prio_owner = find_hp_owner(reg, t); +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + litmus->change_prio_pai_tasklet(t, reg->max_prio_owner); +#endif + } + +#ifdef CONFIG_LITMUS_SOFTIRQD + up_and_set_stat(t, NOT_HELD, &tsk_rt(t)->klitirqd_sem); +#endif + + reg->owners[i] = NULL; + --(reg->nr_owners); + + break; + } + } + + raw_spin_unlock_irqrestore(®->lock, flags); + + __clear_bit(de_reg_device_id, &tsk_rt(t)->held_gpus); + + return(ret); +} + + +int reg_nv_device(int reg_device_id, int reg_action, struct task_struct *t) +{ + int ret; + + if((reg_device_id < NV_DEVICE_NUM) && (reg_device_id >= 0)) + { + if(reg_action) + ret = __reg_nv_device(reg_device_id, t); + else + ret = __clear_reg_nv_device(reg_device_id, t); + } + else + { + ret = -ENODEV; + } + + return(ret); +} + +/* use to get the owner of nv_device_id. */ +struct task_struct* get_nv_max_device_owner(u32 target_device_id) +{ + struct task_struct *owner = NULL; + BUG_ON(target_device_id >= NV_DEVICE_NUM); + owner = NV_DEVICE_REG[target_device_id].max_prio_owner; + return(owner); +} + +void lock_nv_registry(u32 target_device_id, unsigned long* flags) +{ + BUG_ON(target_device_id >= NV_DEVICE_NUM); + + if(in_interrupt()) + TRACE("Locking registry for %d.\n", target_device_id); + else + TRACE_CUR("Locking registry for %d.\n", target_device_id); + + raw_spin_lock_irqsave(&NV_DEVICE_REG[target_device_id].lock, *flags); +} + +void unlock_nv_registry(u32 target_device_id, unsigned long* flags) +{ + BUG_ON(target_device_id >= NV_DEVICE_NUM); + + if(in_interrupt()) + TRACE("Unlocking registry for %d.\n", target_device_id); + else + TRACE_CUR("Unlocking registry for %d.\n", target_device_id); + + raw_spin_unlock_irqrestore(&NV_DEVICE_REG[target_device_id].lock, *flags); +} + + +//void increment_nv_int_count(u32 device) +//{ +// unsigned long flags; +// struct task_struct* owner; +// +// lock_nv_registry(device, &flags); +// +// owner = NV_DEVICE_REG[device].device_owner; +// if(owner) +// { +// atomic_inc(&tsk_rt(owner)->nv_int_count); +// } +// +// unlock_nv_registry(device, &flags); +//} +//EXPORT_SYMBOL(increment_nv_int_count); + + diff --git a/litmus/preempt.c b/litmus/preempt.c new file mode 100644 index 0000000..28368d5 --- /dev/null +++ b/litmus/preempt.c @@ -0,0 +1,138 @@ +#include + +#include +#include + +/* The rescheduling state of each processor. + */ +DEFINE_PER_CPU_SHARED_ALIGNED(atomic_t, resched_state); + +void sched_state_will_schedule(struct task_struct* tsk) +{ + /* Litmus hack: we only care about processor-local invocations of + * set_tsk_need_resched(). We can't reliably set the flag remotely + * since it might race with other updates to the scheduling state. We + * can't rely on the runqueue lock protecting updates to the sched + * state since processors do not acquire the runqueue locks for all + * updates to the sched state (to avoid acquiring two runqueue locks at + * the same time). Further, if tsk is residing on a remote processor, + * then that processor doesn't actually know yet that it is going to + * reschedule; it still must receive an IPI (unless a local invocation + * races). + */ + if (likely(task_cpu(tsk) == smp_processor_id())) { + VERIFY_SCHED_STATE(TASK_SCHEDULED | SHOULD_SCHEDULE | TASK_PICKED | WILL_SCHEDULE); + if (is_in_sched_state(TASK_PICKED | PICKED_WRONG_TASK)) + set_sched_state(PICKED_WRONG_TASK); + else + set_sched_state(WILL_SCHEDULE); + } else + /* Litmus tasks should never be subject to a remote + * set_tsk_need_resched(). */ + BUG_ON(is_realtime(tsk)); + +#ifdef CONFIG_PREEMPT_STATE_TRACE + TRACE_TASK(tsk, "set_tsk_need_resched() ret:%p\n", + __builtin_return_address(0)); +#endif +} + +/* Called by the IPI handler after another CPU called smp_send_resched(). */ +void sched_state_ipi(void) +{ + /* If the IPI was slow, we might be in any state right now. The IPI is + * only meaningful if we are in SHOULD_SCHEDULE. */ + if (is_in_sched_state(SHOULD_SCHEDULE)) { + /* Cause scheduler to be invoked. + * This will cause a transition to WILL_SCHEDULE. */ + set_tsk_need_resched(current); + /* + TRACE_STATE("IPI -> set_tsk_need_resched(%s/%d)\n", + current->comm, current->pid); + */ + } else { + /* ignore */ + /* + TRACE_STATE("ignoring IPI in state %x (%s)\n", + get_sched_state(), + sched_state_name(get_sched_state())); + */ + } +} + +/* Called by plugins to cause a CPU to reschedule. IMPORTANT: the caller must + * hold the lock that is used to serialize scheduling decisions. */ +void litmus_reschedule(int cpu) +{ + int picked_transition_ok = 0; + int scheduled_transition_ok = 0; + + /* The (remote) CPU could be in any state. */ + + /* The critical states are TASK_PICKED and TASK_SCHEDULED, as the CPU + * is not aware of the need to reschedule at this point. */ + + /* is a context switch in progress? */ + if (cpu_is_in_sched_state(cpu, TASK_PICKED)) + picked_transition_ok = sched_state_transition_on( + cpu, TASK_PICKED, PICKED_WRONG_TASK); + + if (!picked_transition_ok && + cpu_is_in_sched_state(cpu, TASK_SCHEDULED)) { + /* We either raced with the end of the context switch, or the + * CPU was in TASK_SCHEDULED anyway. */ + scheduled_transition_ok = sched_state_transition_on( + cpu, TASK_SCHEDULED, SHOULD_SCHEDULE); + } + + /* If the CPU was in state TASK_SCHEDULED, then we need to cause the + * scheduler to be invoked. */ + if (scheduled_transition_ok) { + if (smp_processor_id() == cpu) + set_tsk_need_resched(current); + else + smp_send_reschedule(cpu); + } + + TRACE_STATE("%s picked-ok:%d sched-ok:%d\n", + __FUNCTION__, + picked_transition_ok, + scheduled_transition_ok); +} + +void litmus_reschedule_local(void) +{ + if (is_in_sched_state(TASK_PICKED)) + set_sched_state(PICKED_WRONG_TASK); + else if (is_in_sched_state(TASK_SCHEDULED | SHOULD_SCHEDULE)) { + set_sched_state(WILL_SCHEDULE); + set_tsk_need_resched(current); + } +} + +#ifdef CONFIG_DEBUG_KERNEL + +void sched_state_plugin_check(void) +{ + if (!is_in_sched_state(TASK_PICKED | PICKED_WRONG_TASK)) { + TRACE("!!!! plugin did not call sched_state_task_picked()!" + "Calling sched_state_task_picked() is mandatory---fix this.\n"); + set_sched_state(TASK_PICKED); + } +} + +#define NAME_CHECK(x) case x: return #x +const char* sched_state_name(int s) +{ + switch (s) { + NAME_CHECK(TASK_SCHEDULED); + NAME_CHECK(SHOULD_SCHEDULE); + NAME_CHECK(WILL_SCHEDULE); + NAME_CHECK(TASK_PICKED); + NAME_CHECK(PICKED_WRONG_TASK); + default: + return "UNKNOWN"; + }; +} + +#endif diff --git a/litmus/rsm_lock.c b/litmus/rsm_lock.c new file mode 100644 index 0000000..75ed87c --- /dev/null +++ b/litmus/rsm_lock.c @@ -0,0 +1,796 @@ +#include +#include + +#include +#include +#include + +//#include + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) +#include +#endif + + +/* caller is responsible for locking */ +static struct task_struct* rsm_mutex_find_hp_waiter(struct rsm_mutex *mutex, + struct task_struct* skip) +{ + wait_queue_t *q; + struct list_head *pos; + struct task_struct *queued = NULL, *found = NULL; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + dgl_wait_state_t *dgl_wait = NULL; +#endif + + list_for_each(pos, &mutex->wait.task_list) { + q = list_entry(pos, wait_queue_t, task_list); + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + if(q->func == dgl_wake_up) { + dgl_wait = (dgl_wait_state_t*) q->private; + if(tsk_rt(dgl_wait->task)->blocked_lock == &mutex->litmus_lock) { + queued = dgl_wait->task; + } + else { + queued = NULL; // skip it. + } + } + else { + queued = (struct task_struct*) q->private; + } +#else + queued = (struct task_struct*) q->private; +#endif + + /* Compare task prios, find high prio task. */ + //if (queued && queued != skip && edf_higher_prio(queued, found)) { + if (queued && queued != skip && litmus->compare(queued, found)) { + found = queued; + } + } + return found; +} + + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + +int rsm_mutex_is_owner(struct litmus_lock *l, struct task_struct *t) +{ + struct rsm_mutex *mutex = rsm_mutex_from_lock(l); + return(mutex->owner == t); +} + +// return 1 if resource was immediatly acquired. +// Assumes mutex->lock is held. +// Must set task state to TASK_UNINTERRUPTIBLE if task blocks. +int rsm_mutex_dgl_lock(struct litmus_lock *l, dgl_wait_state_t* dgl_wait, + wait_queue_t* wq_node) +{ + struct rsm_mutex *mutex = rsm_mutex_from_lock(l); + struct task_struct *t = dgl_wait->task; + + int acquired_immediatly = 0; + + BUG_ON(t != current); + + if (mutex->owner) { + TRACE_TASK(t, "Enqueuing on lock %d.\n", l->ident); + + init_dgl_waitqueue_entry(wq_node, dgl_wait); + + set_task_state(t, TASK_UNINTERRUPTIBLE); + __add_wait_queue_tail_exclusive(&mutex->wait, wq_node); + } else { + TRACE_TASK(t, "Acquired lock %d with no blocking.\n", l->ident); + + /* it's ours now */ + mutex->owner = t; + + raw_spin_lock(&tsk_rt(t)->hp_blocked_tasks_lock); + binheap_add(&l->nest.hp_binheap_node, &tsk_rt(t)->hp_blocked_tasks, + struct nested_info, hp_binheap_node); + raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); + + acquired_immediatly = 1; + } + + return acquired_immediatly; +} + +void rsm_mutex_enable_priority(struct litmus_lock *l, + dgl_wait_state_t* dgl_wait) +{ + struct rsm_mutex *mutex = rsm_mutex_from_lock(l); + struct task_struct *t = dgl_wait->task; + struct task_struct *owner = mutex->owner; + unsigned long flags = 0; // these are unused under DGL coarse-grain locking + + BUG_ON(owner == t); + + tsk_rt(t)->blocked_lock = l; + mb(); + + //if (edf_higher_prio(t, mutex->hp_waiter)) { + if (litmus->compare(t, mutex->hp_waiter)) { + + struct task_struct *old_max_eff_prio; + struct task_struct *new_max_eff_prio; + struct task_struct *new_prio = NULL; + + if(mutex->hp_waiter) + TRACE_TASK(t, "has higher prio than hp_waiter (%s/%d).\n", + mutex->hp_waiter->comm, mutex->hp_waiter->pid); + else + TRACE_TASK(t, "has higher prio than hp_waiter (NIL).\n"); + + raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock); + + old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + mutex->hp_waiter = t; + l->nest.hp_waiter_eff_prio = effective_priority(mutex->hp_waiter); + binheap_decrease(&l->nest.hp_binheap_node, + &tsk_rt(owner)->hp_blocked_tasks); + new_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + if(new_max_eff_prio != old_max_eff_prio) { + TRACE_TASK(t, "is new hp_waiter.\n"); + + if ((effective_priority(owner) == old_max_eff_prio) || + //(__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))){ + (litmus->__compare(new_max_eff_prio, BASE, owner, EFFECTIVE))){ + new_prio = new_max_eff_prio; + } + } + else { + TRACE_TASK(t, "no change in max_eff_prio of heap.\n"); + } + + if(new_prio) { + litmus->nested_increase_prio(owner, new_prio, + &mutex->lock, flags); // unlocks lock. + } + else { + raw_spin_unlock(&tsk_rt(owner)->hp_blocked_tasks_lock); + unlock_fine_irqrestore(&mutex->lock, flags); + } + } + else { + TRACE_TASK(t, "no change in hp_waiter.\n"); + unlock_fine_irqrestore(&mutex->lock, flags); + } +} + +static void select_next_lock_if_primary(struct litmus_lock *l, + dgl_wait_state_t *dgl_wait) +{ + if(tsk_rt(dgl_wait->task)->blocked_lock == l) { + TRACE_CUR("Lock %d in DGL was primary for %s/%d.\n", + l->ident, dgl_wait->task->comm, dgl_wait->task->pid); + tsk_rt(dgl_wait->task)->blocked_lock = NULL; + mb(); + select_next_lock(dgl_wait /*, l*/); // pick the next lock to be blocked on + } + else { + TRACE_CUR("Got lock early! Lock %d in DGL was NOT primary for %s/%d.\n", + l->ident, dgl_wait->task->comm, dgl_wait->task->pid); + } +} +#endif + + + + +int rsm_mutex_lock(struct litmus_lock* l) +{ + struct task_struct *t = current; + struct task_struct *owner; + struct rsm_mutex *mutex = rsm_mutex_from_lock(l); + wait_queue_t wait; + unsigned long flags; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + raw_spinlock_t *dgl_lock; +#endif + + if (!is_realtime(t)) + return -EPERM; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + dgl_lock = litmus->get_dgl_spinlock(t); +#endif + + lock_global_irqsave(dgl_lock, flags); + lock_fine_irqsave(&mutex->lock, flags); + + if (mutex->owner) { + TRACE_TASK(t, "Blocking on lock %d.\n", l->ident); + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) + // KLUDGE: don't count this suspension as time in the critical gpu + // critical section + if(tsk_rt(t)->held_gpus) { + tsk_rt(t)->suspend_gpu_tracker_on_block = 1; + } +#endif + + /* resource is not free => must suspend and wait */ + + owner = mutex->owner; + + init_waitqueue_entry(&wait, t); + + tsk_rt(t)->blocked_lock = l; /* record where we are blocked */ + mb(); // needed? + + /* FIXME: interruptible would be nice some day */ + set_task_state(t, TASK_UNINTERRUPTIBLE); + + __add_wait_queue_tail_exclusive(&mutex->wait, &wait); + + /* check if we need to activate priority inheritance */ + //if (edf_higher_prio(t, mutex->hp_waiter)) { + if (litmus->compare(t, mutex->hp_waiter)) { + + struct task_struct *old_max_eff_prio; + struct task_struct *new_max_eff_prio; + struct task_struct *new_prio = NULL; + + if(mutex->hp_waiter) + TRACE_TASK(t, "has higher prio than hp_waiter (%s/%d).\n", + mutex->hp_waiter->comm, mutex->hp_waiter->pid); + else + TRACE_TASK(t, "has higher prio than hp_waiter (NIL).\n"); + + raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock); + + old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + mutex->hp_waiter = t; + l->nest.hp_waiter_eff_prio = effective_priority(mutex->hp_waiter); + binheap_decrease(&l->nest.hp_binheap_node, + &tsk_rt(owner)->hp_blocked_tasks); + new_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + if(new_max_eff_prio != old_max_eff_prio) { + TRACE_TASK(t, "is new hp_waiter.\n"); + + if ((effective_priority(owner) == old_max_eff_prio) || + //(__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))){ + (litmus->__compare(new_max_eff_prio, BASE, owner, EFFECTIVE))){ + new_prio = new_max_eff_prio; + } + } + else { + TRACE_TASK(t, "no change in max_eff_prio of heap.\n"); + } + + if(new_prio) { + litmus->nested_increase_prio(owner, new_prio, &mutex->lock, + flags); // unlocks lock. + } + else { + raw_spin_unlock(&tsk_rt(owner)->hp_blocked_tasks_lock); + unlock_fine_irqrestore(&mutex->lock, flags); + } + } + else { + TRACE_TASK(t, "no change in hp_waiter.\n"); + + unlock_fine_irqrestore(&mutex->lock, flags); + } + + unlock_global_irqrestore(dgl_lock, flags); + + TS_LOCK_SUSPEND; + + /* We depend on the FIFO order. Thus, we don't need to recheck + * when we wake up; we are guaranteed to have the lock since + * there is only one wake up per release. + */ + + schedule(); + + TS_LOCK_RESUME; + + /* Since we hold the lock, no other task will change + * ->owner. We can thus check it without acquiring the spin + * lock. */ + BUG_ON(mutex->owner != t); + + TRACE_TASK(t, "Acquired lock %d.\n", l->ident); + + } else { + TRACE_TASK(t, "Acquired lock %d with no blocking.\n", l->ident); + + /* it's ours now */ + mutex->owner = t; + + raw_spin_lock(&tsk_rt(mutex->owner)->hp_blocked_tasks_lock); + binheap_add(&l->nest.hp_binheap_node, &tsk_rt(t)->hp_blocked_tasks, + struct nested_info, hp_binheap_node); + raw_spin_unlock(&tsk_rt(mutex->owner)->hp_blocked_tasks_lock); + + + unlock_fine_irqrestore(&mutex->lock, flags); + unlock_global_irqrestore(dgl_lock, flags); + } + + return 0; +} + + + +int rsm_mutex_unlock(struct litmus_lock* l) +{ + struct task_struct *t = current, *next = NULL; + struct rsm_mutex *mutex = rsm_mutex_from_lock(l); + unsigned long flags; + + struct task_struct *old_max_eff_prio; + + int wake_up_task = 1; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + dgl_wait_state_t *dgl_wait = NULL; + raw_spinlock_t *dgl_lock = litmus->get_dgl_spinlock(t); +#endif + + int err = 0; + + if (mutex->owner != t) { + err = -EINVAL; + return err; + } + + lock_global_irqsave(dgl_lock, flags); + lock_fine_irqsave(&mutex->lock, flags); + + raw_spin_lock(&tsk_rt(t)->hp_blocked_tasks_lock); + + TRACE_TASK(t, "Freeing lock %d\n", l->ident); + + old_max_eff_prio = top_priority(&tsk_rt(t)->hp_blocked_tasks); + binheap_delete(&l->nest.hp_binheap_node, &tsk_rt(t)->hp_blocked_tasks); + + if(tsk_rt(t)->inh_task){ + struct task_struct *new_max_eff_prio = + top_priority(&tsk_rt(t)->hp_blocked_tasks); + + if((new_max_eff_prio == NULL) || + /* there was a change in eff prio */ + ( (new_max_eff_prio != old_max_eff_prio) && + /* and owner had the old eff prio */ + (effective_priority(t) == old_max_eff_prio)) ) + { + // old_max_eff_prio > new_max_eff_prio + + //if(__edf_higher_prio(new_max_eff_prio, BASE, t, EFFECTIVE)) { + if(litmus->__compare(new_max_eff_prio, BASE, t, EFFECTIVE)) { + TRACE_TASK(t, "new_max_eff_prio > task's eff_prio-- new_max_eff_prio: %s/%d task: %s/%d [%s/%d]\n", + new_max_eff_prio->comm, new_max_eff_prio->pid, + t->comm, t->pid, tsk_rt(t)->inh_task->comm, + tsk_rt(t)->inh_task->pid); + WARN_ON(1); + } + + litmus->decrease_prio(t, new_max_eff_prio); + } + } + + if(binheap_empty(&tsk_rt(t)->hp_blocked_tasks) && + tsk_rt(t)->inh_task != NULL) + { + WARN_ON(tsk_rt(t)->inh_task != NULL); + TRACE_TASK(t, "No more locks are held, but eff_prio = %s/%d\n", + tsk_rt(t)->inh_task->comm, tsk_rt(t)->inh_task->pid); + } + + raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); + + + /* check if there are jobs waiting for this resource */ +#ifdef CONFIG_LITMUS_DGL_SUPPORT + __waitqueue_dgl_remove_first(&mutex->wait, &dgl_wait, &next); + if(dgl_wait) { + next = dgl_wait->task; + //select_next_lock_if_primary(l, dgl_wait); + } +#else + next = __waitqueue_remove_first(&mutex->wait); +#endif + if (next) { + /* next becomes the resouce holder */ + mutex->owner = next; + TRACE_CUR("lock ownership passed to %s/%d\n", next->comm, next->pid); + + /* determine new hp_waiter if necessary */ + if (next == mutex->hp_waiter) { + + TRACE_TASK(next, "was highest-prio waiter\n"); + /* next has the highest priority --- it doesn't need to + * inherit. However, we need to make sure that the + * next-highest priority in the queue is reflected in + * hp_waiter. */ + mutex->hp_waiter = rsm_mutex_find_hp_waiter(mutex, next); + l->nest.hp_waiter_eff_prio = (mutex->hp_waiter) ? + effective_priority(mutex->hp_waiter) : + NULL; + + if (mutex->hp_waiter) + TRACE_TASK(mutex->hp_waiter, "is new highest-prio waiter\n"); + else + TRACE("no further waiters\n"); + + raw_spin_lock(&tsk_rt(next)->hp_blocked_tasks_lock); + + binheap_add(&l->nest.hp_binheap_node, + &tsk_rt(next)->hp_blocked_tasks, + struct nested_info, hp_binheap_node); + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + if(dgl_wait) { + select_next_lock_if_primary(l, dgl_wait); + //wake_up_task = atomic_dec_and_test(&dgl_wait->nr_remaining); + --(dgl_wait->nr_remaining); + wake_up_task = (dgl_wait->nr_remaining == 0); + } +#endif + raw_spin_unlock(&tsk_rt(next)->hp_blocked_tasks_lock); + } + else { + /* Well, if 'next' is not the highest-priority waiter, + * then it (probably) ought to inherit the highest-priority + * waiter's priority. */ + TRACE_TASK(next, "is not hp_waiter of lock %d.\n", l->ident); + + raw_spin_lock(&tsk_rt(next)->hp_blocked_tasks_lock); + + binheap_add(&l->nest.hp_binheap_node, + &tsk_rt(next)->hp_blocked_tasks, + struct nested_info, hp_binheap_node); + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + if(dgl_wait) { + select_next_lock_if_primary(l, dgl_wait); + --(dgl_wait->nr_remaining); + wake_up_task = (dgl_wait->nr_remaining == 0); + } +#endif + + /* It is possible that 'next' *should* be the hp_waiter, but isn't + * because that update hasn't yet executed (update operation is + * probably blocked on mutex->lock). So only inherit if the top of + * 'next's top heap node is indeed the effective prio. of hp_waiter. + * (We use l->hp_waiter_eff_prio instead of effective_priority(hp_waiter) + * since the effective priority of hp_waiter can change (and the + * update has not made it to this lock).) + */ +#ifdef CONFIG_LITMUS_DGL_SUPPORT + if((l->nest.hp_waiter_eff_prio != NULL) && + (top_priority(&tsk_rt(next)->hp_blocked_tasks) == + l->nest.hp_waiter_eff_prio)) + { + if(dgl_wait && tsk_rt(next)->blocked_lock) { + BUG_ON(wake_up_task); + //if(__edf_higher_prio(l->nest.hp_waiter_eff_prio, BASE, next, EFFECTIVE)) { + if(litmus->__compare(l->nest.hp_waiter_eff_prio, BASE, next, EFFECTIVE)) { + litmus->nested_increase_prio(next, + l->nest.hp_waiter_eff_prio, &mutex->lock, flags); // unlocks lock && hp_blocked_tasks_lock. + goto out; // all spinlocks are released. bail out now. + } + } + else { + litmus->increase_prio(next, l->nest.hp_waiter_eff_prio); + } + } + + raw_spin_unlock(&tsk_rt(next)->hp_blocked_tasks_lock); +#else + if(likely(top_priority(&tsk_rt(next)->hp_blocked_tasks) == + l->nest.hp_waiter_eff_prio)) + { + litmus->increase_prio(next, l->nest.hp_waiter_eff_prio); + } + raw_spin_unlock(&tsk_rt(next)->hp_blocked_tasks_lock); +#endif + } + + if(wake_up_task) { + TRACE_TASK(next, "waking up since it is no longer blocked.\n"); + + tsk_rt(next)->blocked_lock = NULL; + mb(); + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) + // re-enable tracking + if(tsk_rt(next)->held_gpus) { + tsk_rt(next)->suspend_gpu_tracker_on_block = 0; + } +#endif + + wake_up_process(next); + } + else { + TRACE_TASK(next, "is still blocked.\n"); + } + } + else { + /* becomes available */ + mutex->owner = NULL; + } + + unlock_fine_irqrestore(&mutex->lock, flags); + +#ifdef CONFIG_LITMUS_DGL_SUPPORT +out: +#endif + unlock_global_irqrestore(dgl_lock, flags); + + return err; +} + + +void rsm_mutex_propagate_increase_inheritance(struct litmus_lock* l, + struct task_struct* t, + raw_spinlock_t* to_unlock, + unsigned long irqflags) +{ + struct rsm_mutex *mutex = rsm_mutex_from_lock(l); + + // relay-style locking + lock_fine(&mutex->lock); + unlock_fine(to_unlock); + + if(tsk_rt(t)->blocked_lock == l) { // prevent race on tsk_rt(t)->blocked + struct task_struct *owner = mutex->owner; + + struct task_struct *old_max_eff_prio; + struct task_struct *new_max_eff_prio; + + raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock); + + old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + //if((t != mutex->hp_waiter) && edf_higher_prio(t, mutex->hp_waiter)) { + if((t != mutex->hp_waiter) && litmus->compare(t, mutex->hp_waiter)) { + TRACE_TASK(t, "is new highest-prio waiter by propagation.\n"); + mutex->hp_waiter = t; + } + if(t == mutex->hp_waiter) { + // reflect the decreased priority in the heap node. + l->nest.hp_waiter_eff_prio = effective_priority(mutex->hp_waiter); + + BUG_ON(!binheap_is_in_heap(&l->nest.hp_binheap_node)); + BUG_ON(!binheap_is_in_this_heap(&l->nest.hp_binheap_node, + &tsk_rt(owner)->hp_blocked_tasks)); + + binheap_decrease(&l->nest.hp_binheap_node, + &tsk_rt(owner)->hp_blocked_tasks); + } + + new_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + + if(new_max_eff_prio != old_max_eff_prio) { + // new_max_eff_prio > old_max_eff_prio holds. + if ((effective_priority(owner) == old_max_eff_prio) || + //(__edf_higher_prio(new_max_eff_prio, BASE, owner, EFFECTIVE))) { + (litmus->__compare(new_max_eff_prio, BASE, owner, EFFECTIVE))) { + TRACE_CUR("Propagating inheritance to holder of lock %d.\n", + l->ident); + + // beware: recursion + litmus->nested_increase_prio(owner, new_max_eff_prio, + &mutex->lock, irqflags); // unlocks mutex->lock + } + else { + TRACE_CUR("Lower priority than holder %s/%d. No propagation.\n", + owner->comm, owner->pid); + raw_spin_unlock(&tsk_rt(owner)->hp_blocked_tasks_lock); + unlock_fine_irqrestore(&mutex->lock, irqflags); + } + } + else { + TRACE_TASK(mutex->owner, "No change in maxiumum effective priority.\n"); + raw_spin_unlock(&tsk_rt(owner)->hp_blocked_tasks_lock); + unlock_fine_irqrestore(&mutex->lock, irqflags); + } + } + else { + struct litmus_lock *still_blocked = tsk_rt(t)->blocked_lock; + + TRACE_TASK(t, "is not blocked on lock %d.\n", l->ident); + if(still_blocked) { + TRACE_TASK(t, "is still blocked on a lock though (lock %d).\n", + still_blocked->ident); + if(still_blocked->ops->propagate_increase_inheritance) { + /* due to relay-style nesting of spinlocks (acq. A, acq. B, free A, free B) + we know that task 't' has not released any locks behind us in this + chain. Propagation just needs to catch up with task 't'. */ + still_blocked->ops->propagate_increase_inheritance(still_blocked, + t, + &mutex->lock, + irqflags); + } + else { + TRACE_TASK(t, + "Inheritor is blocked on lock (%p) that does not " + "support nesting!\n", + still_blocked); + unlock_fine_irqrestore(&mutex->lock, irqflags); + } + } + else { + unlock_fine_irqrestore(&mutex->lock, irqflags); + } + } +} + + +void rsm_mutex_propagate_decrease_inheritance(struct litmus_lock* l, + struct task_struct* t, + raw_spinlock_t* to_unlock, + unsigned long irqflags) +{ + struct rsm_mutex *mutex = rsm_mutex_from_lock(l); + + // relay-style locking + lock_fine(&mutex->lock); + unlock_fine(to_unlock); + + if(tsk_rt(t)->blocked_lock == l) { // prevent race on tsk_rt(t)->blocked + if(t == mutex->hp_waiter) { + struct task_struct *owner = mutex->owner; + + struct task_struct *old_max_eff_prio; + struct task_struct *new_max_eff_prio; + + raw_spin_lock(&tsk_rt(owner)->hp_blocked_tasks_lock); + + old_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + binheap_delete(&l->nest.hp_binheap_node, &tsk_rt(owner)->hp_blocked_tasks); + mutex->hp_waiter = rsm_mutex_find_hp_waiter(mutex, NULL); + l->nest.hp_waiter_eff_prio = (mutex->hp_waiter) ? + effective_priority(mutex->hp_waiter) : NULL; + binheap_add(&l->nest.hp_binheap_node, + &tsk_rt(owner)->hp_blocked_tasks, + struct nested_info, hp_binheap_node); + + new_max_eff_prio = top_priority(&tsk_rt(owner)->hp_blocked_tasks); + + if((old_max_eff_prio != new_max_eff_prio) && + (effective_priority(owner) == old_max_eff_prio)) + { + // Need to set new effective_priority for owner + + struct task_struct *decreased_prio; + + TRACE_CUR("Propagating decreased inheritance to holder of lock %d.\n", + l->ident); + + //if(__edf_higher_prio(new_max_eff_prio, BASE, owner, BASE)) { + if(litmus->__compare(new_max_eff_prio, BASE, owner, BASE)) { + TRACE_CUR("%s/%d has greater base priority than base priority of owner (%s/%d) of lock %d.\n", + (new_max_eff_prio) ? new_max_eff_prio->comm : "nil", + (new_max_eff_prio) ? new_max_eff_prio->pid : -1, + owner->comm, + owner->pid, + l->ident); + + decreased_prio = new_max_eff_prio; + } + else { + TRACE_CUR("%s/%d has lesser base priority than base priority of owner (%s/%d) of lock %d.\n", + (new_max_eff_prio) ? new_max_eff_prio->comm : "nil", + (new_max_eff_prio) ? new_max_eff_prio->pid : -1, + owner->comm, + owner->pid, + l->ident); + + decreased_prio = NULL; + } + + // beware: recursion + litmus->nested_decrease_prio(owner, decreased_prio, &mutex->lock, irqflags); // will unlock mutex->lock + } + else { + raw_spin_unlock(&tsk_rt(owner)->hp_blocked_tasks_lock); + unlock_fine_irqrestore(&mutex->lock, irqflags); + } + } + else { + TRACE_TASK(t, "is not hp_waiter. No propagation.\n"); + unlock_fine_irqrestore(&mutex->lock, irqflags); + } + } + else { + struct litmus_lock *still_blocked = tsk_rt(t)->blocked_lock; + + TRACE_TASK(t, "is not blocked on lock %d.\n", l->ident); + if(still_blocked) { + TRACE_TASK(t, "is still blocked on a lock though (lock %d).\n", + still_blocked->ident); + if(still_blocked->ops->propagate_decrease_inheritance) { + /* due to linked nesting of spinlocks (acq. A, acq. B, free A, free B) + we know that task 't' has not released any locks behind us in this + chain. propagation just needs to catch up with task 't' */ + still_blocked->ops->propagate_decrease_inheritance(still_blocked, + t, + &mutex->lock, + irqflags); + } + else { + TRACE_TASK(t, "Inheritor is blocked on lock (%p) that does not support nesting!\n", + still_blocked); + unlock_fine_irqrestore(&mutex->lock, irqflags); + } + } + else { + unlock_fine_irqrestore(&mutex->lock, irqflags); + } + } +} + + +int rsm_mutex_close(struct litmus_lock* l) +{ + struct task_struct *t = current; + struct rsm_mutex *mutex = rsm_mutex_from_lock(l); + unsigned long flags; + + int owner; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + raw_spinlock_t *dgl_lock = litmus->get_dgl_spinlock(t); +#endif + + lock_global_irqsave(dgl_lock, flags); + lock_fine_irqsave(&mutex->lock, flags); + + owner = (mutex->owner == t); + + unlock_fine_irqrestore(&mutex->lock, flags); + unlock_global_irqrestore(dgl_lock, flags); + + if (owner) + rsm_mutex_unlock(l); + + return 0; +} + +void rsm_mutex_free(struct litmus_lock* lock) +{ + kfree(rsm_mutex_from_lock(lock)); +} + +struct litmus_lock* rsm_mutex_new(struct litmus_lock_ops* ops) +{ + struct rsm_mutex* mutex; + + mutex = kmalloc(sizeof(*mutex), GFP_KERNEL); + if (!mutex) + return NULL; + + mutex->litmus_lock.ops = ops; + mutex->owner = NULL; + mutex->hp_waiter = NULL; + init_waitqueue_head(&mutex->wait); + + +#ifdef CONFIG_DEBUG_SPINLOCK + { + __raw_spin_lock_init(&mutex->lock, + ((struct litmus_lock*)mutex)->cheat_lockdep, + &((struct litmus_lock*)mutex)->key); + } +#else + raw_spin_lock_init(&mutex->lock); +#endif + + ((struct litmus_lock*)mutex)->nest.hp_waiter_ptr = &mutex->hp_waiter; + + return &mutex->litmus_lock; +} + diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c new file mode 100644 index 0000000..d405854 --- /dev/null +++ b/litmus/rt_domain.c @@ -0,0 +1,357 @@ +/* + * litmus/rt_domain.c + * + * LITMUS real-time infrastructure. This file contains the + * functions that manipulate RT domains. RT domains are an abstraction + * of a ready queue and a release queue. + */ + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +#include + +/* Uncomment when debugging timer races... */ +#if 0 +#define VTRACE_TASK TRACE_TASK +#define VTRACE TRACE +#else +#define VTRACE_TASK(t, fmt, args...) /* shut up */ +#define VTRACE(fmt, args...) /* be quiet already */ +#endif + +static int dummy_resched(rt_domain_t *rt) +{ + return 0; +} + +static int dummy_order(struct bheap_node* a, struct bheap_node* b) +{ + return 0; +} + +/* default implementation: use default lock */ +static void default_release_jobs(rt_domain_t* rt, struct bheap* tasks) +{ + merge_ready(rt, tasks); +} + +static unsigned int time2slot(lt_t time) +{ + return (unsigned int) time2quanta(time, FLOOR) % RELEASE_QUEUE_SLOTS; +} + +static enum hrtimer_restart on_release_timer(struct hrtimer *timer) +{ + unsigned long flags; + struct release_heap* rh; + rh = container_of(timer, struct release_heap, timer); + + TS_RELEASE_LATENCY(rh->release_time); + + VTRACE("on_release_timer(0x%p) starts.\n", timer); + + TS_RELEASE_START; + + + raw_spin_lock_irqsave(&rh->dom->release_lock, flags); + VTRACE("CB has the release_lock 0x%p\n", &rh->dom->release_lock); + /* remove from release queue */ + list_del(&rh->list); + raw_spin_unlock_irqrestore(&rh->dom->release_lock, flags); + VTRACE("CB returned release_lock 0x%p\n", &rh->dom->release_lock); + + /* call release callback */ + rh->dom->release_jobs(rh->dom, &rh->heap); + /* WARNING: rh can be referenced from other CPUs from now on. */ + + TS_RELEASE_END; + + VTRACE("on_release_timer(0x%p) ends.\n", timer); + + return HRTIMER_NORESTART; +} + +/* allocated in litmus.c */ +struct kmem_cache * release_heap_cache; + +struct release_heap* release_heap_alloc(int gfp_flags) +{ + struct release_heap* rh; + rh= kmem_cache_alloc(release_heap_cache, gfp_flags); + if (rh) { + /* initialize timer */ + hrtimer_init(&rh->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + rh->timer.function = on_release_timer; + } + return rh; +} + +void release_heap_free(struct release_heap* rh) +{ + /* make sure timer is no longer in use */ + hrtimer_cancel(&rh->timer); + kmem_cache_free(release_heap_cache, rh); +} + +/* Caller must hold release lock. + * Will return heap for given time. If no such heap exists prior to + * the invocation it will be created. + */ +static struct release_heap* get_release_heap(rt_domain_t *rt, + struct task_struct* t, + int use_task_heap) +{ + struct list_head* pos; + struct release_heap* heap = NULL; + struct release_heap* rh; + lt_t release_time = get_release(t); + unsigned int slot = time2slot(release_time); + + /* initialize pos for the case that the list is empty */ + pos = rt->release_queue.slot[slot].next; + list_for_each(pos, &rt->release_queue.slot[slot]) { + rh = list_entry(pos, struct release_heap, list); + if (release_time == rh->release_time) { + /* perfect match -- this happens on hyperperiod + * boundaries + */ + heap = rh; + break; + } else if (lt_before(release_time, rh->release_time)) { + /* we need to insert a new node since rh is + * already in the future + */ + break; + } + } + if (!heap && use_task_heap) { + /* use pre-allocated release heap */ + rh = tsk_rt(t)->rel_heap; + + rh->dom = rt; + rh->release_time = release_time; + + /* add to release queue */ + list_add(&rh->list, pos->prev); + heap = rh; + } + return heap; +} + +static void reinit_release_heap(struct task_struct* t) +{ + struct release_heap* rh; + + /* use pre-allocated release heap */ + rh = tsk_rt(t)->rel_heap; + + /* Make sure it is safe to use. The timer callback could still + * be executing on another CPU; hrtimer_cancel() will wait + * until the timer callback has completed. However, under no + * circumstances should the timer be active (= yet to be + * triggered). + * + * WARNING: If the CPU still holds the release_lock at this point, + * deadlock may occur! + */ + BUG_ON(hrtimer_cancel(&rh->timer)); + + /* initialize */ + bheap_init(&rh->heap); +#ifdef CONFIG_RELEASE_MASTER + atomic_set(&rh->info.state, HRTIMER_START_ON_INACTIVE); +#endif +} +/* arm_release_timer() - start local release timer or trigger + * remote timer (pull timer) + * + * Called by add_release() with: + * - tobe_lock taken + * - IRQ disabled + */ +#ifdef CONFIG_RELEASE_MASTER +#define arm_release_timer(t) arm_release_timer_on((t), NO_CPU) +static void arm_release_timer_on(rt_domain_t *_rt , int target_cpu) +#else +static void arm_release_timer(rt_domain_t *_rt) +#endif +{ + rt_domain_t *rt = _rt; + struct list_head list; + struct list_head *pos, *safe; + struct task_struct* t; + struct release_heap* rh; + + VTRACE("arm_release_timer() at %llu\n", litmus_clock()); + list_replace_init(&rt->tobe_released, &list); + + list_for_each_safe(pos, safe, &list) { + /* pick task of work list */ + t = list_entry(pos, struct task_struct, rt_param.list); + sched_trace_task_release(t); + list_del(pos); + + /* put into release heap while holding release_lock */ + raw_spin_lock(&rt->release_lock); + VTRACE_TASK(t, "I have the release_lock 0x%p\n", &rt->release_lock); + + rh = get_release_heap(rt, t, 0); + if (!rh) { + /* need to use our own, but drop lock first */ + raw_spin_unlock(&rt->release_lock); + VTRACE_TASK(t, "Dropped release_lock 0x%p\n", + &rt->release_lock); + + reinit_release_heap(t); + VTRACE_TASK(t, "release_heap ready\n"); + + raw_spin_lock(&rt->release_lock); + VTRACE_TASK(t, "Re-acquired release_lock 0x%p\n", + &rt->release_lock); + + rh = get_release_heap(rt, t, 1); + } + bheap_insert(rt->order, &rh->heap, tsk_rt(t)->heap_node); + VTRACE_TASK(t, "arm_release_timer(): added to release heap\n"); + + raw_spin_unlock(&rt->release_lock); + VTRACE_TASK(t, "Returned the release_lock 0x%p\n", &rt->release_lock); + + /* To avoid arming the timer multiple times, we only let the + * owner do the arming (which is the "first" task to reference + * this release_heap anyway). + */ + if (rh == tsk_rt(t)->rel_heap) { + VTRACE_TASK(t, "arming timer 0x%p\n", &rh->timer); + /* we cannot arm the timer using hrtimer_start() + * as it may deadlock on rq->lock + * + * PINNED mode is ok on both local and remote CPU + */ +#ifdef CONFIG_RELEASE_MASTER + if (rt->release_master == NO_CPU && + target_cpu == NO_CPU) +#endif + __hrtimer_start_range_ns(&rh->timer, + ns_to_ktime(rh->release_time), + 0, HRTIMER_MODE_ABS_PINNED, 0); +#ifdef CONFIG_RELEASE_MASTER + else + hrtimer_start_on( + /* target_cpu overrides release master */ + (target_cpu != NO_CPU ? + target_cpu : rt->release_master), + &rh->info, &rh->timer, + ns_to_ktime(rh->release_time), + HRTIMER_MODE_ABS_PINNED); +#endif + } else + VTRACE_TASK(t, "0x%p is not my timer\n", &rh->timer); + } +} + +void rt_domain_init(rt_domain_t *rt, + bheap_prio_t order, + check_resched_needed_t check, + release_jobs_t release + ) +{ + int i; + + BUG_ON(!rt); + if (!check) + check = dummy_resched; + if (!release) + release = default_release_jobs; + if (!order) + order = dummy_order; + +#ifdef CONFIG_RELEASE_MASTER + rt->release_master = NO_CPU; +#endif + + bheap_init(&rt->ready_queue); + INIT_LIST_HEAD(&rt->tobe_released); + for (i = 0; i < RELEASE_QUEUE_SLOTS; i++) + INIT_LIST_HEAD(&rt->release_queue.slot[i]); + + raw_spin_lock_init(&rt->ready_lock); + raw_spin_lock_init(&rt->release_lock); + raw_spin_lock_init(&rt->tobe_lock); + + rt->check_resched = check; + rt->release_jobs = release; + rt->order = order; +} + +/* add_ready - add a real-time task to the rt ready queue. It must be runnable. + * @new: the newly released task + */ +void __add_ready(rt_domain_t* rt, struct task_struct *new) +{ + TRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n", + new->comm, new->pid, get_exec_cost(new), get_rt_period(new), + get_release(new), litmus_clock()); + + BUG_ON(bheap_node_in_heap(tsk_rt(new)->heap_node)); + + bheap_insert(rt->order, &rt->ready_queue, tsk_rt(new)->heap_node); + rt->check_resched(rt); +} + +/* merge_ready - Add a sorted set of tasks to the rt ready queue. They must be runnable. + * @tasks - the newly released tasks + */ +void __merge_ready(rt_domain_t* rt, struct bheap* tasks) +{ + bheap_union(rt->order, &rt->ready_queue, tasks); + rt->check_resched(rt); +} + + +#ifdef CONFIG_RELEASE_MASTER +void __add_release_on(rt_domain_t* rt, struct task_struct *task, + int target_cpu) +{ + TRACE_TASK(task, "add_release_on(), rel=%llu, target=%d\n", + get_release(task), target_cpu); + list_add(&tsk_rt(task)->list, &rt->tobe_released); + task->rt_param.domain = rt; + + /* start release timer */ + TS_SCHED2_START(task); + + arm_release_timer_on(rt, target_cpu); + + TS_SCHED2_END(task); +} +#endif + +/* add_release - add a real-time task to the rt release queue. + * @task: the sleeping task + */ +void __add_release(rt_domain_t* rt, struct task_struct *task) +{ + TRACE_TASK(task, "add_release(), rel=%llu\n", get_release(task)); + list_add(&tsk_rt(task)->list, &rt->tobe_released); + task->rt_param.domain = rt; + + /* start release timer */ + TS_SCHED2_START(task); + + arm_release_timer(rt); + + TS_SCHED2_END(task); +} + diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c new file mode 100644 index 0000000..be14dbe --- /dev/null +++ b/litmus/sched_cedf.c @@ -0,0 +1,1849 @@ +/* + * litmus/sched_cedf.c + * + * Implementation of the C-EDF scheduling algorithm. + * + * This implementation is based on G-EDF: + * - CPUs are clustered around L2 or L3 caches. + * - Clusters topology is automatically detected (this is arch dependent + * and is working only on x86 at the moment --- and only with modern + * cpus that exports cpuid4 information) + * - The plugins _does not_ attempt to put tasks in the right cluster i.e. + * the programmer needs to be aware of the topology to place tasks + * in the desired cluster + * - default clustering is around L2 cache (cache index = 2) + * supported clusters are: L1 (private cache: pedf), L2, L3, ALL (all + * online_cpus are placed in a single cluster). + * + * For details on functions, take a look at sched_gsn_edf.c + * + * Currently, we do not support changes in the number of online cpus. + * If the num_online_cpus() dynamically changes, the plugin is broken. + * + * This version uses the simple approach and serializes all scheduling + * decisions by the use of a queue lock. This is probably not the + * best way to do it, but it should suffice for now. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#ifdef CONFIG_LITMUS_LOCKING +#include +#endif + +#ifdef CONFIG_LITMUS_NESTED_LOCKING +#include +#include +#endif + +#ifdef CONFIG_SCHED_CPU_AFFINITY +#include +#endif + +/* to configure the cluster size */ +#include + +#ifdef CONFIG_SCHED_CPU_AFFINITY +#include +#endif + +#ifdef CONFIG_LITMUS_SOFTIRQD +#include +#endif + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD +#include +#include +#endif + +#ifdef CONFIG_LITMUS_NVIDIA +#include +#endif + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) +#include +#endif + +/* Reference configuration variable. Determines which cache level is used to + * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that + * all CPUs form a single cluster (just like GSN-EDF). + */ +static enum cache_level cluster_config = GLOBAL_CLUSTER; + +struct clusterdomain; + +/* cpu_entry_t - maintain the linked and scheduled state + * + * A cpu also contains a pointer to the cedf_domain_t cluster + * that owns it (struct clusterdomain*) + */ +typedef struct { + int cpu; + struct clusterdomain* cluster; /* owning cluster */ + struct task_struct* linked; /* only RT tasks */ + struct task_struct* scheduled; /* only RT tasks */ + atomic_t will_schedule; /* prevent unneeded IPIs */ + struct binheap_node hn; +} cpu_entry_t; + +/* one cpu_entry_t per CPU */ +DEFINE_PER_CPU(cpu_entry_t, cedf_cpu_entries); + +#define set_will_schedule() \ + (atomic_set(&__get_cpu_var(cedf_cpu_entries).will_schedule, 1)) +#define clear_will_schedule() \ + (atomic_set(&__get_cpu_var(cedf_cpu_entries).will_schedule, 0)) +#define test_will_schedule(cpu) \ + (atomic_read(&per_cpu(cedf_cpu_entries, cpu).will_schedule)) + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD +struct tasklet_head +{ + struct tasklet_struct *head; + struct tasklet_struct **tail; +}; +#endif + +/* + * In C-EDF there is a cedf domain _per_ cluster + * The number of clusters is dynamically determined accordingly to the + * total cpu number and the cluster size + */ +typedef struct clusterdomain { + /* rt_domain for this cluster */ + rt_domain_t domain; + /* cpus in this cluster */ + cpu_entry_t* *cpus; + /* map of this cluster cpus */ + cpumask_var_t cpu_map; + /* the cpus queue themselves according to priority in here */ + struct binheap_handle cpu_heap; + /* lock for this cluster */ +#define cluster_lock domain.ready_lock + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + struct tasklet_head pending_tasklets; +#endif + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + raw_spinlock_t dgl_lock; +#endif +} cedf_domain_t; + +/* a cedf_domain per cluster; allocation is done at init/activation time */ +cedf_domain_t *cedf; + +#define remote_cluster(cpu) ((cedf_domain_t *) per_cpu(cedf_cpu_entries, cpu).cluster) +#define task_cpu_cluster(task) remote_cluster(get_partition(task)) + +/* total number of cluster */ +static int num_clusters; +/* we do not support cluster of different sizes */ +static unsigned int cluster_size; + +static int clusters_allocated = 0; + +#ifdef CONFIG_LITMUS_DGL_SUPPORT +static raw_spinlock_t* cedf_get_dgl_spinlock(struct task_struct *t) +{ + cedf_domain_t *cluster = task_cpu_cluster(t); + return(&cluster->dgl_lock); +} +#endif + + +/* Uncomment WANT_ALL_SCHED_EVENTS if you want to see all scheduling + * decisions in the TRACE() log; uncomment VERBOSE_INIT for verbose + * information during the initialization of the plugin (e.g., topology) +#define WANT_ALL_SCHED_EVENTS + */ +#define VERBOSE_INIT + +static int cpu_lower_prio(struct binheap_node *_a, struct binheap_node *_b) +{ + cpu_entry_t *a = binheap_entry(_a, cpu_entry_t, hn); + cpu_entry_t *b = binheap_entry(_b, cpu_entry_t, hn); + + /* Note that a and b are inverted: we want the lowest-priority CPU at + * the top of the heap. + */ + return edf_higher_prio(b->linked, a->linked); +} + +/* update_cpu_position - Move the cpu entry to the correct place to maintain + * order in the cpu queue. Caller must hold cedf lock. + */ +static void update_cpu_position(cpu_entry_t *entry) +{ + cedf_domain_t *cluster = entry->cluster; + + if (likely(binheap_is_in_heap(&entry->hn))) { + binheap_delete(&entry->hn, &cluster->cpu_heap); + } + + binheap_add(&entry->hn, &cluster->cpu_heap, cpu_entry_t, hn); +} + +/* caller must hold cedf lock */ +static cpu_entry_t* lowest_prio_cpu(cedf_domain_t *cluster) +{ + return binheap_top_entry(&cluster->cpu_heap, cpu_entry_t, hn); +} + + +/* link_task_to_cpu - Update the link of a CPU. + * Handles the case where the to-be-linked task is already + * scheduled on a different CPU. + */ +static noinline void link_task_to_cpu(struct task_struct* linked, + cpu_entry_t *entry) +{ + cpu_entry_t *sched; + struct task_struct* tmp; + int on_cpu; + + BUG_ON(linked && !is_realtime(linked)); + + /* Currently linked task is set to be unlinked. */ + if (entry->linked) { + entry->linked->rt_param.linked_on = NO_CPU; + } + + /* Link new task to CPU. */ + if (linked) { + set_rt_flags(linked, RT_F_RUNNING); + /* handle task is already scheduled somewhere! */ + on_cpu = linked->rt_param.scheduled_on; + if (on_cpu != NO_CPU) { + sched = &per_cpu(cedf_cpu_entries, on_cpu); + /* this should only happen if not linked already */ + BUG_ON(sched->linked == linked); + + /* If we are already scheduled on the CPU to which we + * wanted to link, we don't need to do the swap -- + * we just link ourselves to the CPU and depend on + * the caller to get things right. + */ + if (entry != sched) { + TRACE_TASK(linked, + "already scheduled on %d, updating link.\n", + sched->cpu); + tmp = sched->linked; + linked->rt_param.linked_on = sched->cpu; + sched->linked = linked; + update_cpu_position(sched); + linked = tmp; + } + } + if (linked) /* might be NULL due to swap */ + linked->rt_param.linked_on = entry->cpu; + } + entry->linked = linked; +#ifdef WANT_ALL_SCHED_EVENTS + if (linked) + TRACE_TASK(linked, "linked to %d.\n", entry->cpu); + else + TRACE("NULL linked to %d.\n", entry->cpu); +#endif + update_cpu_position(entry); +} + +/* unlink - Make sure a task is not linked any longer to an entry + * where it was linked before. Must hold cluster_lock. + */ +static noinline void unlink(struct task_struct* t) +{ + cpu_entry_t *entry; + + if (t->rt_param.linked_on != NO_CPU) { + /* unlink */ + entry = &per_cpu(cedf_cpu_entries, t->rt_param.linked_on); + t->rt_param.linked_on = NO_CPU; + link_task_to_cpu(NULL, entry); + } else if (is_queued(t)) { + /* This is an interesting situation: t is scheduled, + * but was just recently unlinked. It cannot be + * linked anywhere else (because then it would have + * been relinked to this CPU), thus it must be in some + * queue. We must remove it from the list in this + * case. + * + * in C-EDF case is should be somewhere in the queue for + * its domain, therefore and we can get the domain using + * task_cpu_cluster + */ + remove(&(task_cpu_cluster(t))->domain, t); + } +} + + +/* preempt - force a CPU to reschedule + */ +static void preempt(cpu_entry_t *entry) +{ + preempt_if_preemptable(entry->scheduled, entry->cpu); +} + +/* requeue - Put an unlinked task into gsn-edf domain. + * Caller must hold cluster_lock. + */ +static noinline void requeue(struct task_struct* task) +{ + cedf_domain_t *cluster = task_cpu_cluster(task); + BUG_ON(!task); + /* sanity check before insertion */ + BUG_ON(is_queued(task)); + + if (is_released(task, litmus_clock())) + __add_ready(&cluster->domain, task); + else { + /* it has got to wait */ + add_release(&cluster->domain, task); + } +} + +#ifdef CONFIG_SCHED_CPU_AFFINITY +static cpu_entry_t* cedf_get_nearest_available_cpu( + cedf_domain_t *cluster, cpu_entry_t *start) +{ + cpu_entry_t *affinity; + + get_nearest_available_cpu(affinity, start, cedf_cpu_entries, +#ifdef CONFIG_RELEASE_MASTER + cluster->domain.release_master +#else + NO_CPU +#endif + ); + + /* make sure CPU is in our cluster */ + if (affinity && cpu_isset(affinity->cpu, *cluster->cpu_map)) + return(affinity); + else + return(NULL); +} +#endif + + +/* check for any necessary preemptions */ +static void check_for_preemptions(cedf_domain_t *cluster) +{ + struct task_struct *task; + cpu_entry_t *last; + + for(last = lowest_prio_cpu(cluster); + edf_preemption_needed(&cluster->domain, last->linked); + last = lowest_prio_cpu(cluster)) { + /* preemption necessary */ + task = __take_ready(&cluster->domain); + TRACE("check_for_preemptions: attempting to link task %d to %d\n", + task->pid, last->cpu); +#ifdef CONFIG_SCHED_CPU_AFFINITY + { + cpu_entry_t *affinity = + cedf_get_nearest_available_cpu(cluster, + &per_cpu(cedf_cpu_entries, task_cpu(task))); + if(affinity) + last = affinity; + else if(last->linked) + requeue(last->linked); + } +#else + if (last->linked) + requeue(last->linked); +#endif + link_task_to_cpu(task, last); + preempt(last); + } +} + +/* cedf_job_arrival: task is either resumed or released */ +static noinline void cedf_job_arrival(struct task_struct* task) +{ + cedf_domain_t *cluster = task_cpu_cluster(task); + BUG_ON(!task); + + requeue(task); + check_for_preemptions(cluster); +} + +static void cedf_release_jobs(rt_domain_t* rt, struct bheap* tasks) +{ + cedf_domain_t* cluster = container_of(rt, cedf_domain_t, domain); + unsigned long flags; + + raw_spin_lock_irqsave(&cluster->cluster_lock, flags); + + __merge_ready(&cluster->domain, tasks); + check_for_preemptions(cluster); + + raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); +} + +/* caller holds cluster_lock */ +static noinline void job_completion(struct task_struct *t, int forced) +{ + BUG_ON(!t); + + sched_trace_task_completion(t, forced); + +#ifdef CONFIG_LITMUS_NVIDIA + atomic_set(&tsk_rt(t)->nv_int_count, 0); +#endif + + TRACE_TASK(t, "job_completion().\n"); + + /* set flags */ + set_rt_flags(t, RT_F_SLEEP); + /* prepare for next period */ + prepare_for_next_period(t); + if (is_released(t, litmus_clock())) + sched_trace_task_release(t); + /* unlink */ + unlink(t); + /* requeue + * But don't requeue a blocking task. */ + if (is_running(t)) + cedf_job_arrival(t); +} + +/* cedf_tick - this function is called for every local timer + * interrupt. + * + * checks whether the current task has expired and checks + * whether we need to preempt it if it has not expired + */ +static void cedf_tick(struct task_struct* t) +{ + if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) { + if (!is_np(t)) { + /* np tasks will be preempted when they become + * preemptable again + */ + litmus_reschedule_local(); + set_will_schedule(); + TRACE("cedf_scheduler_tick: " + "%d is preemptable " + " => FORCE_RESCHED\n", t->pid); + } else if (is_user_np(t)) { + TRACE("cedf_scheduler_tick: " + "%d is non-preemptable, " + "preemption delayed.\n", t->pid); + request_exit_np(t); + } + } +} + + + + + + + + + + + + + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + + +static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flushed) +{ + if (!atomic_read(&tasklet->count)) { + if(tasklet->owner) { + sched_trace_tasklet_begin(tasklet->owner); + } + + if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) + { + BUG(); + } + TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n", + __FUNCTION__, + (tasklet->owner) ? tasklet->owner->pid : -1, + (tasklet->owner) ? 0 : 1); + tasklet->func(tasklet->data); + tasklet_unlock(tasklet); + + if(tasklet->owner) { + sched_trace_tasklet_end(tasklet->owner, flushed); + } + } + else { + BUG(); + } +} + + +static void do_lit_tasklets(cedf_domain_t* cluster, struct task_struct* sched_task) +{ + int work_to_do = 1; + struct tasklet_struct *tasklet = NULL; + unsigned long flags; + + while(work_to_do) { + + TS_NV_SCHED_BOTISR_START; + + raw_spin_lock_irqsave(&cluster->cluster_lock, flags); + + if(cluster->pending_tasklets.head != NULL) { + // remove tasklet at head. + struct tasklet_struct *prev = NULL; + tasklet = cluster->pending_tasklets.head; + + // find a tasklet with prio to execute; skip ones where + // sched_task has a higher priority. + // We use the '!edf' test instead of swaping function arguments since + // both sched_task and owner could be NULL. In this case, we want to + // still execute the tasklet. + while(tasklet && !edf_higher_prio(tasklet->owner, sched_task)) { + prev = tasklet; + tasklet = tasklet->next; + } + + if(tasklet) { // found something to execuite + // remove the tasklet from the queue + if(prev) { + prev->next = tasklet->next; + if(prev->next == NULL) { + TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); + cluster->pending_tasklets.tail = &(prev); + } + } + else { + cluster->pending_tasklets.head = tasklet->next; + if(tasklet->next == NULL) { + TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); + cluster->pending_tasklets.tail = &(cluster->pending_tasklets.head); + } + } + } + else { + TRACE("%s: No tasklets with eligible priority.\n", __FUNCTION__); + } + } + else { + TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__); + } + + raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); + + if(tasklet) { + __do_lit_tasklet(tasklet, 0ul); + tasklet = NULL; + } + else { + work_to_do = 0; + } + + TS_NV_SCHED_BOTISR_END; + } +} + +static void __add_pai_tasklet(struct tasklet_struct* tasklet, cedf_domain_t* cluster) +{ + struct tasklet_struct* step; + + tasklet->next = NULL; // make sure there are no old values floating around + + step = cluster->pending_tasklets.head; + if(step == NULL) { + TRACE("%s: tasklet queue empty. inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid); + // insert at tail. + *(cluster->pending_tasklets.tail) = tasklet; + cluster->pending_tasklets.tail = &(tasklet->next); + } + else if((*(cluster->pending_tasklets.tail) != NULL) && + edf_higher_prio((*(cluster->pending_tasklets.tail))->owner, tasklet->owner)) { + // insert at tail. + TRACE("%s: tasklet belongs at end. inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid); + + *(cluster->pending_tasklets.tail) = tasklet; + cluster->pending_tasklets.tail = &(tasklet->next); + } + else { + + // insert the tasklet somewhere in the middle. + + TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__); + + while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) { + step = step->next; + } + + // insert tasklet right before step->next. + + TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__, + tasklet->owner->pid, + (step->owner) ? + step->owner->pid : + -1, + (step->next) ? + ((step->next->owner) ? + step->next->owner->pid : + -1) : + -1); + + tasklet->next = step->next; + step->next = tasklet; + + // patch up the head if needed. + if(cluster->pending_tasklets.head == step) + { + TRACE("%s: %d is the new tasklet queue head.\n", __FUNCTION__, tasklet->owner->pid); + cluster->pending_tasklets.head = tasklet; + } + } +} + +static void cedf_run_tasklets(struct task_struct* sched_task) +{ + cedf_domain_t* cluster; + + preempt_disable(); + + cluster = (is_realtime(sched_task)) ? + task_cpu_cluster(sched_task) : + remote_cluster(smp_processor_id()); + + if(cluster && cluster->pending_tasklets.head != NULL) { + TRACE("%s: There are tasklets to process.\n", __FUNCTION__); + do_lit_tasklets(cluster, sched_task); + } + + preempt_enable_no_resched(); +} + + + +static int cedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet) +{ +#if 0 + cedf_domain_t *cluster = NULL; + cpu_entry_t *targetCPU = NULL; + int thisCPU; + int runLocal = 0; + int runNow = 0; + unsigned long flags; + + if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner))) + { + TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); + return 0; + } + + cluster = task_cpu_cluster(tasklet->owner); + + raw_spin_lock_irqsave(&cluster->cluster_lock, flags); + + thisCPU = smp_processor_id(); + +#ifdef CONFIG_SCHED_CPU_AFFINITY + { + cpu_entry_t* affinity = NULL; + + // use this CPU if it is in our cluster and isn't running any RT work. + if(cpu_isset(thisCPU, *cluster->cpu_map) && (__get_cpu_var(cedf_cpu_entries).linked == NULL)) { + affinity = &(__get_cpu_var(cedf_cpu_entries)); + } + else { + // this CPU is busy or shouldn't run tasklet in this cluster. + // look for available near by CPUs. + // NOTE: Affinity towards owner and not this CPU. Is this right? + affinity = + cedf_get_nearest_available_cpu(cluster, + &per_cpu(cedf_cpu_entries, task_cpu(tasklet->owner))); + } + + targetCPU = affinity; + } +#endif + + if (targetCPU == NULL) { + targetCPU = lowest_prio_cpu(cluster); + } + + if (edf_higher_prio(tasklet->owner, targetCPU->linked)) { + if (thisCPU == targetCPU->cpu) { + TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__); + runLocal = 1; + runNow = 1; + } + else { + TRACE("%s: Run tasklet remotely (and now).\n", __FUNCTION__); + runLocal = 0; + runNow = 1; + } + } + else { + runLocal = 0; + runNow = 0; + } + + if(!runLocal) { + // enqueue the tasklet + __add_pai_tasklet(tasklet, cluster); + } + + raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); + + + if (runLocal /*&& runNow */) { // runNow == 1 is implied + TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__); + __do_lit_tasklet(tasklet, 0ul); + } + else if (runNow /*&& !runLocal */) { // runLocal == 0 is implied + TRACE("%s: Triggering CPU %d to run tasklet.\n", __FUNCTION__, targetCPU->cpu); + preempt(targetCPU); // need to be protected by cluster_lock? + } + else { + TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__); + } +#else + TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__); + __do_lit_tasklet(tasklet, 0ul); +#endif + return(1); // success +} + +static void cedf_change_prio_pai_tasklet(struct task_struct *old_prio, + struct task_struct *new_prio) +{ + struct tasklet_struct* step; + unsigned long flags; + cedf_domain_t *cluster; + struct task_struct *probe; + + // identify the cluster by the assignment of these tasks. one should + // be non-NULL. + probe = (old_prio) ? old_prio : new_prio; + + if(probe) { + cluster = task_cpu_cluster(probe); + + if(cluster->pending_tasklets.head != NULL) { + raw_spin_lock_irqsave(&cluster->cluster_lock, flags); + for(step = cluster->pending_tasklets.head; step != NULL; step = step->next) { + if(step->owner == old_prio) { + TRACE("%s: Found tasklet to change: %d\n", __FUNCTION__, step->owner->pid); + step->owner = new_prio; + } + } + raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); + } + } + else { + TRACE("%s: Both priorities were NULL\n"); + } +} + +#endif // PAI + +/* Getting schedule() right is a bit tricky. schedule() may not make any + * assumptions on the state of the current task since it may be called for a + * number of reasons. The reasons include a scheduler_tick() determined that it + * was necessary, because sys_exit_np() was called, because some Linux + * subsystem determined so, or even (in the worst case) because there is a bug + * hidden somewhere. Thus, we must take extreme care to determine what the + * current state is. + * + * The CPU could currently be scheduling a task (or not), be linked (or not). + * + * The following assertions for the scheduled task could hold: + * + * - !is_running(scheduled) // the job blocks + * - scheduled->timeslice == 0 // the job completed (forcefully) + * - get_rt_flag() == RT_F_SLEEP // the job completed (by syscall) + * - linked != scheduled // we need to reschedule (for any reason) + * - is_np(scheduled) // rescheduling must be delayed, + * sys_exit_np must be requested + * + * Any of these can occur together. + */ +static struct task_struct* cedf_schedule(struct task_struct * prev) +{ + cpu_entry_t* entry = &__get_cpu_var(cedf_cpu_entries); + cedf_domain_t *cluster = entry->cluster; + int out_of_time, sleep, preempt, np, exists, blocks; + struct task_struct* next = NULL; + +#ifdef CONFIG_RELEASE_MASTER + /* Bail out early if we are the release master. + * The release master never schedules any real-time tasks. + */ + if (unlikely(cluster->domain.release_master == entry->cpu)) { + sched_state_task_picked(); + return NULL; + } +#endif + + raw_spin_lock(&cluster->cluster_lock); + clear_will_schedule(); + + /* sanity checking */ + BUG_ON(entry->scheduled && entry->scheduled != prev); + BUG_ON(entry->scheduled && !is_realtime(prev)); + BUG_ON(is_realtime(prev) && !entry->scheduled); + + /* (0) Determine state */ + exists = entry->scheduled != NULL; + blocks = exists && !is_running(entry->scheduled); + out_of_time = exists && + budget_enforced(entry->scheduled) && + budget_exhausted(entry->scheduled); + np = exists && is_np(entry->scheduled); + sleep = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP; + preempt = entry->scheduled != entry->linked; + +#ifdef WANT_ALL_SCHED_EVENTS + TRACE_TASK(prev, "invoked cedf_schedule.\n"); +#endif + + if (exists) + TRACE_TASK(prev, + "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d " + "state:%d sig:%d\n", + blocks, out_of_time, np, sleep, preempt, + prev->state, signal_pending(prev)); + if (entry->linked && preempt) + TRACE_TASK(prev, "will be preempted by %s/%d\n", + entry->linked->comm, entry->linked->pid); + + + /* If a task blocks we have no choice but to reschedule. + */ + if (blocks) + unlink(entry->scheduled); + +#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) + if(exists && is_realtime(entry->scheduled) && tsk_rt(entry->scheduled)->held_gpus) { + if(!blocks || tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) { + // don't track preemptions or locking protocol suspensions. + TRACE_TASK(entry->scheduled, "stopping GPU tracker.\n"); + stop_gpu_tracker(entry->scheduled); + } + else if(blocks && !tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) { + TRACE_TASK(entry->scheduled, "GPU tracker remains on during suspension.\n"); + } + } +#endif + + /* Request a sys_exit_np() call if we would like to preempt but cannot. + * We need to make sure to update the link structure anyway in case + * that we are still linked. Multiple calls to request_exit_np() don't + * hurt. + */ + if (np && (out_of_time || preempt || sleep)) { + unlink(entry->scheduled); + request_exit_np(entry->scheduled); + } + + /* Any task that is preemptable and either exhausts its execution + * budget or wants to sleep completes. We may have to reschedule after + * this. Don't do a job completion if we block (can't have timers running + * for blocked jobs). Preemption go first for the same reason. + */ + if (!np && (out_of_time || sleep) && !blocks && !preempt) + job_completion(entry->scheduled, !sleep); + + /* Link pending task if we became unlinked. + */ + if (!entry->linked) + link_task_to_cpu(__take_ready(&cluster->domain), entry); + + /* The final scheduling decision. Do we need to switch for some reason? + * If linked is different from scheduled, then select linked as next. + */ + if ((!np || blocks) && + entry->linked != entry->scheduled) { + /* Schedule a linked job? */ + if (entry->linked) { + entry->linked->rt_param.scheduled_on = entry->cpu; + next = entry->linked; + } + if (entry->scheduled) { + /* not gonna be scheduled soon */ + entry->scheduled->rt_param.scheduled_on = NO_CPU; + TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n"); + } + } else + /* Only override Linux scheduler if we have a real-time task + * scheduled that needs to continue. + */ + if (exists) + next = prev; + + sched_state_task_picked(); + raw_spin_unlock(&cluster->cluster_lock); + +#ifdef WANT_ALL_SCHED_EVENTS + TRACE("cluster_lock released, next=0x%p\n", next); + + if (next) + TRACE_TASK(next, "scheduled at %llu\n", litmus_clock()); + else if (exists && !next) + TRACE("becomes idle at %llu.\n", litmus_clock()); +#endif + + return next; +} + + +/* _finish_switch - we just finished the switch away from prev + */ +static void cedf_finish_switch(struct task_struct *prev) +{ + cpu_entry_t* entry = &__get_cpu_var(cedf_cpu_entries); + + entry->scheduled = is_realtime(current) ? current : NULL; +#ifdef WANT_ALL_SCHED_EVENTS + TRACE_TASK(prev, "switched away from\n"); +#endif +} + + +/* Prepare a task for running in RT mode + */ +static void cedf_task_new(struct task_struct * t, int on_rq, int running) +{ + unsigned long flags; + cpu_entry_t* entry; + cedf_domain_t* cluster; + + TRACE("c-edf: task new %d\n", t->pid); + + /* the cluster doesn't change even if t is running */ + cluster = task_cpu_cluster(t); + + raw_spin_lock_irqsave(&cluster->cluster_lock, flags); + + /* setup job params */ + release_at(t, litmus_clock()); + + if (running) { + entry = &per_cpu(cedf_cpu_entries, task_cpu(t)); + BUG_ON(entry->scheduled); + +#ifdef CONFIG_RELEASE_MASTER + if (entry->cpu != cluster->domain.release_master) { +#endif + entry->scheduled = t; + tsk_rt(t)->scheduled_on = task_cpu(t); +#ifdef CONFIG_RELEASE_MASTER + } else { + /* do not schedule on release master */ + preempt(entry); /* force resched */ + tsk_rt(t)->scheduled_on = NO_CPU; + } +#endif + } else { + t->rt_param.scheduled_on = NO_CPU; + } + t->rt_param.linked_on = NO_CPU; + + cedf_job_arrival(t); + raw_spin_unlock_irqrestore(&(cluster->cluster_lock), flags); +} + +static void cedf_task_wake_up(struct task_struct *task) +{ + unsigned long flags; + //lt_t now; + cedf_domain_t *cluster; + + TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); + + cluster = task_cpu_cluster(task); + + raw_spin_lock_irqsave(&cluster->cluster_lock, flags); + +#if 0 // sproadic task model + /* We need to take suspensions because of semaphores into + * account! If a job resumes after being suspended due to acquiring + * a semaphore, it should never be treated as a new job release. + */ + if (get_rt_flags(task) == RT_F_EXIT_SEM) { + set_rt_flags(task, RT_F_RUNNING); + } else { + now = litmus_clock(); + if (is_tardy(task, now)) { + /* new sporadic release */ + release_at(task, now); + sched_trace_task_release(task); + } + else { + if (task->rt.time_slice) { + /* came back in time before deadline + */ + set_rt_flags(task, RT_F_RUNNING); + } + } + } +#else + set_rt_flags(task, RT_F_RUNNING); // periodic model +#endif + + if(tsk_rt(task)->linked_on == NO_CPU) + cedf_job_arrival(task); + + raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); +} + +static void cedf_task_block(struct task_struct *t) +{ + unsigned long flags; + cedf_domain_t *cluster; + + TRACE_TASK(t, "block at %llu\n", litmus_clock()); + + cluster = task_cpu_cluster(t); + + /* unlink if necessary */ + raw_spin_lock_irqsave(&cluster->cluster_lock, flags); + unlink(t); + raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); + + BUG_ON(!is_realtime(t)); +} + + +static void cedf_task_exit(struct task_struct * t) +{ + unsigned long flags; + cedf_domain_t *cluster = task_cpu_cluster(t); + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + cedf_change_prio_pai_tasklet(t, NULL); +#endif + + /* unlink if necessary */ + raw_spin_lock_irqsave(&cluster->cluster_lock, flags); + unlink(t); + if (tsk_rt(t)->scheduled_on != NO_CPU) { + cpu_entry_t *cpu; + cpu = &per_cpu(cedf_cpu_entries, tsk_rt(t)->scheduled_on); + cpu->scheduled = NULL; + tsk_rt(t)->scheduled_on = NO_CPU; + } + raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); + + BUG_ON(!is_realtime(t)); + TRACE_TASK(t, "RIP\n"); +} + +static long cedf_admit_task(struct task_struct* tsk) +{ +#ifdef CONFIG_LITMUS_NESTED_LOCKING + INIT_BINHEAP_HANDLE(&tsk_rt(tsk)->hp_blocked_tasks, + edf_max_heap_base_priority_order); +#endif + + return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL; +} + + + +#ifdef CONFIG_LITMUS_LOCKING + +#include + + + +/* called with IRQs off */ +static void __increase_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh) +{ + int linked_on; + int check_preempt = 0; + + cedf_domain_t* cluster = task_cpu_cluster(t); + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + /* this sanity check allows for weaker locking in protocols */ + /* TODO (klitirqd): Skip this check if 't' is a proxy thread (???) */ + if(__edf_higher_prio(prio_inh, BASE, t, EFFECTIVE)) { +#endif + TRACE_TASK(t, "inherits priority from %s/%d\n", + prio_inh->comm, prio_inh->pid); + tsk_rt(t)->inh_task = prio_inh; + + linked_on = tsk_rt(t)->linked_on; + + /* If it is scheduled, then we need to reorder the CPU heap. */ + if (linked_on != NO_CPU) { + TRACE_TASK(t, "%s: linked on %d\n", + __FUNCTION__, linked_on); + /* Holder is scheduled; need to re-order CPUs. + * We can't use heap_decrease() here since + * the cpu_heap is ordered in reverse direction, so + * it is actually an increase. */ + binheap_delete(&per_cpu(cedf_cpu_entries, linked_on).hn, + &cluster->cpu_heap); + binheap_add(&per_cpu(cedf_cpu_entries, linked_on).hn, + &cluster->cpu_heap, cpu_entry_t, hn); + + } else { + /* holder may be queued: first stop queue changes */ + raw_spin_lock(&cluster->domain.release_lock); + if (is_queued(t)) { + TRACE_TASK(t, "%s: is queued\n", + __FUNCTION__); + /* We need to update the position of holder in some + * heap. Note that this could be a release heap if we + * budget enforcement is used and this job overran. */ + check_preempt = + !bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node); + } else { + /* Nothing to do: if it is not queued and not linked + * then it is either sleeping or currently being moved + * by other code (e.g., a timer interrupt handler) that + * will use the correct priority when enqueuing the + * task. */ + TRACE_TASK(t, "%s: is NOT queued => Done.\n", + __FUNCTION__); + } + raw_spin_unlock(&cluster->domain.release_lock); + + /* If holder was enqueued in a release heap, then the following + * preemption check is pointless, but we can't easily detect + * that case. If you want to fix this, then consider that + * simply adding a state flag requires O(n) time to update when + * releasing n tasks, which conflicts with the goal to have + * O(log n) merges. */ + if (check_preempt) { + /* heap_decrease() hit the top level of the heap: make + * sure preemption checks get the right task, not the + * potentially stale cache. */ + bheap_uncache_min(edf_ready_order, + &cluster->domain.ready_queue); + check_for_preemptions(cluster); + } + } +#ifdef CONFIG_LITMUS_NESTED_LOCKING + } + else { + TRACE_TASK(t, "Spurious invalid priority increase. " + "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n" + "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n", + t->comm, t->pid, + effective_priority(t)->comm, effective_priority(t)->pid, + (prio_inh) ? prio_inh->comm : "nil", + (prio_inh) ? prio_inh->pid : -1); + WARN_ON(!prio_inh); + } +#endif +} + +/* called with IRQs off */ +static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) +{ + cedf_domain_t* cluster = task_cpu_cluster(t); + + raw_spin_lock(&cluster->cluster_lock); + + __increase_priority_inheritance(t, prio_inh); + +#ifdef CONFIG_LITMUS_SOFTIRQD + if(tsk_rt(t)->cur_klitirqd != NULL) + { + TRACE_TASK(t, "%s/%d inherits a new priority!\n", + tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); + + __increase_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); + } +#endif + + raw_spin_unlock(&cluster->cluster_lock); + +#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA) + if(tsk_rt(t)->held_gpus) { + int i; + for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus)); + i < NV_DEVICE_NUM; + i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) { + pai_check_priority_increase(t, i); + } + } +#endif +} + +/* called with IRQs off */ +static void __decrease_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh) +{ +#ifdef CONFIG_LITMUS_NESTED_LOCKING + if(__edf_higher_prio(t, EFFECTIVE, prio_inh, BASE)) { +#endif + /* A job only stops inheriting a priority when it releases a + * resource. Thus we can make the following assumption.*/ + if(prio_inh) + TRACE_TASK(t, "EFFECTIVE priority decreased to %s/%d\n", + prio_inh->comm, prio_inh->pid); + else + TRACE_TASK(t, "base priority restored.\n"); + + tsk_rt(t)->inh_task = prio_inh; + + if(tsk_rt(t)->scheduled_on != NO_CPU) { + TRACE_TASK(t, "is scheduled.\n"); + + /* Check if rescheduling is necessary. We can't use heap_decrease() + * since the priority was effectively lowered. */ + unlink(t); + cedf_job_arrival(t); + } + else { + cedf_domain_t* cluster = task_cpu_cluster(t); + /* task is queued */ + raw_spin_lock(&cluster->domain.release_lock); + if (is_queued(t)) { + TRACE_TASK(t, "is queued.\n"); + + /* decrease in priority, so we have to re-add to binomial heap */ + unlink(t); + cedf_job_arrival(t); + } + else { + TRACE_TASK(t, "is not in scheduler. Probably on wait queue somewhere.\n"); + } + raw_spin_unlock(&cluster->domain.release_lock); + } +#ifdef CONFIG_LITMUS_NESTED_LOCKING + } + else { + TRACE_TASK(t, "Spurious invalid priority decrease. " + "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n" + "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n", + t->comm, t->pid, + effective_priority(t)->comm, effective_priority(t)->pid, + (prio_inh) ? prio_inh->comm : "nil", + (prio_inh) ? prio_inh->pid : -1); + } +#endif +} + +static void decrease_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh) +{ + cedf_domain_t* cluster = task_cpu_cluster(t); + + raw_spin_lock(&cluster->cluster_lock); + __decrease_priority_inheritance(t, prio_inh); + +#ifdef CONFIG_LITMUS_SOFTIRQD + if(tsk_rt(t)->cur_klitirqd != NULL) + { + TRACE_TASK(t, "%s/%d decreases in priority!\n", + tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); + + __decrease_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); + } +#endif + + raw_spin_unlock(&cluster->cluster_lock); + +#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA) + if(tsk_rt(t)->held_gpus) { + int i; + for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus)); + i < NV_DEVICE_NUM; + i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) { + pai_check_priority_decrease(t, i); + } + } +#endif +} + + + + + +#ifdef CONFIG_LITMUS_SOFTIRQD +/* called with IRQs off */ +static void increase_priority_inheritance_klitirqd(struct task_struct* klitirqd, + struct task_struct* old_owner, + struct task_struct* new_owner) +{ + cedf_domain_t* cluster = task_cpu_cluster(klitirqd); + + BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); + + raw_spin_lock(&cluster->cluster_lock); + + if(old_owner != new_owner) + { + if(old_owner) + { + // unreachable? + tsk_rt(old_owner)->cur_klitirqd = NULL; + } + + TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n", + new_owner->comm, new_owner->pid); + + tsk_rt(new_owner)->cur_klitirqd = klitirqd; + } + + __decrease_priority_inheritance(klitirqd, NULL); // kludge to clear out cur prio. + + __increase_priority_inheritance(klitirqd, + (tsk_rt(new_owner)->inh_task == NULL) ? + new_owner : + tsk_rt(new_owner)->inh_task); + + raw_spin_unlock(&cluster->cluster_lock); +} + + +/* called with IRQs off */ +static void decrease_priority_inheritance_klitirqd(struct task_struct* klitirqd, + struct task_struct* old_owner, + struct task_struct* new_owner) +{ + cedf_domain_t* cluster = task_cpu_cluster(klitirqd); + + BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); + + raw_spin_lock(&cluster->cluster_lock); + + TRACE_TASK(klitirqd, "priority restored\n"); + + __decrease_priority_inheritance(klitirqd, new_owner); + + tsk_rt(old_owner)->cur_klitirqd = NULL; + + raw_spin_unlock(&cluster->cluster_lock); +} +#endif // CONFIG_LITMUS_SOFTIRQD + + + + + + + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + +/* called with IRQs off */ +/* preconditions: + (1) The 'hp_blocked_tasks_lock' of task 't' is held. + (2) The lock 'to_unlock' is held. + */ +static void nested_increase_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh, + raw_spinlock_t *to_unlock, + unsigned long irqflags) +{ + struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock; + + if(tsk_rt(t)->inh_task != prio_inh) { // shield redundent calls. + increase_priority_inheritance(t, prio_inh); // increase our prio. + } + + raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); // unlock the t's heap. + + + if(blocked_lock) { + if(blocked_lock->ops->propagate_increase_inheritance) { + TRACE_TASK(t, "Inheritor is blocked (...perhaps). Checking lock %d.\n", + blocked_lock->ident); + + // beware: recursion + blocked_lock->ops->propagate_increase_inheritance(blocked_lock, + t, to_unlock, + irqflags); + } + else { + TRACE_TASK(t, "Inheritor is blocked on lock (%d) that does not support nesting!\n", + blocked_lock->ident); + unlock_fine_irqrestore(to_unlock, irqflags); + } + } + else { + TRACE_TASK(t, "is not blocked. No propagation.\n"); + unlock_fine_irqrestore(to_unlock, irqflags); + } +} + +/* called with IRQs off */ +/* preconditions: + (1) The 'hp_blocked_tasks_lock' of task 't' is held. + (2) The lock 'to_unlock' is held. + */ +static void nested_decrease_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh, + raw_spinlock_t *to_unlock, + unsigned long irqflags) +{ + struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock; + decrease_priority_inheritance(t, prio_inh); + + raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); // unlock the t's heap. + + if(blocked_lock) { + if(blocked_lock->ops->propagate_decrease_inheritance) { + TRACE_TASK(t, "Inheritor is blocked (...perhaps). Checking lock %d.\n", + blocked_lock->ident); + + // beware: recursion + blocked_lock->ops->propagate_decrease_inheritance(blocked_lock, t, + to_unlock, + irqflags); + } + else { + TRACE_TASK(t, "Inheritor is blocked on lock (%p) that does not support nesting!\n", + blocked_lock); + unlock_fine_irqrestore(to_unlock, irqflags); + } + } + else { + TRACE_TASK(t, "is not blocked. No propagation.\n"); + unlock_fine_irqrestore(to_unlock, irqflags); + } +} + + +/* ******************** RSM MUTEX ********************** */ + +static struct litmus_lock_ops cedf_rsm_mutex_lock_ops = { + .lock = rsm_mutex_lock, + .unlock = rsm_mutex_unlock, + .close = rsm_mutex_close, + .deallocate = rsm_mutex_free, + + .propagate_increase_inheritance = rsm_mutex_propagate_increase_inheritance, + .propagate_decrease_inheritance = rsm_mutex_propagate_decrease_inheritance, + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + .dgl_lock = rsm_mutex_dgl_lock, + .is_owner = rsm_mutex_is_owner, + .enable_priority = rsm_mutex_enable_priority, +#endif +}; + +static struct litmus_lock* cedf_new_rsm_mutex(void) +{ + return rsm_mutex_new(&cedf_rsm_mutex_lock_ops); +} + +/* ******************** IKGLP ********************** */ + +static struct litmus_lock_ops cedf_ikglp_lock_ops = { + .lock = ikglp_lock, + .unlock = ikglp_unlock, + .close = ikglp_close, + .deallocate = ikglp_free, + + // ikglp can only be an outer-most lock. + .propagate_increase_inheritance = NULL, + .propagate_decrease_inheritance = NULL, +}; + +static struct litmus_lock* cedf_new_ikglp(void* __user arg) +{ + // assumes clusters of uniform size. + return ikglp_new(cluster_size/num_clusters, &cedf_ikglp_lock_ops, arg); +} + +#endif /* CONFIG_LITMUS_NESTED_LOCKING */ + + + + +/* ******************** KFMLP support ********************** */ + +static struct litmus_lock_ops cedf_kfmlp_lock_ops = { + .lock = kfmlp_lock, + .unlock = kfmlp_unlock, + .close = kfmlp_close, + .deallocate = kfmlp_free, + + // kfmlp can only be an outer-most lock. + .propagate_increase_inheritance = NULL, + .propagate_decrease_inheritance = NULL, +}; + + +static struct litmus_lock* cedf_new_kfmlp(void* __user arg) +{ + return kfmlp_new(&cedf_kfmlp_lock_ops, arg); +} + + +/* **** lock constructor **** */ + +static long cedf_allocate_lock(struct litmus_lock **lock, int type, + void* __user args) +{ + int err; + + switch (type) { +#ifdef CONFIG_LITMUS_NESTED_LOCKING + case RSM_MUTEX: + *lock = cedf_new_rsm_mutex(); + break; + + case IKGLP_SEM: + *lock = cedf_new_ikglp(args); + break; +#endif + case KFMLP_SEM: + *lock = cedf_new_kfmlp(args); + break; + + default: + err = -ENXIO; + goto UNSUPPORTED_LOCK; + }; + + if (*lock) + err = 0; + else + err = -ENOMEM; + +UNSUPPORTED_LOCK: + return err; +} + +#endif // CONFIG_LITMUS_LOCKING + + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING +static struct affinity_observer_ops cedf_kfmlp_affinity_ops = { + .close = kfmlp_aff_obs_close, + .deallocate = kfmlp_aff_obs_free, +}; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING +static struct affinity_observer_ops cedf_ikglp_affinity_ops = { + .close = ikglp_aff_obs_close, + .deallocate = ikglp_aff_obs_free, +}; +#endif + +static long cedf_allocate_affinity_observer(struct affinity_observer **aff_obs, + int type, + void* __user args) +{ + int err; + + switch (type) { + + case KFMLP_SIMPLE_GPU_AFF_OBS: + *aff_obs = kfmlp_simple_gpu_aff_obs_new(&cedf_kfmlp_affinity_ops, args); + break; + + case KFMLP_GPU_AFF_OBS: + *aff_obs = kfmlp_gpu_aff_obs_new(&cedf_kfmlp_affinity_ops, args); + break; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + case IKGLP_SIMPLE_GPU_AFF_OBS: + *aff_obs = ikglp_simple_gpu_aff_obs_new(&cedf_ikglp_affinity_ops, args); + break; + + case IKGLP_GPU_AFF_OBS: + *aff_obs = ikglp_gpu_aff_obs_new(&cedf_ikglp_affinity_ops, args); + break; +#endif + default: + err = -ENXIO; + goto UNSUPPORTED_AFF_OBS; + }; + + if (*aff_obs) + err = 0; + else + err = -ENOMEM; + +UNSUPPORTED_AFF_OBS: + return err; +} +#endif + + + + +#ifdef VERBOSE_INIT +static void print_cluster_topology(cpumask_var_t mask, int cpu) +{ + int chk; + char buf[255]; + + chk = cpulist_scnprintf(buf, 254, mask); + buf[chk] = '\0'; + printk(KERN_INFO "CPU = %d, shared cpu(s) = %s\n", cpu, buf); + +} +#endif + +static void cleanup_cedf(void) +{ + int i; + +#ifdef CONFIG_LITMUS_NVIDIA + shutdown_nvidia_info(); +#endif + + if (clusters_allocated) { + for (i = 0; i < num_clusters; i++) { + kfree(cedf[i].cpus); + free_cpumask_var(cedf[i].cpu_map); + } + + kfree(cedf); + } +} + +static long cedf_activate_plugin(void) +{ + int i, j, cpu, ccpu, cpu_count; + cpu_entry_t *entry; + + cpumask_var_t mask; + int chk = 0; + + /* de-allocate old clusters, if any */ + cleanup_cedf(); + + printk(KERN_INFO "C-EDF: Activate Plugin, cluster configuration = %d\n", + cluster_config); + + /* need to get cluster_size first */ + if(!zalloc_cpumask_var(&mask, GFP_ATOMIC)) + return -ENOMEM; + + if (unlikely(cluster_config == GLOBAL_CLUSTER)) { + cluster_size = num_online_cpus(); + } else { + chk = get_shared_cpu_map(mask, 0, cluster_config); + if (chk) { + /* if chk != 0 then it is the max allowed index */ + printk(KERN_INFO "C-EDF: Cluster configuration = %d " + "is not supported on this hardware.\n", + cluster_config); + /* User should notice that the configuration failed, so + * let's bail out. */ + return -EINVAL; + } + + cluster_size = cpumask_weight(mask); + } + + if ((num_online_cpus() % cluster_size) != 0) { + /* this can't be right, some cpus are left out */ + printk(KERN_ERR "C-EDF: Trying to group %d cpus in %d!\n", + num_online_cpus(), cluster_size); + return -1; + } + + num_clusters = num_online_cpus() / cluster_size; + printk(KERN_INFO "C-EDF: %d cluster(s) of size = %d\n", + num_clusters, cluster_size); + + /* initialize clusters */ + cedf = kmalloc(num_clusters * sizeof(cedf_domain_t), GFP_ATOMIC); + for (i = 0; i < num_clusters; i++) { + + cedf[i].cpus = kmalloc(cluster_size * sizeof(cpu_entry_t), + GFP_ATOMIC); + INIT_BINHEAP_HANDLE(&(cedf[i].cpu_heap), cpu_lower_prio); + edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs); + + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + cedf[i].pending_tasklets.head = NULL; + cedf[i].pending_tasklets.tail = &(cedf[i].pending_tasklets.head); +#endif + + + if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC)) + return -ENOMEM; +#ifdef CONFIG_RELEASE_MASTER + cedf[i].domain.release_master = atomic_read(&release_master_cpu); +#endif + } + + /* cycle through cluster and add cpus to them */ + for (i = 0; i < num_clusters; i++) { + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + raw_spin_lock_init(&cedf[i].dgl_lock); +#endif + + for_each_online_cpu(cpu) { + /* check if the cpu is already in a cluster */ + for (j = 0; j < num_clusters; j++) + if (cpumask_test_cpu(cpu, cedf[j].cpu_map)) + break; + /* if it is in a cluster go to next cpu */ + if (j < num_clusters && + cpumask_test_cpu(cpu, cedf[j].cpu_map)) + continue; + + /* this cpu isn't in any cluster */ + /* get the shared cpus */ + if (unlikely(cluster_config == GLOBAL_CLUSTER)) + cpumask_copy(mask, cpu_online_mask); + else + get_shared_cpu_map(mask, cpu, cluster_config); + + cpumask_copy(cedf[i].cpu_map, mask); +#ifdef VERBOSE_INIT + print_cluster_topology(mask, cpu); +#endif + /* add cpus to current cluster and init cpu_entry_t */ + cpu_count = 0; + for_each_cpu(ccpu, cedf[i].cpu_map) { + + entry = &per_cpu(cedf_cpu_entries, ccpu); + cedf[i].cpus[cpu_count] = entry; + atomic_set(&entry->will_schedule, 0); + entry->cpu = ccpu; + entry->cluster = &cedf[i]; + + INIT_BINHEAP_NODE(&entry->hn); + + cpu_count++; + + entry->linked = NULL; + entry->scheduled = NULL; +#ifdef CONFIG_RELEASE_MASTER + /* only add CPUs that should schedule jobs */ + if (entry->cpu != entry->cluster->domain.release_master) +#endif + update_cpu_position(entry); + } + /* done with this cluster */ + break; + } + } + +#ifdef CONFIG_LITMUS_SOFTIRQD + { + /* distribute the daemons evenly across the clusters. */ + int* affinity = kmalloc(NR_LITMUS_SOFTIRQD * sizeof(int), GFP_ATOMIC); + int num_daemons_per_cluster = NR_LITMUS_SOFTIRQD / num_clusters; + int left_over = NR_LITMUS_SOFTIRQD % num_clusters; + + int daemon = 0; + for(i = 0; i < num_clusters; ++i) + { + int num_on_this_cluster = num_daemons_per_cluster; + if(left_over) + { + ++num_on_this_cluster; + --left_over; + } + + for(j = 0; j < num_on_this_cluster; ++j) + { + // first CPU of this cluster + affinity[daemon++] = i*cluster_size; + } + } + + spawn_klitirqd(affinity); + + kfree(affinity); + } +#endif + +#ifdef CONFIG_LITMUS_NVIDIA + init_nvidia_info(); +#endif + + free_cpumask_var(mask); + clusters_allocated = 1; + return 0; +} + +/* Plugin object */ +static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = { + .plugin_name = "C-EDF", + .finish_switch = cedf_finish_switch, + .tick = cedf_tick, + .task_new = cedf_task_new, + .complete_job = complete_job, + .task_exit = cedf_task_exit, + .schedule = cedf_schedule, + .task_wake_up = cedf_task_wake_up, + .task_block = cedf_task_block, + .admit_task = cedf_admit_task, + .activate_plugin = cedf_activate_plugin, + .compare = edf_higher_prio, +#ifdef CONFIG_LITMUS_LOCKING + .allocate_lock = cedf_allocate_lock, + .increase_prio = increase_priority_inheritance, + .decrease_prio = decrease_priority_inheritance, +#endif +#ifdef CONFIG_LITMUS_NESTED_LOCKING + .nested_increase_prio = nested_increase_priority_inheritance, + .nested_decrease_prio = nested_decrease_priority_inheritance, + .__compare = __edf_higher_prio, +#endif +#ifdef CONFIG_LITMUS_DGL_SUPPORT + .get_dgl_spinlock = cedf_get_dgl_spinlock, +#endif +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + .allocate_aff_obs = cedf_allocate_affinity_observer, +#endif +#ifdef CONFIG_LITMUS_SOFTIRQD + .increase_prio_klitirqd = increase_priority_inheritance_klitirqd, + .decrease_prio_klitirqd = decrease_priority_inheritance_klitirqd, +#endif +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + .enqueue_pai_tasklet = cedf_enqueue_pai_tasklet, + .change_prio_pai_tasklet = cedf_change_prio_pai_tasklet, + .run_tasklets = cedf_run_tasklets, +#endif +}; + +static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL; + +static int __init init_cedf(void) +{ + int err, fs; + + err = register_sched_plugin(&cedf_plugin); + if (!err) { + fs = make_plugin_proc_dir(&cedf_plugin, &cedf_dir); + if (!fs) + cluster_file = create_cluster_file(cedf_dir, &cluster_config); + else + printk(KERN_ERR "Could not allocate C-EDF procfs dir.\n"); + } + return err; +} + +static void clean_cedf(void) +{ + cleanup_cedf(); + if (cluster_file) + remove_proc_entry("cluster", cedf_dir); + if (cedf_dir) + remove_plugin_proc_dir(&cedf_plugin); +} + +module_init(init_cedf); +module_exit(clean_cedf); diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c new file mode 100644 index 0000000..8c48757 --- /dev/null +++ b/litmus/sched_gsn_edf.c @@ -0,0 +1,1862 @@ +/* + * litmus/sched_gsn_edf.c + * + * Implementation of the GSN-EDF scheduling algorithm. + * + * This version uses the simple approach and serializes all scheduling + * decisions by the use of a queue lock. This is probably not the + * best way to do it, but it should suffice for now. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +#ifdef CONFIG_LITMUS_LOCKING +#include +#endif + +#ifdef CONFIG_LITMUS_NESTED_LOCKING +#include +#include +#endif + +#ifdef CONFIG_SCHED_CPU_AFFINITY +#include +#endif + +#ifdef CONFIG_LITMUS_SOFTIRQD +#include +#endif + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD +#include +#include +#endif + +#ifdef CONFIG_LITMUS_NVIDIA +#include +#endif + +#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) +#include +#endif + +/* Overview of GSN-EDF operations. + * + * For a detailed explanation of GSN-EDF have a look at the FMLP paper. This + * description only covers how the individual operations are implemented in + * LITMUS. + * + * link_task_to_cpu(T, cpu) - Low-level operation to update the linkage + * structure (NOT the actually scheduled + * task). If there is another linked task To + * already it will set To->linked_on = NO_CPU + * (thereby removing its association with this + * CPU). However, it will not requeue the + * previously linked task (if any). It will set + * T's state to RT_F_RUNNING and check whether + * it is already running somewhere else. If T + * is scheduled somewhere else it will link + * it to that CPU instead (and pull the linked + * task to cpu). T may be NULL. + * + * unlink(T) - Unlink removes T from all scheduler data + * structures. If it is linked to some CPU it + * will link NULL to that CPU. If it is + * currently queued in the gsnedf queue it will + * be removed from the rt_domain. It is safe to + * call unlink(T) if T is not linked. T may not + * be NULL. + * + * requeue(T) - Requeue will insert T into the appropriate + * queue. If the system is in real-time mode and + * the T is released already, it will go into the + * ready queue. If the system is not in + * real-time mode is T, then T will go into the + * release queue. If T's release time is in the + * future, it will go into the release + * queue. That means that T's release time/job + * no/etc. has to be updated before requeu(T) is + * called. It is not safe to call requeue(T) + * when T is already queued. T may not be NULL. + * + * gsnedf_job_arrival(T) - This is the catch all function when T enters + * the system after either a suspension or at a + * job release. It will queue T (which means it + * is not safe to call gsnedf_job_arrival(T) if + * T is already queued) and then check whether a + * preemption is necessary. If a preemption is + * necessary it will update the linkage + * accordingly and cause scheduled to be called + * (either with an IPI or need_resched). It is + * safe to call gsnedf_job_arrival(T) if T's + * next job has not been actually released yet + * (releast time in the future). T will be put + * on the release queue in that case. + * + * job_completion(T) - Take care of everything that needs to be done + * to prepare T for its next release and place + * it in the right queue with + * gsnedf_job_arrival(). + * + * + * When we now that T is linked to CPU then link_task_to_cpu(NULL, CPU) is + * equivalent to unlink(T). Note that if you unlink a task from a CPU none of + * the functions will automatically propagate pending task from the ready queue + * to a linked task. This is the job of the calling function ( by means of + * __take_ready). + */ + + +/* cpu_entry_t - maintain the linked and scheduled state + */ +typedef struct { + int cpu; + struct task_struct* linked; /* only RT tasks */ + struct task_struct* scheduled; /* only RT tasks */ + struct binheap_node hn; +} cpu_entry_t; +DEFINE_PER_CPU(cpu_entry_t, gsnedf_cpu_entries); + +cpu_entry_t* gsnedf_cpus[NR_CPUS]; + +/* the cpus queue themselves according to priority in here */ +static struct binheap_handle gsnedf_cpu_heap; + +static rt_domain_t gsnedf; +#define gsnedf_lock (gsnedf.ready_lock) + +#ifdef CONFIG_LITMUS_DGL_SUPPORT +static raw_spinlock_t dgl_lock; + +static raw_spinlock_t* gsnedf_get_dgl_spinlock(struct task_struct *t) +{ + return(&dgl_lock); +} +#endif + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD +struct tasklet_head +{ + struct tasklet_struct *head; + struct tasklet_struct **tail; +}; + +struct tasklet_head gsnedf_pending_tasklets; +#endif + + +/* Uncomment this if you want to see all scheduling decisions in the + * TRACE() log. +#define WANT_ALL_SCHED_EVENTS + */ + +static int cpu_lower_prio(struct binheap_node *_a, struct binheap_node *_b) +{ + cpu_entry_t *a = binheap_entry(_a, cpu_entry_t, hn); + cpu_entry_t *b = binheap_entry(_b, cpu_entry_t, hn); + + /* Note that a and b are inverted: we want the lowest-priority CPU at + * the top of the heap. + */ + return edf_higher_prio(b->linked, a->linked); +} + + +/* update_cpu_position - Move the cpu entry to the correct place to maintain + * order in the cpu queue. Caller must hold gsnedf lock. + */ +static void update_cpu_position(cpu_entry_t *entry) +{ + if (likely(binheap_is_in_heap(&entry->hn))) { + binheap_delete(&entry->hn, &gsnedf_cpu_heap); + } + binheap_add(&entry->hn, &gsnedf_cpu_heap, cpu_entry_t, hn); +} + +/* caller must hold gsnedf lock */ +static cpu_entry_t* lowest_prio_cpu(void) +{ + return binheap_top_entry(&gsnedf_cpu_heap, cpu_entry_t, hn); +} + + +/* link_task_to_cpu - Update the link of a CPU. + * Handles the case where the to-be-linked task is already + * scheduled on a different CPU. + */ +static noinline void link_task_to_cpu(struct task_struct* linked, + cpu_entry_t *entry) +{ + cpu_entry_t *sched; + struct task_struct* tmp; + int on_cpu; + + BUG_ON(linked && !is_realtime(linked)); + + /* Currently linked task is set to be unlinked. */ + if (entry->linked) { + entry->linked->rt_param.linked_on = NO_CPU; + } + + /* Link new task to CPU. */ + if (linked) { + set_rt_flags(linked, RT_F_RUNNING); + /* handle task is already scheduled somewhere! */ + on_cpu = linked->rt_param.scheduled_on; + if (on_cpu != NO_CPU) { + sched = &per_cpu(gsnedf_cpu_entries, on_cpu); + /* this should only happen if not linked already */ + BUG_ON(sched->linked == linked); + + /* If we are already scheduled on the CPU to which we + * wanted to link, we don't need to do the swap -- + * we just link ourselves to the CPU and depend on + * the caller to get things right. + */ + if (entry != sched) { + TRACE_TASK(linked, + "already scheduled on %d, updating link.\n", + sched->cpu); + tmp = sched->linked; + linked->rt_param.linked_on = sched->cpu; + sched->linked = linked; + update_cpu_position(sched); + linked = tmp; + } + } + if (linked) /* might be NULL due to swap */ + linked->rt_param.linked_on = entry->cpu; + } + entry->linked = linked; +#ifdef WANT_ALL_SCHED_EVENTS + if (linked) + TRACE_TASK(linked, "linked to %d.\n", entry->cpu); + else + TRACE("NULL linked to %d.\n", entry->cpu); +#endif + update_cpu_position(entry); +} + +/* unlink - Make sure a task is not linked any longer to an entry + * where it was linked before. Must hold gsnedf_lock. + */ +static noinline void unlink(struct task_struct* t) +{ + cpu_entry_t *entry; + + if (t->rt_param.linked_on != NO_CPU) { + /* unlink */ + entry = &per_cpu(gsnedf_cpu_entries, t->rt_param.linked_on); + t->rt_param.linked_on = NO_CPU; + link_task_to_cpu(NULL, entry); + } else if (is_queued(t)) { + /* This is an interesting situation: t is scheduled, + * but was just recently unlinked. It cannot be + * linked anywhere else (because then it would have + * been relinked to this CPU), thus it must be in some + * queue. We must remove it from the list in this + * case. + */ + remove(&gsnedf, t); + } +} + + +/* preempt - force a CPU to reschedule + */ +static void preempt(cpu_entry_t *entry) +{ + preempt_if_preemptable(entry->scheduled, entry->cpu); +} + +/* requeue - Put an unlinked task into gsn-edf domain. + * Caller must hold gsnedf_lock. + */ +static noinline void requeue(struct task_struct* task) +{ + BUG_ON(!task); + /* sanity check before insertion */ + BUG_ON(is_queued(task)); + + if (is_released(task, litmus_clock())) + __add_ready(&gsnedf, task); + else { + /* it has got to wait */ + add_release(&gsnedf, task); + } +} + +#ifdef CONFIG_SCHED_CPU_AFFINITY +static cpu_entry_t* gsnedf_get_nearest_available_cpu(cpu_entry_t *start) +{ + cpu_entry_t *affinity; + + get_nearest_available_cpu(affinity, start, gsnedf_cpu_entries, +#ifdef CONFIG_RELEASE_MASTER + gsnedf.release_master +#else + NO_CPU +#endif + ); + + return(affinity); +} +#endif + +/* check for any necessary preemptions */ +static void check_for_preemptions(void) +{ + struct task_struct *task; + cpu_entry_t *last; + + for (last = lowest_prio_cpu(); + edf_preemption_needed(&gsnedf, last->linked); + last = lowest_prio_cpu()) { + /* preemption necessary */ + task = __take_ready(&gsnedf); + TRACE("check_for_preemptions: attempting to link task %d to %d\n", + task->pid, last->cpu); + +#ifdef CONFIG_SCHED_CPU_AFFINITY + { + cpu_entry_t *affinity = + gsnedf_get_nearest_available_cpu( + &per_cpu(gsnedf_cpu_entries, task_cpu(task))); + if (affinity) + last = affinity; + else if (last->linked) + requeue(last->linked); + } +#else + if (last->linked) + requeue(last->linked); +#endif + + link_task_to_cpu(task, last); + preempt(last); + } +} + +/* gsnedf_job_arrival: task is either resumed or released */ +static noinline void gsnedf_job_arrival(struct task_struct* task) +{ + BUG_ON(!task); + + requeue(task); + check_for_preemptions(); +} + +static void gsnedf_release_jobs(rt_domain_t* rt, struct bheap* tasks) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&gsnedf_lock, flags); + + __merge_ready(rt, tasks); + check_for_preemptions(); + + raw_spin_unlock_irqrestore(&gsnedf_lock, flags); +} + +/* caller holds gsnedf_lock */ +static noinline void job_completion(struct task_struct *t, int forced) +{ + BUG_ON(!t); + + sched_trace_task_completion(t, forced); + +#ifdef CONFIG_LITMUS_NVIDIA + atomic_set(&tsk_rt(t)->nv_int_count, 0); +#endif + + TRACE_TASK(t, "job_completion().\n"); + + /* set flags */ + set_rt_flags(t, RT_F_SLEEP); + /* prepare for next period */ + prepare_for_next_period(t); + if (is_released(t, litmus_clock())) + sched_trace_task_release(t); + /* unlink */ + unlink(t); + /* requeue + * But don't requeue a blocking task. */ + if (is_running(t)) + gsnedf_job_arrival(t); +} + +/* gsnedf_tick - this function is called for every local timer + * interrupt. + * + * checks whether the current task has expired and checks + * whether we need to preempt it if it has not expired + */ +static void gsnedf_tick(struct task_struct* t) +{ + if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) { + if (!is_np(t)) { + /* np tasks will be preempted when they become + * preemptable again + */ + litmus_reschedule_local(); + TRACE("gsnedf_scheduler_tick: " + "%d is preemptable " + " => FORCE_RESCHED\n", t->pid); + } else if (is_user_np(t)) { + TRACE("gsnedf_scheduler_tick: " + "%d is non-preemptable, " + "preemption delayed.\n", t->pid); + request_exit_np(t); + } + } +} + + + + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + + +static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flushed) +{ + if (!atomic_read(&tasklet->count)) { + if(tasklet->owner) { + sched_trace_tasklet_begin(tasklet->owner); + } + + if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state)) + { + BUG(); + } + TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n", + __FUNCTION__, + (tasklet->owner) ? tasklet->owner->pid : -1, + (tasklet->owner) ? 0 : 1); + tasklet->func(tasklet->data); + tasklet_unlock(tasklet); + + if(tasklet->owner) { + sched_trace_tasklet_end(tasklet->owner, flushed); + } + } + else { + BUG(); + } +} + +static void do_lit_tasklets(struct task_struct* sched_task) +{ + int work_to_do = 1; + struct tasklet_struct *tasklet = NULL; + unsigned long flags; + + while(work_to_do) { + + TS_NV_SCHED_BOTISR_START; + + // execute one tasklet that has higher priority + raw_spin_lock_irqsave(&gsnedf_lock, flags); + + if(gsnedf_pending_tasklets.head != NULL) { + struct tasklet_struct *prev = NULL; + tasklet = gsnedf_pending_tasklets.head; + + while(tasklet && edf_higher_prio(sched_task, tasklet->owner)) { + prev = tasklet; + tasklet = tasklet->next; + } + + // remove the tasklet from the queue + if(prev) { + prev->next = tasklet->next; + if(prev->next == NULL) { + TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); + gsnedf_pending_tasklets.tail = &(prev); + } + } + else { + gsnedf_pending_tasklets.head = tasklet->next; + if(tasklet->next == NULL) { + TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); + gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); + } + } + } + else { + TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__); + } + + raw_spin_unlock_irqrestore(&gsnedf_lock, flags); + + if(tasklet) { + __do_lit_tasklet(tasklet, 0ul); + tasklet = NULL; + } + else { + work_to_do = 0; + } + + TS_NV_SCHED_BOTISR_END; + } +} + +//static void do_lit_tasklets(struct task_struct* sched_task) +//{ +// int work_to_do = 1; +// struct tasklet_struct *tasklet = NULL; +// //struct tasklet_struct *step; +// unsigned long flags; +// +// while(work_to_do) { +// +// TS_NV_SCHED_BOTISR_START; +// +// // remove tasklet at head of list if it has higher priority. +// raw_spin_lock_irqsave(&gsnedf_lock, flags); +// +// if(gsnedf_pending_tasklets.head != NULL) { +// // remove tasklet at head. +// tasklet = gsnedf_pending_tasklets.head; +// +// if(edf_higher_prio(tasklet->owner, sched_task)) { +// +// if(NULL == tasklet->next) { +// // tasklet is at the head, list only has one element +// TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); +// gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); +// } +// +// // remove the tasklet from the queue +// gsnedf_pending_tasklets.head = tasklet->next; +// +// TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, tasklet->owner->pid); +// } +// else { +// TRACE("%s: Pending tasklet (%d) does not have priority to run on this CPU (%d).\n", __FUNCTION__, tasklet->owner->pid, smp_processor_id()); +// tasklet = NULL; +// } +// } +// else { +// TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__); +// } +// +// raw_spin_unlock_irqrestore(&gsnedf_lock, flags); +// +// TS_NV_SCHED_BOTISR_END; +// +// if(tasklet) { +// __do_lit_tasklet(tasklet, 0ul); +// tasklet = NULL; +// } +// else { +// work_to_do = 0; +// } +// } +// +// //TRACE("%s: exited.\n", __FUNCTION__); +//} + +static void __add_pai_tasklet(struct tasklet_struct* tasklet) +{ + struct tasklet_struct* step; + + tasklet->next = NULL; // make sure there are no old values floating around + + step = gsnedf_pending_tasklets.head; + if(step == NULL) { + TRACE("%s: tasklet queue empty. inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid); + // insert at tail. + *(gsnedf_pending_tasklets.tail) = tasklet; + gsnedf_pending_tasklets.tail = &(tasklet->next); + } + else if((*(gsnedf_pending_tasklets.tail) != NULL) && + edf_higher_prio((*(gsnedf_pending_tasklets.tail))->owner, tasklet->owner)) { + // insert at tail. + TRACE("%s: tasklet belongs at end. inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid); + + *(gsnedf_pending_tasklets.tail) = tasklet; + gsnedf_pending_tasklets.tail = &(tasklet->next); + } + else { + // insert the tasklet somewhere in the middle. + + TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__); + + while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) { + step = step->next; + } + + // insert tasklet right before step->next. + + TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__, tasklet->owner->pid, step->owner->pid, (step->next) ? step->next->owner->pid : -1); + + tasklet->next = step->next; + step->next = tasklet; + + // patch up the head if needed. + if(gsnedf_pending_tasklets.head == step) + { + TRACE("%s: %d is the new tasklet queue head.\n", __FUNCTION__, tasklet->owner->pid); + gsnedf_pending_tasklets.head = tasklet; + } + } +} + +static void gsnedf_run_tasklets(struct task_struct* sched_task) +{ + preempt_disable(); + + if(gsnedf_pending_tasklets.head != NULL) { + TRACE("%s: There are tasklets to process.\n", __FUNCTION__); + do_lit_tasklets(sched_task); + } + + preempt_enable_no_resched(); +} + +static int gsnedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet) +{ + cpu_entry_t *targetCPU = NULL; + int thisCPU; + int runLocal = 0; + int runNow = 0; + unsigned long flags; + + if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner))) + { + TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__); + return 0; + } + + + raw_spin_lock_irqsave(&gsnedf_lock, flags); + + thisCPU = smp_processor_id(); + +#ifdef CONFIG_SCHED_CPU_AFFINITY + { + cpu_entry_t* affinity = NULL; + + // use this CPU if it is in our cluster and isn't running any RT work. + if( +#ifdef CONFIG_RELEASE_MASTER + (thisCPU != gsnedf.release_master) && +#endif + (__get_cpu_var(gsnedf_cpu_entries).linked == NULL)) { + affinity = &(__get_cpu_var(gsnedf_cpu_entries)); + } + else { + // this CPU is busy or shouldn't run tasklet in this cluster. + // look for available near by CPUs. + // NOTE: Affinity towards owner and not this CPU. Is this right? + affinity = + gsnedf_get_nearest_available_cpu( + &per_cpu(gsnedf_cpu_entries, task_cpu(tasklet->owner))); + } + + targetCPU = affinity; + } +#endif + + if (targetCPU == NULL) { + targetCPU = lowest_prio_cpu(); + } + + if (edf_higher_prio(tasklet->owner, targetCPU->linked)) { + if (thisCPU == targetCPU->cpu) { + TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__); + runLocal = 1; + runNow = 1; + } + else { + TRACE("%s: Run tasklet remotely (and now).\n", __FUNCTION__); + runLocal = 0; + runNow = 1; + } + } + else { + runLocal = 0; + runNow = 0; + } + + if(!runLocal) { + // enqueue the tasklet + __add_pai_tasklet(tasklet); + } + + raw_spin_unlock_irqrestore(&gsnedf_lock, flags); + + + if (runLocal /*&& runNow */) { // runNow == 1 is implied + TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__); + __do_lit_tasklet(tasklet, 0ul); + } + else if (runNow /*&& !runLocal */) { // runLocal == 0 is implied + TRACE("%s: Triggering CPU %d to run tasklet.\n", __FUNCTION__, targetCPU->cpu); + preempt(targetCPU); // need to be protected by cedf_lock? + } + else { + TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__); + } + + return(1); // success +} + +static void gsnedf_change_prio_pai_tasklet(struct task_struct *old_prio, + struct task_struct *new_prio) +{ + struct tasklet_struct* step; + unsigned long flags; + + if(gsnedf_pending_tasklets.head != NULL) { + raw_spin_lock_irqsave(&gsnedf_lock, flags); + for(step = gsnedf_pending_tasklets.head; step != NULL; step = step->next) { + if(step->owner == old_prio) { + TRACE("%s: Found tasklet to change: %d\n", __FUNCTION__, step->owner->pid); + step->owner = new_prio; + } + } + raw_spin_unlock_irqrestore(&gsnedf_lock, flags); + } +} + +#endif // end PAI + + +/* Getting schedule() right is a bit tricky. schedule() may not make any + * assumptions on the state of the current task since it may be called for a + * number of reasons. The reasons include a scheduler_tick() determined that it + * was necessary, because sys_exit_np() was called, because some Linux + * subsystem determined so, or even (in the worst case) because there is a bug + * hidden somewhere. Thus, we must take extreme care to determine what the + * current state is. + * + * The CPU could currently be scheduling a task (or not), be linked (or not). + * + * The following assertions for the scheduled task could hold: + * + * - !is_running(scheduled) // the job blocks + * - scheduled->timeslice == 0 // the job completed (forcefully) + * - get_rt_flag() == RT_F_SLEEP // the job completed (by syscall) + * - linked != scheduled // we need to reschedule (for any reason) + * - is_np(scheduled) // rescheduling must be delayed, + * sys_exit_np must be requested + * + * Any of these can occur together. + */ +static struct task_struct* gsnedf_schedule(struct task_struct * prev) +{ + cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries); + int out_of_time, sleep, preempt, np, exists, blocks; + struct task_struct* next = NULL; + +#ifdef CONFIG_RELEASE_MASTER + /* Bail out early if we are the release master. + * The release master never schedules any real-time tasks. + */ + if (unlikely(gsnedf.release_master == entry->cpu)) { + sched_state_task_picked(); + return NULL; + } +#endif + + raw_spin_lock(&gsnedf_lock); + + /* sanity checking */ + BUG_ON(entry->scheduled && entry->scheduled != prev); + BUG_ON(entry->scheduled && !is_realtime(prev)); + BUG_ON(is_realtime(prev) && !entry->scheduled); + + /* (0) Determine state */ + exists = entry->scheduled != NULL; + blocks = exists && !is_running(entry->scheduled); + out_of_time = exists && + budget_enforced(entry->scheduled) && + budget_exhausted(entry->scheduled); + np = exists && is_np(entry->scheduled); + sleep = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP; + preempt = entry->scheduled != entry->linked; + +#ifdef WANT_ALL_SCHED_EVENTS + TRACE_TASK(prev, "invoked gsnedf_schedule.\n"); +#endif + + /* + if (exists) + TRACE_TASK(prev, + "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d " + "state:%d sig:%d\n", + blocks, out_of_time, np, sleep, preempt, + prev->state, signal_pending(prev)); + */ + + if (entry->linked && preempt) + TRACE_TASK(prev, "will be preempted by %s/%d\n", + entry->linked->comm, entry->linked->pid); + + /* If a task blocks we have no choice but to reschedule. + */ + if (blocks) { + unlink(entry->scheduled); + } + +#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) + if(exists && is_realtime(entry->scheduled) && tsk_rt(entry->scheduled)->held_gpus) { + if(!blocks || tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) { + stop_gpu_tracker(entry->scheduled); + } + } +#endif + + /* Request a sys_exit_np() call if we would like to preempt but cannot. + * We need to make sure to update the link structure anyway in case + * that we are still linked. Multiple calls to request_exit_np() don't + * hurt. + */ + if (np && (out_of_time || preempt || sleep)) { + unlink(entry->scheduled); + request_exit_np(entry->scheduled); + } + + /* Any task that is preemptable and either exhausts its execution + * budget or wants to sleep completes. We may have to reschedule after + * this. Don't do a job completion if we block (can't have timers running + * for blocked jobs). Preemption go first for the same reason. + */ + if (!np && (out_of_time || sleep) && !blocks && !preempt) + job_completion(entry->scheduled, !sleep); + + /* Link pending task if we became unlinked. + */ + if (!entry->linked) + link_task_to_cpu(__take_ready(&gsnedf), entry); + + /* The final scheduling decision. Do we need to switch for some reason? + * If linked is different from scheduled, then select linked as next. + */ + if ((!np || blocks) && + entry->linked != entry->scheduled) { + /* Schedule a linked job? */ + if (entry->linked) { + entry->linked->rt_param.scheduled_on = entry->cpu; + next = entry->linked; + TRACE_TASK(next, "scheduled_on = P%d\n", smp_processor_id()); + } + if (entry->scheduled) { + /* not gonna be scheduled soon */ + entry->scheduled->rt_param.scheduled_on = NO_CPU; + TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n"); + } + } + else + { + /* Only override Linux scheduler if we have a real-time task + * scheduled that needs to continue. + */ + if (exists) + next = prev; + } + + sched_state_task_picked(); + + raw_spin_unlock(&gsnedf_lock); + +#ifdef WANT_ALL_SCHED_EVENTS + TRACE("gsnedf_lock released, next=0x%p\n", next); + + if (next) + TRACE_TASK(next, "scheduled at %llu\n", litmus_clock()); + else if (exists && !next) + TRACE("becomes idle at %llu.\n", litmus_clock()); +#endif + + + return next; +} + + +/* _finish_switch - we just finished the switch away from prev + */ +static void gsnedf_finish_switch(struct task_struct *prev) +{ + cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries); + + entry->scheduled = is_realtime(current) ? current : NULL; + +#ifdef WANT_ALL_SCHED_EVENTS + TRACE_TASK(prev, "switched away from\n"); +#endif +} + + +/* Prepare a task for running in RT mode + */ +static void gsnedf_task_new(struct task_struct * t, int on_rq, int running) +{ + unsigned long flags; + cpu_entry_t* entry; + + TRACE("gsn edf: task new %d\n", t->pid); + + raw_spin_lock_irqsave(&gsnedf_lock, flags); + + /* setup job params */ + release_at(t, litmus_clock()); + + if (running) { + entry = &per_cpu(gsnedf_cpu_entries, task_cpu(t)); + BUG_ON(entry->scheduled); + +#ifdef CONFIG_RELEASE_MASTER + if (entry->cpu != gsnedf.release_master) { +#endif + entry->scheduled = t; + tsk_rt(t)->scheduled_on = task_cpu(t); +#ifdef CONFIG_RELEASE_MASTER + } else { + /* do not schedule on release master */ + preempt(entry); /* force resched */ + tsk_rt(t)->scheduled_on = NO_CPU; + } +#endif + } else { + t->rt_param.scheduled_on = NO_CPU; + } + t->rt_param.linked_on = NO_CPU; + + gsnedf_job_arrival(t); + raw_spin_unlock_irqrestore(&gsnedf_lock, flags); +} + +static void gsnedf_task_wake_up(struct task_struct *task) +{ + unsigned long flags; + //lt_t now; + + TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); + + raw_spin_lock_irqsave(&gsnedf_lock, flags); + + +#if 0 // sporadic task model + /* We need to take suspensions because of semaphores into + * account! If a job resumes after being suspended due to acquiring + * a semaphore, it should never be treated as a new job release. + */ + if (get_rt_flags(task) == RT_F_EXIT_SEM) { + set_rt_flags(task, RT_F_RUNNING); + } else { + now = litmus_clock(); + if (is_tardy(task, now)) { + /* new sporadic release */ + release_at(task, now); + sched_trace_task_release(task); + } + else { + if (task->rt.time_slice) { + /* came back in time before deadline + */ + set_rt_flags(task, RT_F_RUNNING); + } + } + } +#else // periodic task model + set_rt_flags(task, RT_F_RUNNING); +#endif + + gsnedf_job_arrival(task); + raw_spin_unlock_irqrestore(&gsnedf_lock, flags); +} + +static void gsnedf_task_block(struct task_struct *t) +{ + // TODO: is this called on preemption?? + unsigned long flags; + + TRACE_TASK(t, "block at %llu\n", litmus_clock()); + + /* unlink if necessary */ + raw_spin_lock_irqsave(&gsnedf_lock, flags); + + unlink(t); + + raw_spin_unlock_irqrestore(&gsnedf_lock, flags); + + BUG_ON(!is_realtime(t)); +} + + +static void gsnedf_task_exit(struct task_struct * t) +{ + unsigned long flags; + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + gsnedf_change_prio_pai_tasklet(t, NULL); +#endif + + /* unlink if necessary */ + raw_spin_lock_irqsave(&gsnedf_lock, flags); + unlink(t); + if (tsk_rt(t)->scheduled_on != NO_CPU) { + gsnedf_cpus[tsk_rt(t)->scheduled_on]->scheduled = NULL; + tsk_rt(t)->scheduled_on = NO_CPU; + } + raw_spin_unlock_irqrestore(&gsnedf_lock, flags); + + BUG_ON(!is_realtime(t)); + TRACE_TASK(t, "RIP\n"); +} + + +static long gsnedf_admit_task(struct task_struct* tsk) +{ +#ifdef CONFIG_LITMUS_NESTED_LOCKING + INIT_BINHEAP_HANDLE(&tsk_rt(tsk)->hp_blocked_tasks, + edf_max_heap_base_priority_order); +#endif + + return 0; +} + + + + + + +#ifdef CONFIG_LITMUS_LOCKING + +#include + +/* called with IRQs off */ +static void __increase_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh) +{ + int linked_on; + int check_preempt = 0; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + /* this sanity check allows for weaker locking in protocols */ + /* TODO (klitirqd): Skip this check if 't' is a proxy thread (???) */ + if(__edf_higher_prio(prio_inh, BASE, t, EFFECTIVE)) { +#endif + TRACE_TASK(t, "inherits priority from %s/%d\n", + prio_inh->comm, prio_inh->pid); + tsk_rt(t)->inh_task = prio_inh; + + linked_on = tsk_rt(t)->linked_on; + + /* If it is scheduled, then we need to reorder the CPU heap. */ + if (linked_on != NO_CPU) { + TRACE_TASK(t, "%s: linked on %d\n", + __FUNCTION__, linked_on); + /* Holder is scheduled; need to re-order CPUs. + * We can't use heap_decrease() here since + * the cpu_heap is ordered in reverse direction, so + * it is actually an increase. */ + binheap_delete(&gsnedf_cpus[linked_on]->hn, &gsnedf_cpu_heap); + binheap_add(&gsnedf_cpus[linked_on]->hn, + &gsnedf_cpu_heap, cpu_entry_t, hn); + } else { + /* holder may be queued: first stop queue changes */ + raw_spin_lock(&gsnedf.release_lock); + if (is_queued(t)) { + TRACE_TASK(t, "%s: is queued\n", + __FUNCTION__); + /* We need to update the position of holder in some + * heap. Note that this could be a release heap if we + * budget enforcement is used and this job overran. */ + check_preempt = + !bheap_decrease(edf_ready_order, + tsk_rt(t)->heap_node); + } else { + /* Nothing to do: if it is not queued and not linked + * then it is either sleeping or currently being moved + * by other code (e.g., a timer interrupt handler) that + * will use the correct priority when enqueuing the + * task. */ + TRACE_TASK(t, "%s: is NOT queued => Done.\n", + __FUNCTION__); + } + raw_spin_unlock(&gsnedf.release_lock); + + /* If holder was enqueued in a release heap, then the following + * preemption check is pointless, but we can't easily detect + * that case. If you want to fix this, then consider that + * simply adding a state flag requires O(n) time to update when + * releasing n tasks, which conflicts with the goal to have + * O(log n) merges. */ + if (check_preempt) { + /* heap_decrease() hit the top level of the heap: make + * sure preemption checks get the right task, not the + * potentially stale cache. */ + bheap_uncache_min(edf_ready_order, + &gsnedf.ready_queue); + check_for_preemptions(); + } + } +#ifdef CONFIG_LITMUS_NESTED_LOCKING + } + else { + TRACE_TASK(t, "Spurious invalid priority increase. " + "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n" + "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n", + t->comm, t->pid, + effective_priority(t)->comm, effective_priority(t)->pid, + (prio_inh) ? prio_inh->comm : "nil", + (prio_inh) ? prio_inh->pid : -1); + WARN_ON(!prio_inh); + } +#endif +} + +/* called with IRQs off */ +static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) +{ + raw_spin_lock(&gsnedf_lock); + + __increase_priority_inheritance(t, prio_inh); + +#ifdef CONFIG_LITMUS_SOFTIRQD + if(tsk_rt(t)->cur_klitirqd != NULL) + { + TRACE_TASK(t, "%s/%d inherits a new priority!\n", + tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); + + __increase_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); + } +#endif + + raw_spin_unlock(&gsnedf_lock); + +#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA) + if(tsk_rt(t)->held_gpus) { + int i; + for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus)); + i < NV_DEVICE_NUM; + i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) { + pai_check_priority_increase(t, i); + } + } +#endif +} + + +/* called with IRQs off */ +static void __decrease_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh) +{ +#ifdef CONFIG_LITMUS_NESTED_LOCKING + if(__edf_higher_prio(t, EFFECTIVE, prio_inh, BASE)) { +#endif + /* A job only stops inheriting a priority when it releases a + * resource. Thus we can make the following assumption.*/ + if(prio_inh) + TRACE_TASK(t, "EFFECTIVE priority decreased to %s/%d\n", + prio_inh->comm, prio_inh->pid); + else + TRACE_TASK(t, "base priority restored.\n"); + + tsk_rt(t)->inh_task = prio_inh; + + if(tsk_rt(t)->scheduled_on != NO_CPU) { + TRACE_TASK(t, "is scheduled.\n"); + + /* Check if rescheduling is necessary. We can't use heap_decrease() + * since the priority was effectively lowered. */ + unlink(t); + gsnedf_job_arrival(t); + } + else { + /* task is queued */ + raw_spin_lock(&gsnedf.release_lock); + if (is_queued(t)) { + TRACE_TASK(t, "is queued.\n"); + + /* decrease in priority, so we have to re-add to binomial heap */ + unlink(t); + gsnedf_job_arrival(t); + } + else { + TRACE_TASK(t, "is not in scheduler. Probably on wait queue somewhere.\n"); + } + raw_spin_unlock(&gsnedf.release_lock); + } +#ifdef CONFIG_LITMUS_NESTED_LOCKING + } + else { + TRACE_TASK(t, "Spurious invalid priority decrease. " + "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n" + "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n", + t->comm, t->pid, + effective_priority(t)->comm, effective_priority(t)->pid, + (prio_inh) ? prio_inh->comm : "nil", + (prio_inh) ? prio_inh->pid : -1); + } +#endif +} + +static void decrease_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh) +{ + raw_spin_lock(&gsnedf_lock); + __decrease_priority_inheritance(t, prio_inh); + +#ifdef CONFIG_LITMUS_SOFTIRQD + if(tsk_rt(t)->cur_klitirqd != NULL) + { + TRACE_TASK(t, "%s/%d decreases in priority!\n", + tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid); + + __decrease_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh); + } +#endif + + raw_spin_unlock(&gsnedf_lock); + +#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA) + if(tsk_rt(t)->held_gpus) { + int i; + for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus)); + i < NV_DEVICE_NUM; + i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) { + pai_check_priority_decrease(t, i); + } + } +#endif +} + + +#ifdef CONFIG_LITMUS_SOFTIRQD +/* called with IRQs off */ +static void increase_priority_inheritance_klitirqd(struct task_struct* klitirqd, + struct task_struct* old_owner, + struct task_struct* new_owner) +{ + BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); + + raw_spin_lock(&gsnedf_lock); + + if(old_owner != new_owner) + { + if(old_owner) + { + // unreachable? + tsk_rt(old_owner)->cur_klitirqd = NULL; + } + + TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n", + new_owner->comm, new_owner->pid); + + tsk_rt(new_owner)->cur_klitirqd = klitirqd; + } + + __decrease_priority_inheritance(klitirqd, NULL); // kludge to clear out cur prio. + + __increase_priority_inheritance(klitirqd, + (tsk_rt(new_owner)->inh_task == NULL) ? + new_owner : + tsk_rt(new_owner)->inh_task); + + raw_spin_unlock(&gsnedf_lock); +} + + +/* called with IRQs off */ +static void decrease_priority_inheritance_klitirqd(struct task_struct* klitirqd, + struct task_struct* old_owner, + struct task_struct* new_owner) +{ + BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread)); + + raw_spin_lock(&gsnedf_lock); + + TRACE_TASK(klitirqd, "priority restored\n"); + + __decrease_priority_inheritance(klitirqd, new_owner); + + tsk_rt(old_owner)->cur_klitirqd = NULL; + + raw_spin_unlock(&gsnedf_lock); +} +#endif + + + + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + +/* called with IRQs off */ +/* preconditions: + (1) The 'hp_blocked_tasks_lock' of task 't' is held. + (2) The lock 'to_unlock' is held. + */ +static void nested_increase_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh, + raw_spinlock_t *to_unlock, + unsigned long irqflags) +{ + struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock; + + if(tsk_rt(t)->inh_task != prio_inh) { // shield redundent calls. + increase_priority_inheritance(t, prio_inh); // increase our prio. + } + + raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); // unlock the t's heap. + + + if(blocked_lock) { + if(blocked_lock->ops->propagate_increase_inheritance) { + TRACE_TASK(t, "Inheritor is blocked (...perhaps). Checking lock %d.\n", + blocked_lock->ident); + + // beware: recursion + blocked_lock->ops->propagate_increase_inheritance(blocked_lock, + t, to_unlock, + irqflags); + } + else { + TRACE_TASK(t, "Inheritor is blocked on lock (%d) that does not support nesting!\n", + blocked_lock->ident); + unlock_fine_irqrestore(to_unlock, irqflags); + } + } + else { + TRACE_TASK(t, "is not blocked. No propagation.\n"); + unlock_fine_irqrestore(to_unlock, irqflags); + } +} + +/* called with IRQs off */ +/* preconditions: + (1) The 'hp_blocked_tasks_lock' of task 't' is held. + (2) The lock 'to_unlock' is held. + */ +static void nested_decrease_priority_inheritance(struct task_struct* t, + struct task_struct* prio_inh, + raw_spinlock_t *to_unlock, + unsigned long irqflags) +{ + struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock; + decrease_priority_inheritance(t, prio_inh); + + raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); // unlock the t's heap. + + if(blocked_lock) { + if(blocked_lock->ops->propagate_decrease_inheritance) { + TRACE_TASK(t, "Inheritor is blocked (...perhaps). Checking lock %d.\n", + blocked_lock->ident); + + // beware: recursion + blocked_lock->ops->propagate_decrease_inheritance(blocked_lock, t, + to_unlock, + irqflags); + } + else { + TRACE_TASK(t, "Inheritor is blocked on lock (%p) that does not support nesting!\n", + blocked_lock); + unlock_fine_irqrestore(to_unlock, irqflags); + } + } + else { + TRACE_TASK(t, "is not blocked. No propagation.\n"); + unlock_fine_irqrestore(to_unlock, irqflags); + } +} + + +/* ******************** RSM MUTEX ********************** */ + +static struct litmus_lock_ops gsnedf_rsm_mutex_lock_ops = { + .lock = rsm_mutex_lock, + .unlock = rsm_mutex_unlock, + .close = rsm_mutex_close, + .deallocate = rsm_mutex_free, + + .propagate_increase_inheritance = rsm_mutex_propagate_increase_inheritance, + .propagate_decrease_inheritance = rsm_mutex_propagate_decrease_inheritance, + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + .dgl_lock = rsm_mutex_dgl_lock, + .is_owner = rsm_mutex_is_owner, + .enable_priority = rsm_mutex_enable_priority, +#endif +}; + +static struct litmus_lock* gsnedf_new_rsm_mutex(void) +{ + return rsm_mutex_new(&gsnedf_rsm_mutex_lock_ops); +} + +/* ******************** IKGLP ********************** */ + +static struct litmus_lock_ops gsnedf_ikglp_lock_ops = { + .lock = ikglp_lock, + .unlock = ikglp_unlock, + .close = ikglp_close, + .deallocate = ikglp_free, + + // ikglp can only be an outer-most lock. + .propagate_increase_inheritance = NULL, + .propagate_decrease_inheritance = NULL, +}; + +static struct litmus_lock* gsnedf_new_ikglp(void* __user arg) +{ + return ikglp_new(num_online_cpus(), &gsnedf_ikglp_lock_ops, arg); +} + +#endif /* CONFIG_LITMUS_NESTED_LOCKING */ + + +/* ******************** KFMLP support ********************** */ + +static struct litmus_lock_ops gsnedf_kfmlp_lock_ops = { + .lock = kfmlp_lock, + .unlock = kfmlp_unlock, + .close = kfmlp_close, + .deallocate = kfmlp_free, + + // kfmlp can only be an outer-most lock. + .propagate_increase_inheritance = NULL, + .propagate_decrease_inheritance = NULL, +}; + + +static struct litmus_lock* gsnedf_new_kfmlp(void* __user arg) +{ + return kfmlp_new(&gsnedf_kfmlp_lock_ops, arg); +} + +/* ******************** FMLP support ********************** */ + +/* struct for semaphore with priority inheritance */ +struct fmlp_semaphore { + struct litmus_lock litmus_lock; + + /* current resource holder */ + struct task_struct *owner; + + /* highest-priority waiter */ + struct task_struct *hp_waiter; + + /* FIFO queue of waiting tasks */ + wait_queue_head_t wait; +}; + +static inline struct fmlp_semaphore* fmlp_from_lock(struct litmus_lock* lock) +{ + return container_of(lock, struct fmlp_semaphore, litmus_lock); +} + +/* caller is responsible for locking */ +struct task_struct* find_hp_waiter(struct fmlp_semaphore *sem, + struct task_struct* skip) +{ + struct list_head *pos; + struct task_struct *queued, *found = NULL; + + list_for_each(pos, &sem->wait.task_list) { + queued = (struct task_struct*) list_entry(pos, wait_queue_t, + task_list)->private; + + /* Compare task prios, find high prio task. */ + if (queued != skip && edf_higher_prio(queued, found)) + found = queued; + } + return found; +} + +int gsnedf_fmlp_lock(struct litmus_lock* l) +{ + struct task_struct* t = current; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + wait_queue_t wait; + unsigned long flags; + + if (!is_realtime(t)) + return -EPERM; + + spin_lock_irqsave(&sem->wait.lock, flags); + + if (sem->owner) { + /* resource is not free => must suspend and wait */ + + init_waitqueue_entry(&wait, t); + + /* FIXME: interruptible would be nice some day */ + set_task_state(t, TASK_UNINTERRUPTIBLE); + + __add_wait_queue_tail_exclusive(&sem->wait, &wait); + + /* check if we need to activate priority inheritance */ + if (edf_higher_prio(t, sem->hp_waiter)) { + sem->hp_waiter = t; + if (edf_higher_prio(t, sem->owner)) + increase_priority_inheritance(sem->owner, sem->hp_waiter); + } + + TS_LOCK_SUSPEND; + + /* release lock before sleeping */ + spin_unlock_irqrestore(&sem->wait.lock, flags); + + /* We depend on the FIFO order. Thus, we don't need to recheck + * when we wake up; we are guaranteed to have the lock since + * there is only one wake up per release. + */ + + schedule(); + + TS_LOCK_RESUME; + + /* Since we hold the lock, no other task will change + * ->owner. We can thus check it without acquiring the spin + * lock. */ + BUG_ON(sem->owner != t); + } else { + /* it's ours now */ + sem->owner = t; + + spin_unlock_irqrestore(&sem->wait.lock, flags); + } + + return 0; +} + +int gsnedf_fmlp_unlock(struct litmus_lock* l) +{ + struct task_struct *t = current, *next; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + unsigned long flags; + int err = 0; + + spin_lock_irqsave(&sem->wait.lock, flags); + + if (sem->owner != t) { + err = -EINVAL; + goto out; + } + + /* check if there are jobs waiting for this resource */ + next = __waitqueue_remove_first(&sem->wait); + if (next) { + /* next becomes the resouce holder */ + sem->owner = next; + TRACE_CUR("lock ownership passed to %s/%d\n", next->comm, next->pid); + + /* determine new hp_waiter if necessary */ + if (next == sem->hp_waiter) { + TRACE_TASK(next, "was highest-prio waiter\n"); + /* next has the highest priority --- it doesn't need to + * inherit. However, we need to make sure that the + * next-highest priority in the queue is reflected in + * hp_waiter. */ + sem->hp_waiter = find_hp_waiter(sem, next); + if (sem->hp_waiter) + TRACE_TASK(sem->hp_waiter, "is new highest-prio waiter\n"); + else + TRACE("no further waiters\n"); + } else { + /* Well, if next is not the highest-priority waiter, + * then it ought to inherit the highest-priority + * waiter's priority. */ + increase_priority_inheritance(next, sem->hp_waiter); + } + + /* wake up next */ + wake_up_process(next); + } else + /* becomes available */ + sem->owner = NULL; + + /* we lose the benefit of priority inheritance (if any) */ + if (tsk_rt(t)->inh_task) + decrease_priority_inheritance(t, NULL); + +out: + spin_unlock_irqrestore(&sem->wait.lock, flags); + + return err; +} + +int gsnedf_fmlp_close(struct litmus_lock* l) +{ + struct task_struct *t = current; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + unsigned long flags; + + int owner; + + spin_lock_irqsave(&sem->wait.lock, flags); + + owner = sem->owner == t; + + spin_unlock_irqrestore(&sem->wait.lock, flags); + + if (owner) + gsnedf_fmlp_unlock(l); + + return 0; +} + +void gsnedf_fmlp_free(struct litmus_lock* lock) +{ + kfree(fmlp_from_lock(lock)); +} + +static struct litmus_lock_ops gsnedf_fmlp_lock_ops = { + .close = gsnedf_fmlp_close, + .lock = gsnedf_fmlp_lock, + .unlock = gsnedf_fmlp_unlock, + .deallocate = gsnedf_fmlp_free, + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + .propagate_increase_inheritance = NULL, + .propagate_decrease_inheritance = NULL +#endif +}; + +static struct litmus_lock* gsnedf_new_fmlp(void) +{ + struct fmlp_semaphore* sem; + + sem = kmalloc(sizeof(*sem), GFP_KERNEL); + if (!sem) + return NULL; + + sem->owner = NULL; + sem->hp_waiter = NULL; + init_waitqueue_head(&sem->wait); + sem->litmus_lock.ops = &gsnedf_fmlp_lock_ops; + + return &sem->litmus_lock; +} + + +static long gsnedf_allocate_lock(struct litmus_lock **lock, int type, + void* __user args) +{ + int err; + + switch (type) { + + case FMLP_SEM: + /* Flexible Multiprocessor Locking Protocol */ + *lock = gsnedf_new_fmlp(); + break; +#ifdef CONFIG_LITMUS_NESTED_LOCKING + case RSM_MUTEX: + *lock = gsnedf_new_rsm_mutex(); + break; + + case IKGLP_SEM: + *lock = gsnedf_new_ikglp(args); + break; +#endif + case KFMLP_SEM: + *lock = gsnedf_new_kfmlp(args); + break; + default: + err = -ENXIO; + goto UNSUPPORTED_LOCK; + }; + + if (*lock) + err = 0; + else + err = -ENOMEM; + +UNSUPPORTED_LOCK: + return err; +} + +#endif // CONFIG_LITMUS_LOCKING + + + + + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING +static struct affinity_observer_ops gsnedf_kfmlp_affinity_ops = { + .close = kfmlp_aff_obs_close, + .deallocate = kfmlp_aff_obs_free, +}; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING +static struct affinity_observer_ops gsnedf_ikglp_affinity_ops = { + .close = ikglp_aff_obs_close, + .deallocate = ikglp_aff_obs_free, +}; +#endif + +static long gsnedf_allocate_affinity_observer( + struct affinity_observer **aff_obs, + int type, + void* __user args) +{ + int err; + + switch (type) { + + case KFMLP_SIMPLE_GPU_AFF_OBS: + *aff_obs = kfmlp_simple_gpu_aff_obs_new(&gsnedf_kfmlp_affinity_ops, args); + break; + + case KFMLP_GPU_AFF_OBS: + *aff_obs = kfmlp_gpu_aff_obs_new(&gsnedf_kfmlp_affinity_ops, args); + break; + +#ifdef CONFIG_LITMUS_NESTED_LOCKING + case IKGLP_SIMPLE_GPU_AFF_OBS: + *aff_obs = ikglp_simple_gpu_aff_obs_new(&gsnedf_ikglp_affinity_ops, args); + break; + + case IKGLP_GPU_AFF_OBS: + *aff_obs = ikglp_gpu_aff_obs_new(&gsnedf_ikglp_affinity_ops, args); + break; +#endif + default: + err = -ENXIO; + goto UNSUPPORTED_AFF_OBS; + }; + + if (*aff_obs) + err = 0; + else + err = -ENOMEM; + +UNSUPPORTED_AFF_OBS: + return err; +} +#endif + + + + + +static long gsnedf_activate_plugin(void) +{ + int cpu; + cpu_entry_t *entry; + + INIT_BINHEAP_HANDLE(&gsnedf_cpu_heap, cpu_lower_prio); +#ifdef CONFIG_RELEASE_MASTER + gsnedf.release_master = atomic_read(&release_master_cpu); +#endif + + for_each_online_cpu(cpu) { + entry = &per_cpu(gsnedf_cpu_entries, cpu); + INIT_BINHEAP_NODE(&entry->hn); + entry->linked = NULL; + entry->scheduled = NULL; +#ifdef CONFIG_RELEASE_MASTER + if (cpu != gsnedf.release_master) { +#endif + TRACE("GSN-EDF: Initializing CPU #%d.\n", cpu); + update_cpu_position(entry); +#ifdef CONFIG_RELEASE_MASTER + } else { + TRACE("GSN-EDF: CPU %d is release master.\n", cpu); + } +#endif + } + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + gsnedf_pending_tasklets.head = NULL; + gsnedf_pending_tasklets.tail = &(gsnedf_pending_tasklets.head); +#endif + +#ifdef CONFIG_LITMUS_SOFTIRQD + spawn_klitirqd(NULL); +#endif + +#ifdef CONFIG_LITMUS_NVIDIA + init_nvidia_info(); +#endif + + return 0; +} + +/* Plugin object */ +static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = { + .plugin_name = "GSN-EDF", + .finish_switch = gsnedf_finish_switch, + .tick = gsnedf_tick, + .task_new = gsnedf_task_new, + .complete_job = complete_job, + .task_exit = gsnedf_task_exit, + .schedule = gsnedf_schedule, + .task_wake_up = gsnedf_task_wake_up, + .task_block = gsnedf_task_block, + .admit_task = gsnedf_admit_task, + .activate_plugin = gsnedf_activate_plugin, + .compare = edf_higher_prio, +#ifdef CONFIG_LITMUS_LOCKING + .allocate_lock = gsnedf_allocate_lock, + .increase_prio = increase_priority_inheritance, + .decrease_prio = decrease_priority_inheritance, +#endif +#ifdef CONFIG_LITMUS_NESTED_LOCKING + .nested_increase_prio = nested_increase_priority_inheritance, + .nested_decrease_prio = nested_decrease_priority_inheritance, + .__compare = __edf_higher_prio, +#endif +#ifdef CONFIG_LITMUS_DGL_SUPPORT + .get_dgl_spinlock = gsnedf_get_dgl_spinlock, +#endif +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + .allocate_aff_obs = gsnedf_allocate_affinity_observer, +#endif +#ifdef CONFIG_LITMUS_SOFTIRQD + .increase_prio_klitirqd = increase_priority_inheritance_klitirqd, + .decrease_prio_klitirqd = decrease_priority_inheritance_klitirqd, +#endif +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + .enqueue_pai_tasklet = gsnedf_enqueue_pai_tasklet, + .change_prio_pai_tasklet = gsnedf_change_prio_pai_tasklet, + .run_tasklets = gsnedf_run_tasklets, +#endif +}; + + +static int __init init_gsn_edf(void) +{ + int cpu; + cpu_entry_t *entry; + + INIT_BINHEAP_HANDLE(&gsnedf_cpu_heap, cpu_lower_prio); + /* initialize CPU state */ + for (cpu = 0; cpu < NR_CPUS; ++cpu) { + entry = &per_cpu(gsnedf_cpu_entries, cpu); + gsnedf_cpus[cpu] = entry; + entry->cpu = cpu; + + INIT_BINHEAP_NODE(&entry->hn); + } + +#ifdef CONFIG_LITMUS_DGL_SUPPORT + raw_spin_lock_init(&dgl_lock); +#endif + + edf_domain_init(&gsnedf, NULL, gsnedf_release_jobs); + return register_sched_plugin(&gsn_edf_plugin); +} + + +module_init(init_gsn_edf); diff --git a/litmus/sched_litmus.c b/litmus/sched_litmus.c new file mode 100644 index 0000000..9a6fe48 --- /dev/null +++ b/litmus/sched_litmus.c @@ -0,0 +1,327 @@ +/* This file is included from kernel/sched.c */ + +#include +#include +#include +#include + +static void update_time_litmus(struct rq *rq, struct task_struct *p) +{ + u64 delta = rq->clock - p->se.exec_start; + if (unlikely((s64)delta < 0)) + delta = 0; + /* per job counter */ + p->rt_param.job_params.exec_time += delta; + /* task counter */ + p->se.sum_exec_runtime += delta; + /* sched_clock() */ + p->se.exec_start = rq->clock; + cpuacct_charge(p, delta); +} + +static void double_rq_lock(struct rq *rq1, struct rq *rq2); +static void double_rq_unlock(struct rq *rq1, struct rq *rq2); + +/* + * litmus_tick gets called by scheduler_tick() with HZ freq + * Interrupts are disabled + */ +static void litmus_tick(struct rq *rq, struct task_struct *p) +{ + TS_PLUGIN_TICK_START; + + if (is_realtime(p)) + update_time_litmus(rq, p); + + /* plugin tick */ + litmus->tick(p); + + TS_PLUGIN_TICK_END; + + return; +} + +static struct task_struct * +litmus_schedule(struct rq *rq, struct task_struct *prev) +{ + struct rq* other_rq; + struct task_struct *next; + + long was_running; + lt_t _maybe_deadlock = 0; + + /* let the plugin schedule */ + next = litmus->schedule(prev); + + sched_state_plugin_check(); + + /* check if a global plugin pulled a task from a different RQ */ + if (next && task_rq(next) != rq) { + /* we need to migrate the task */ + other_rq = task_rq(next); + TRACE_TASK(next, "migrate from %d\n", other_rq->cpu); + + /* while we drop the lock, the prev task could change its + * state + */ + was_running = is_running(prev); + mb(); + raw_spin_unlock(&rq->lock); + + /* Don't race with a concurrent switch. This could deadlock in + * the case of cross or circular migrations. It's the job of + * the plugin to make sure that doesn't happen. + */ + TRACE_TASK(next, "stack_in_use=%d\n", + next->rt_param.stack_in_use); + if (next->rt_param.stack_in_use != NO_CPU) { + TRACE_TASK(next, "waiting to deschedule\n"); + _maybe_deadlock = litmus_clock(); + } + while (next->rt_param.stack_in_use != NO_CPU) { + cpu_relax(); + mb(); + if (next->rt_param.stack_in_use == NO_CPU) + TRACE_TASK(next,"descheduled. Proceeding.\n"); + + if (lt_before(_maybe_deadlock + 10000000, + litmus_clock())) { + /* We've been spinning for 10ms. + * Something can't be right! + * Let's abandon the task and bail out; at least + * we will have debug info instead of a hard + * deadlock. + */ + TRACE_TASK(next,"stack too long in use. " + "Deadlock?\n"); + next = NULL; + + /* bail out */ + raw_spin_lock(&rq->lock); + return next; + } + } +#ifdef __ARCH_WANT_UNLOCKED_CTXSW + if (next->oncpu) + { + TRACE_TASK(next, "waiting for !oncpu"); + } + while (next->oncpu) { + cpu_relax(); + mb(); + } +#endif + double_rq_lock(rq, other_rq); + mb(); + if (is_realtime(prev) && is_running(prev) != was_running) { + TRACE_TASK(prev, + "state changed while we dropped" + " the lock: is_running=%d, was_running=%d\n", + is_running(prev), was_running); + if (is_running(prev) && !was_running) { + /* prev task became unblocked + * we need to simulate normal sequence of events + * to scheduler plugins. + */ + litmus->task_block(prev); + litmus->task_wake_up(prev); + } + } + + set_task_cpu(next, smp_processor_id()); + + /* DEBUG: now that we have the lock we need to make sure a + * couple of things still hold: + * - it is still a real-time task + * - it is still runnable (could have been stopped) + * If either is violated, then the active plugin is + * doing something wrong. + */ + if (!is_realtime(next) || !is_running(next)) { + /* BAD BAD BAD */ + TRACE_TASK(next,"BAD: migration invariant FAILED: " + "rt=%d running=%d\n", + is_realtime(next), + is_running(next)); + /* drop the task */ + next = NULL; + } + /* release the other CPU's runqueue, but keep ours */ + raw_spin_unlock(&other_rq->lock); + } + if (next) { + next->rt_param.stack_in_use = rq->cpu; + next->se.exec_start = rq->clock; + } + + update_enforcement_timer(next); + return next; +} + +static void enqueue_task_litmus(struct rq *rq, struct task_struct *p, + int flags) +{ + if (flags & ENQUEUE_WAKEUP) { + sched_trace_task_resume(p); + tsk_rt(p)->present = 1; + /* LITMUS^RT plugins need to update the state + * _before_ making it available in global structures. + * Linux gets away with being lazy about the task state + * update. We can't do that, hence we update the task + * state already here. + * + * WARNING: this needs to be re-evaluated when porting + * to newer kernel versions. + */ + p->state = TASK_RUNNING; + litmus->task_wake_up(p); + + rq->litmus.nr_running++; + } else + TRACE_TASK(p, "ignoring an enqueue, not a wake up.\n"); +} + +static void dequeue_task_litmus(struct rq *rq, struct task_struct *p, + int flags) +{ + if (flags & DEQUEUE_SLEEP) { + litmus->task_block(p); + tsk_rt(p)->present = 0; + sched_trace_task_block(p); + + rq->litmus.nr_running--; + } else + TRACE_TASK(p, "ignoring a dequeue, not going to sleep.\n"); +} + +static void yield_task_litmus(struct rq *rq) +{ + BUG_ON(rq->curr != current); + /* sched_yield() is called to trigger delayed preemptions. + * Thus, mark the current task as needing to be rescheduled. + * This will cause the scheduler plugin to be invoked, which can + * then determine if a preemption is still required. + */ + clear_exit_np(current); + litmus_reschedule_local(); +} + +/* Plugins are responsible for this. + */ +static void check_preempt_curr_litmus(struct rq *rq, struct task_struct *p, int flags) +{ +} + +static void put_prev_task_litmus(struct rq *rq, struct task_struct *p) +{ +} + +static void pre_schedule_litmus(struct rq *rq, struct task_struct *prev) +{ + update_time_litmus(rq, prev); + if (!is_running(prev)) + tsk_rt(prev)->present = 0; +} + +/* pick_next_task_litmus() - litmus_schedule() function + * + * return the next task to be scheduled + */ +static struct task_struct *pick_next_task_litmus(struct rq *rq) +{ + /* get the to-be-switched-out task (prev) */ + struct task_struct *prev = rq->litmus.prev; + struct task_struct *next; + + /* if not called from schedule() but from somewhere + * else (e.g., migration), return now! + */ + if(!rq->litmus.prev) + return NULL; + + rq->litmus.prev = NULL; + + TS_PLUGIN_SCHED_START; + next = litmus_schedule(rq, prev); + TS_PLUGIN_SCHED_END; + + return next; +} + +static void task_tick_litmus(struct rq *rq, struct task_struct *p, int queued) +{ + /* nothing to do; tick related tasks are done by litmus_tick() */ + return; +} + +static void switched_to_litmus(struct rq *rq, struct task_struct *p) +{ +} + +static void prio_changed_litmus(struct rq *rq, struct task_struct *p, + int oldprio) +{ +} + +unsigned int get_rr_interval_litmus(struct rq *rq, struct task_struct *p) +{ + /* return infinity */ + return 0; +} + +/* This is called when a task became a real-time task, either due to a SCHED_* + * class transition or due to PI mutex inheritance. We don't handle Linux PI + * mutex inheritance yet (and probably never will). Use LITMUS provided + * synchronization primitives instead. + */ +static void set_curr_task_litmus(struct rq *rq) +{ + rq->curr->se.exec_start = rq->clock; +} + + +#ifdef CONFIG_SMP +/* execve tries to rebalance task in this scheduling domain. + * We don't care about the scheduling domain; can gets called from + * exec, fork, wakeup. + */ +static int +select_task_rq_litmus(struct task_struct *p, int sd_flag, int flags) +{ + /* preemption is already disabled. + * We don't want to change cpu here + */ + return task_cpu(p); +} +#endif + +static const struct sched_class litmus_sched_class = { + /* From 34f971f6 the stop/migrate worker threads have a class on + * their own, which is the highest prio class. We don't support + * cpu-hotplug or cpu throttling. Allows Litmus to use up to 1.0 + * CPU capacity. + */ + .next = &stop_sched_class, + .enqueue_task = enqueue_task_litmus, + .dequeue_task = dequeue_task_litmus, + .yield_task = yield_task_litmus, + + .check_preempt_curr = check_preempt_curr_litmus, + + .pick_next_task = pick_next_task_litmus, + .put_prev_task = put_prev_task_litmus, + +#ifdef CONFIG_SMP + .select_task_rq = select_task_rq_litmus, + + .pre_schedule = pre_schedule_litmus, +#endif + + .set_curr_task = set_curr_task_litmus, + .task_tick = task_tick_litmus, + + .get_rr_interval = get_rr_interval_litmus, + + .prio_changed = prio_changed_litmus, + .switched_to = switched_to_litmus, +}; diff --git a/litmus/sched_pfair.c b/litmus/sched_pfair.c new file mode 100644 index 0000000..16f1065 --- /dev/null +++ b/litmus/sched_pfair.c @@ -0,0 +1,1067 @@ +/* + * kernel/sched_pfair.c + * + * Implementation of the PD^2 pfair scheduling algorithm. This + * implementation realizes "early releasing," i.e., it is work-conserving. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +/* to configure the cluster size */ +#include + +#include + +static enum cache_level pfair_cluster_level = GLOBAL_CLUSTER; + +struct subtask { + /* measured in quanta relative to job release */ + quanta_t release; + quanta_t deadline; + quanta_t overlap; /* called "b bit" by PD^2 */ + quanta_t group_deadline; +}; + +struct pfair_param { + quanta_t quanta; /* number of subtasks */ + quanta_t cur; /* index of current subtask */ + + quanta_t release; /* in quanta */ + quanta_t period; /* in quanta */ + + quanta_t last_quantum; /* when scheduled last */ + int last_cpu; /* where scheduled last */ + + struct pfair_cluster* cluster; /* where this task is scheduled */ + + struct subtask subtasks[0]; /* allocate together with pfair_param */ +}; + +#define tsk_pfair(tsk) ((tsk)->rt_param.pfair) + +struct pfair_state { + struct cluster_cpu topology; + + volatile quanta_t cur_tick; /* updated by the CPU that is advancing + * the time */ + volatile quanta_t local_tick; /* What tick is the local CPU currently + * executing? Updated only by the local + * CPU. In QEMU, this may lag behind the + * current tick. In a real system, with + * proper timers and aligned quanta, + * that should only be the case for a + * very short time after the time + * advanced. With staggered quanta, it + * will lag for the duration of the + * offset. + */ + + struct task_struct* linked; /* the task that should be executing */ + struct task_struct* local; /* the local copy of linked */ + struct task_struct* scheduled; /* what is actually scheduled */ + + lt_t offset; /* stagger offset */ + unsigned int missed_updates; + unsigned int missed_quanta; +}; + +struct pfair_cluster { + struct scheduling_cluster topology; + + /* The "global" time in this cluster. */ + quanta_t pfair_time; /* the "official" PFAIR clock */ + + /* The ready queue for this cluster. */ + rt_domain_t pfair; + + /* The set of jobs that should have their release enacted at the next + * quantum boundary. + */ + struct bheap release_queue; + raw_spinlock_t release_lock; +}; + +#define RT_F_REQUEUE 0x2 + +static inline struct pfair_cluster* cpu_cluster(struct pfair_state* state) +{ + return container_of(state->topology.cluster, struct pfair_cluster, topology); +} + +static inline int cpu_id(struct pfair_state* state) +{ + return state->topology.id; +} + +static inline struct pfair_state* from_cluster_list(struct list_head* pos) +{ + return list_entry(pos, struct pfair_state, topology.cluster_list); +} + +static inline struct pfair_cluster* from_domain(rt_domain_t* rt) +{ + return container_of(rt, struct pfair_cluster, pfair); +} + +static inline raw_spinlock_t* cluster_lock(struct pfair_cluster* cluster) +{ + /* The ready_lock is used to serialize all scheduling events. */ + return &cluster->pfair.ready_lock; +} + +static inline raw_spinlock_t* cpu_lock(struct pfair_state* state) +{ + return cluster_lock(cpu_cluster(state)); +} + +DEFINE_PER_CPU(struct pfair_state, pfair_state); +struct pfair_state* *pstate; /* short cut */ + +static struct pfair_cluster* pfair_clusters; +static int num_pfair_clusters; + +/* Enable for lots of trace info. + * #define PFAIR_DEBUG + */ + +#ifdef PFAIR_DEBUG +#define PTRACE_TASK(t, f, args...) TRACE_TASK(t, f, ## args) +#define PTRACE(f, args...) TRACE(f, ## args) +#else +#define PTRACE_TASK(t, f, args...) +#define PTRACE(f, args...) +#endif + +/* gcc will inline all of these accessor functions... */ +static struct subtask* cur_subtask(struct task_struct* t) +{ + return tsk_pfair(t)->subtasks + tsk_pfair(t)->cur; +} + +static quanta_t cur_deadline(struct task_struct* t) +{ + return cur_subtask(t)->deadline + tsk_pfair(t)->release; +} + +static quanta_t cur_release(struct task_struct* t) +{ + /* This is early releasing: only the release of the first subtask + * counts. */ + return tsk_pfair(t)->release; +} + +static quanta_t cur_overlap(struct task_struct* t) +{ + return cur_subtask(t)->overlap; +} + +static quanta_t cur_group_deadline(struct task_struct* t) +{ + quanta_t gdl = cur_subtask(t)->group_deadline; + if (gdl) + return gdl + tsk_pfair(t)->release; + else + return gdl; +} + + +static int pfair_higher_prio(struct task_struct* first, + struct task_struct* second) +{ + return /* first task must exist */ + first && ( + /* Does the second task exist and is it a real-time task? If + * not, the first task (which is a RT task) has higher + * priority. + */ + !second || !is_realtime(second) || + + /* Is the (subtask) deadline of the first task earlier? + * Then it has higher priority. + */ + time_before(cur_deadline(first), cur_deadline(second)) || + + /* Do we have a deadline tie? + * Then break by B-bit. + */ + (cur_deadline(first) == cur_deadline(second) && + (cur_overlap(first) > cur_overlap(second) || + + /* Do we have a B-bit tie? + * Then break by group deadline. + */ + (cur_overlap(first) == cur_overlap(second) && + (time_after(cur_group_deadline(first), + cur_group_deadline(second)) || + + /* Do we have a group deadline tie? + * Then break by PID, which are unique. + */ + (cur_group_deadline(first) == + cur_group_deadline(second) && + first->pid < second->pid)))))); +} + +int pfair_ready_order(struct bheap_node* a, struct bheap_node* b) +{ + return pfair_higher_prio(bheap2task(a), bheap2task(b)); +} + +static void pfair_release_jobs(rt_domain_t* rt, struct bheap* tasks) +{ + struct pfair_cluster* cluster = from_domain(rt); + unsigned long flags; + + raw_spin_lock_irqsave(&cluster->release_lock, flags); + + bheap_union(pfair_ready_order, &cluster->release_queue, tasks); + + raw_spin_unlock_irqrestore(&cluster->release_lock, flags); +} + +static void prepare_release(struct task_struct* t, quanta_t at) +{ + tsk_pfair(t)->release = at; + tsk_pfair(t)->cur = 0; +} + +/* pull released tasks from the release queue */ +static void poll_releases(struct pfair_cluster* cluster) +{ + raw_spin_lock(&cluster->release_lock); + __merge_ready(&cluster->pfair, &cluster->release_queue); + raw_spin_unlock(&cluster->release_lock); +} + +static void check_preempt(struct task_struct* t) +{ + int cpu = NO_CPU; + if (tsk_rt(t)->linked_on != tsk_rt(t)->scheduled_on && + tsk_rt(t)->present) { + /* the task can be scheduled and + * is not scheduled where it ought to be scheduled + */ + cpu = tsk_rt(t)->linked_on != NO_CPU ? + tsk_rt(t)->linked_on : + tsk_rt(t)->scheduled_on; + PTRACE_TASK(t, "linked_on:%d, scheduled_on:%d\n", + tsk_rt(t)->linked_on, tsk_rt(t)->scheduled_on); + /* preempt */ + litmus_reschedule(cpu); + } +} + +/* caller must hold pfair.ready_lock */ +static void drop_all_references(struct task_struct *t) +{ + int cpu; + struct pfair_state* s; + struct pfair_cluster* cluster; + if (bheap_node_in_heap(tsk_rt(t)->heap_node)) { + /* It must be in the ready queue; drop references isn't called + * when the job is in a release queue. */ + cluster = tsk_pfair(t)->cluster; + bheap_delete(pfair_ready_order, &cluster->pfair.ready_queue, + tsk_rt(t)->heap_node); + } + for (cpu = 0; cpu < num_online_cpus(); cpu++) { + s = &per_cpu(pfair_state, cpu); + if (s->linked == t) + s->linked = NULL; + if (s->local == t) + s->local = NULL; + if (s->scheduled == t) + s->scheduled = NULL; + } + /* make sure we don't have a stale linked_on field */ + tsk_rt(t)->linked_on = NO_CPU; +} + +static void pfair_prepare_next_period(struct task_struct* t) +{ + struct pfair_param* p = tsk_pfair(t); + + prepare_for_next_period(t); + get_rt_flags(t) = RT_F_RUNNING; + p->release += p->period; +} + +/* returns 1 if the task needs to go the release queue */ +static int advance_subtask(quanta_t time, struct task_struct* t, int cpu) +{ + struct pfair_param* p = tsk_pfair(t); + int to_relq; + p->cur = (p->cur + 1) % p->quanta; + if (!p->cur) { + if (tsk_rt(t)->present) { + /* The job overran; we start a new budget allocation. */ + pfair_prepare_next_period(t); + } else { + /* remove task from system until it wakes */ + drop_all_references(t); + tsk_rt(t)->flags = RT_F_REQUEUE; + TRACE_TASK(t, "on %d advanced to subtask %lu (not present)\n", + cpu, p->cur); + return 0; + } + } + to_relq = time_after(cur_release(t), time); + TRACE_TASK(t, "on %d advanced to subtask %lu -> to_relq=%d (cur_release:%lu time:%lu)\n", + cpu, p->cur, to_relq, cur_release(t), time); + return to_relq; +} + +static void advance_subtasks(struct pfair_cluster *cluster, quanta_t time) +{ + struct task_struct* l; + struct pfair_param* p; + struct list_head* pos; + struct pfair_state* cpu; + + list_for_each(pos, &cluster->topology.cpus) { + cpu = from_cluster_list(pos); + l = cpu->linked; + cpu->missed_updates += cpu->linked != cpu->local; + if (l) { + p = tsk_pfair(l); + p->last_quantum = time; + p->last_cpu = cpu_id(cpu); + if (advance_subtask(time, l, cpu_id(cpu))) { + //cpu->linked = NULL; + PTRACE_TASK(l, "should go to release queue. " + "scheduled_on=%d present=%d\n", + tsk_rt(l)->scheduled_on, + tsk_rt(l)->present); + } + } + } +} + +static int target_cpu(quanta_t time, struct task_struct* t, int default_cpu) +{ + int cpu; + if (tsk_rt(t)->scheduled_on != NO_CPU) { + /* always observe scheduled_on linkage */ + default_cpu = tsk_rt(t)->scheduled_on; + } else if (tsk_pfair(t)->last_quantum == time - 1) { + /* back2back quanta */ + /* Only observe last_quantum if no scheduled_on is in the way. + * This should only kick in if a CPU missed quanta, and that + * *should* only happen in QEMU. + */ + cpu = tsk_pfair(t)->last_cpu; + if (!pstate[cpu]->linked || + tsk_rt(pstate[cpu]->linked)->scheduled_on != cpu) { + default_cpu = cpu; + } + } + return default_cpu; +} + +/* returns one if linking was redirected */ +static int pfair_link(quanta_t time, int cpu, + struct task_struct* t) +{ + int target = target_cpu(time, t, cpu); + struct task_struct* prev = pstate[cpu]->linked; + struct task_struct* other; + struct pfair_cluster* cluster = cpu_cluster(pstate[cpu]); + + if (target != cpu) { + BUG_ON(pstate[target]->topology.cluster != pstate[cpu]->topology.cluster); + other = pstate[target]->linked; + pstate[target]->linked = t; + tsk_rt(t)->linked_on = target; + if (!other) + /* linked ok, but reschedule this CPU */ + return 1; + if (target < cpu) { + /* link other to cpu instead */ + tsk_rt(other)->linked_on = cpu; + pstate[cpu]->linked = other; + if (prev) { + /* prev got pushed back into the ready queue */ + tsk_rt(prev)->linked_on = NO_CPU; + __add_ready(&cluster->pfair, prev); + } + /* we are done with this cpu */ + return 0; + } else { + /* re-add other, it's original CPU was not considered yet */ + tsk_rt(other)->linked_on = NO_CPU; + __add_ready(&cluster->pfair, other); + /* reschedule this CPU */ + return 1; + } + } else { + pstate[cpu]->linked = t; + tsk_rt(t)->linked_on = cpu; + if (prev) { + /* prev got pushed back into the ready queue */ + tsk_rt(prev)->linked_on = NO_CPU; + __add_ready(&cluster->pfair, prev); + } + /* we are done with this CPU */ + return 0; + } +} + +static void schedule_subtasks(struct pfair_cluster *cluster, quanta_t time) +{ + int retry; + struct list_head *pos; + struct pfair_state *cpu_state; + + list_for_each(pos, &cluster->topology.cpus) { + cpu_state = from_cluster_list(pos); + retry = 1; +#ifdef CONFIG_RELEASE_MASTER + /* skip release master */ + if (cluster->pfair.release_master == cpu_id(cpu_state)) + continue; +#endif + while (retry) { + if (pfair_higher_prio(__peek_ready(&cluster->pfair), + cpu_state->linked)) + retry = pfair_link(time, cpu_id(cpu_state), + __take_ready(&cluster->pfair)); + else + retry = 0; + } + } +} + +static void schedule_next_quantum(struct pfair_cluster *cluster, quanta_t time) +{ + struct pfair_state *cpu; + struct list_head* pos; + + /* called with interrupts disabled */ + PTRACE("--- Q %lu at %llu PRE-SPIN\n", + time, litmus_clock()); + raw_spin_lock(cluster_lock(cluster)); + PTRACE("<<< Q %lu at %llu\n", + time, litmus_clock()); + + sched_trace_quantum_boundary(); + + advance_subtasks(cluster, time); + poll_releases(cluster); + schedule_subtasks(cluster, time); + + list_for_each(pos, &cluster->topology.cpus) { + cpu = from_cluster_list(pos); + if (cpu->linked) + PTRACE_TASK(cpu->linked, + " linked on %d.\n", cpu_id(cpu)); + else + PTRACE("(null) linked on %d.\n", cpu_id(cpu)); + } + /* We are done. Advance time. */ + mb(); + list_for_each(pos, &cluster->topology.cpus) { + cpu = from_cluster_list(pos); + if (cpu->local_tick != cpu->cur_tick) { + TRACE("BAD Quantum not acked on %d " + "(l:%lu c:%lu p:%lu)\n", + cpu_id(cpu), + cpu->local_tick, + cpu->cur_tick, + cluster->pfair_time); + cpu->missed_quanta++; + } + cpu->cur_tick = time; + } + PTRACE(">>> Q %lu at %llu\n", + time, litmus_clock()); + raw_spin_unlock(cluster_lock(cluster)); +} + +static noinline void wait_for_quantum(quanta_t q, struct pfair_state* state) +{ + quanta_t loc; + + goto first; /* skip mb() on first iteration */ + do { + cpu_relax(); + mb(); + first: loc = state->cur_tick; + /* FIXME: what if loc > cur? */ + } while (time_before(loc, q)); + PTRACE("observed cur_tick:%lu >= q:%lu\n", + loc, q); +} + +static quanta_t current_quantum(struct pfair_state* state) +{ + lt_t t = litmus_clock() - state->offset; + return time2quanta(t, FLOOR); +} + +static void catchup_quanta(quanta_t from, quanta_t target, + struct pfair_state* state) +{ + quanta_t cur = from, time; + TRACE("+++< BAD catching up quanta from %lu to %lu\n", + from, target); + while (time_before(cur, target)) { + wait_for_quantum(cur, state); + cur++; + time = cmpxchg(&cpu_cluster(state)->pfair_time, + cur - 1, /* expected */ + cur /* next */ + ); + if (time == cur - 1) + schedule_next_quantum(cpu_cluster(state), cur); + } + TRACE("+++> catching up done\n"); +} + +/* pfair_tick - this function is called for every local timer + * interrupt. + */ +static void pfair_tick(struct task_struct* t) +{ + struct pfair_state* state = &__get_cpu_var(pfair_state); + quanta_t time, cur; + int retry = 10; + + do { + cur = current_quantum(state); + PTRACE("q %lu at %llu\n", cur, litmus_clock()); + + /* Attempt to advance time. First CPU to get here + * will prepare the next quantum. + */ + time = cmpxchg(&cpu_cluster(state)->pfair_time, + cur - 1, /* expected */ + cur /* next */ + ); + if (time == cur - 1) { + /* exchange succeeded */ + wait_for_quantum(cur - 1, state); + schedule_next_quantum(cpu_cluster(state), cur); + retry = 0; + } else if (time_before(time, cur - 1)) { + /* the whole system missed a tick !? */ + catchup_quanta(time, cur, state); + retry--; + } else if (time_after(time, cur)) { + /* our timer lagging behind!? */ + TRACE("BAD pfair_time:%lu > cur:%lu\n", time, cur); + retry--; + } else { + /* Some other CPU already started scheduling + * this quantum. Let it do its job and then update. + */ + retry = 0; + } + } while (retry); + + /* Spin locally until time advances. */ + wait_for_quantum(cur, state); + + /* copy assignment */ + /* FIXME: what if we race with a future update? Corrupted state? */ + state->local = state->linked; + /* signal that we are done */ + mb(); + state->local_tick = state->cur_tick; + + if (state->local != current + && (is_realtime(current) || is_present(state->local))) + litmus_reschedule_local(); +} + +static int safe_to_schedule(struct task_struct* t, int cpu) +{ + int where = tsk_rt(t)->scheduled_on; + if (where != NO_CPU && where != cpu) { + TRACE_TASK(t, "BAD: can't be scheduled on %d, " + "scheduled already on %d.\n", cpu, where); + return 0; + } else + return tsk_rt(t)->present && get_rt_flags(t) == RT_F_RUNNING; +} + +static struct task_struct* pfair_schedule(struct task_struct * prev) +{ + struct pfair_state* state = &__get_cpu_var(pfair_state); + struct pfair_cluster* cluster = cpu_cluster(state); + int blocks, completion, out_of_time; + struct task_struct* next = NULL; + +#ifdef CONFIG_RELEASE_MASTER + /* Bail out early if we are the release master. + * The release master never schedules any real-time tasks. + */ + if (unlikely(cluster->pfair.release_master == cpu_id(state))) { + sched_state_task_picked(); + return NULL; + } +#endif + + raw_spin_lock(cpu_lock(state)); + + blocks = is_realtime(prev) && !is_running(prev); + completion = is_realtime(prev) && get_rt_flags(prev) == RT_F_SLEEP; + out_of_time = is_realtime(prev) && time_after(cur_release(prev), + state->local_tick); + + if (is_realtime(prev)) + PTRACE_TASK(prev, "blocks:%d completion:%d out_of_time:%d\n", + blocks, completion, out_of_time); + + if (completion) { + sched_trace_task_completion(prev, 0); + pfair_prepare_next_period(prev); + prepare_release(prev, cur_release(prev)); + } + + if (!blocks && (completion || out_of_time)) { + drop_all_references(prev); + sched_trace_task_release(prev); + add_release(&cluster->pfair, prev); + } + + if (state->local && safe_to_schedule(state->local, cpu_id(state))) + next = state->local; + + if (prev != next) { + tsk_rt(prev)->scheduled_on = NO_CPU; + if (next) + tsk_rt(next)->scheduled_on = cpu_id(state); + } + sched_state_task_picked(); + raw_spin_unlock(cpu_lock(state)); + + if (next) + TRACE_TASK(next, "scheduled rel=%lu at %lu (%llu)\n", + tsk_pfair(next)->release, cpu_cluster(state)->pfair_time, litmus_clock()); + else if (is_realtime(prev)) + TRACE("Becomes idle at %lu (%llu)\n", cpu_cluster(state)->pfair_time, litmus_clock()); + + return next; +} + +static void pfair_task_new(struct task_struct * t, int on_rq, int running) +{ + unsigned long flags; + struct pfair_cluster* cluster; + + TRACE("pfair: task new %d state:%d\n", t->pid, t->state); + + cluster = tsk_pfair(t)->cluster; + + raw_spin_lock_irqsave(cluster_lock(cluster), flags); + + prepare_release(t, cluster->pfair_time + 1); + + t->rt_param.scheduled_on = NO_CPU; + + if (running) { +#ifdef CONFIG_RELEASE_MASTER + if (task_cpu(t) != cluster->pfair.release_master) +#endif + t->rt_param.scheduled_on = task_cpu(t); + __add_ready(&cluster->pfair, t); + } + + check_preempt(t); + + raw_spin_unlock_irqrestore(cluster_lock(cluster), flags); +} + +static void pfair_task_wake_up(struct task_struct *t) +{ + unsigned long flags; + lt_t now; + int requeue = 0; + struct pfair_cluster* cluster; + + cluster = tsk_pfair(t)->cluster; + + TRACE_TASK(t, "wakes at %llu, release=%lu, pfair_time:%lu\n", + litmus_clock(), cur_release(t), cluster->pfair_time); + + raw_spin_lock_irqsave(cluster_lock(cluster), flags); + + /* If a task blocks and wakes before its next job release, + * then it may resume if it is currently linked somewhere + * (as if it never blocked at all). Otherwise, we have a + * new sporadic job release. + */ + requeue = tsk_rt(t)->flags == RT_F_REQUEUE; + now = litmus_clock(); + if (lt_before(get_deadline(t), now)) { + TRACE_TASK(t, "sporadic release!\n"); + release_at(t, now); + prepare_release(t, time2quanta(now, CEIL)); + sched_trace_task_release(t); + } + + /* only add to ready queue if the task isn't still linked somewhere */ + if (requeue) { + TRACE_TASK(t, "requeueing required\n"); + tsk_rt(t)->flags = RT_F_RUNNING; + __add_ready(&cluster->pfair, t); + } + + check_preempt(t); + + raw_spin_unlock_irqrestore(cluster_lock(cluster), flags); + TRACE_TASK(t, "wake up done at %llu\n", litmus_clock()); +} + +static void pfair_task_block(struct task_struct *t) +{ + BUG_ON(!is_realtime(t)); + TRACE_TASK(t, "blocks at %llu, state:%d\n", + litmus_clock(), t->state); +} + +static void pfair_task_exit(struct task_struct * t) +{ + unsigned long flags; + struct pfair_cluster *cluster; + + BUG_ON(!is_realtime(t)); + + cluster = tsk_pfair(t)->cluster; + + /* Remote task from release or ready queue, and ensure + * that it is not the scheduled task for ANY CPU. We + * do this blanket check because occassionally when + * tasks exit while blocked, the task_cpu of the task + * might not be the same as the CPU that the PFAIR scheduler + * has chosen for it. + */ + raw_spin_lock_irqsave(cluster_lock(cluster), flags); + + TRACE_TASK(t, "RIP, state:%d\n", t->state); + drop_all_references(t); + + raw_spin_unlock_irqrestore(cluster_lock(cluster), flags); + + kfree(t->rt_param.pfair); + t->rt_param.pfair = NULL; +} + + +static void pfair_release_at(struct task_struct* task, lt_t start) +{ + unsigned long flags; + quanta_t release; + + struct pfair_cluster *cluster; + + cluster = tsk_pfair(task)->cluster; + + BUG_ON(!is_realtime(task)); + + raw_spin_lock_irqsave(cluster_lock(cluster), flags); + release_at(task, start); + release = time2quanta(start, CEIL); + + TRACE_TASK(task, "sys release at %lu\n", release); + + drop_all_references(task); + prepare_release(task, release); + add_release(&cluster->pfair, task); + + raw_spin_unlock_irqrestore(cluster_lock(cluster), flags); +} + +static void init_subtask(struct subtask* sub, unsigned long i, + lt_t quanta, lt_t period) +{ + /* since i is zero-based, the formulas are shifted by one */ + lt_t tmp; + + /* release */ + tmp = period * i; + do_div(tmp, quanta); /* floor */ + sub->release = (quanta_t) tmp; + + /* deadline */ + tmp = period * (i + 1); + if (do_div(tmp, quanta)) /* ceil */ + tmp++; + sub->deadline = (quanta_t) tmp; + + /* next release */ + tmp = period * (i + 1); + do_div(tmp, quanta); /* floor */ + sub->overlap = sub->deadline - (quanta_t) tmp; + + /* Group deadline. + * Based on the formula given in Uma's thesis. + */ + if (2 * quanta >= period) { + /* heavy */ + tmp = (sub->deadline - (i + 1)) * period; + if (period > quanta && + do_div(tmp, (period - quanta))) /* ceil */ + tmp++; + sub->group_deadline = (quanta_t) tmp; + } else + sub->group_deadline = 0; +} + +static void dump_subtasks(struct task_struct* t) +{ + unsigned long i; + for (i = 0; i < t->rt_param.pfair->quanta; i++) + TRACE_TASK(t, "SUBTASK %lu: rel=%lu dl=%lu bbit:%lu gdl:%lu\n", + i + 1, + t->rt_param.pfair->subtasks[i].release, + t->rt_param.pfair->subtasks[i].deadline, + t->rt_param.pfair->subtasks[i].overlap, + t->rt_param.pfair->subtasks[i].group_deadline); +} + +static long pfair_admit_task(struct task_struct* t) +{ + lt_t quanta; + lt_t period; + s64 quantum_length = ktime_to_ns(tick_period); + struct pfair_param* param; + unsigned long i; + + /* first check that the task is in the right cluster */ + if (cpu_cluster(pstate[tsk_rt(t)->task_params.cpu]) != + cpu_cluster(pstate[task_cpu(t)])) + return -EINVAL; + + /* Pfair is a tick-based method, so the time + * of interest is jiffies. Calculate tick-based + * times for everything. + * (Ceiling of exec cost, floor of period.) + */ + + quanta = get_exec_cost(t); + period = get_rt_period(t); + + quanta = time2quanta(get_exec_cost(t), CEIL); + + if (do_div(period, quantum_length)) + printk(KERN_WARNING + "The period of %s/%d is not a multiple of %llu.\n", + t->comm, t->pid, (unsigned long long) quantum_length); + + if (quanta == period) { + /* special case: task has weight 1.0 */ + printk(KERN_INFO + "Admitting weight 1.0 task. (%s/%d, %llu, %llu).\n", + t->comm, t->pid, quanta, period); + quanta = 1; + period = 1; + } + + param = kmalloc(sizeof(*param) + + quanta * sizeof(struct subtask), GFP_ATOMIC); + + if (!param) + return -ENOMEM; + + param->quanta = quanta; + param->cur = 0; + param->release = 0; + param->period = period; + + param->cluster = cpu_cluster(pstate[tsk_rt(t)->task_params.cpu]); + + for (i = 0; i < quanta; i++) + init_subtask(param->subtasks + i, i, quanta, period); + + if (t->rt_param.pfair) + /* get rid of stale allocation */ + kfree(t->rt_param.pfair); + + t->rt_param.pfair = param; + + /* spew out some debug info */ + dump_subtasks(t); + + return 0; +} + +static void pfair_init_cluster(struct pfair_cluster* cluster) +{ + rt_domain_init(&cluster->pfair, pfair_ready_order, NULL, pfair_release_jobs); + bheap_init(&cluster->release_queue); + raw_spin_lock_init(&cluster->release_lock); + INIT_LIST_HEAD(&cluster->topology.cpus); +} + +static void cleanup_clusters(void) +{ + int i; + + if (num_pfair_clusters) + kfree(pfair_clusters); + pfair_clusters = NULL; + num_pfair_clusters = 0; + + /* avoid stale pointers */ + for (i = 0; i < num_online_cpus(); i++) { + pstate[i]->topology.cluster = NULL; + printk("P%d missed %u updates and %u quanta.\n", cpu_id(pstate[i]), + pstate[i]->missed_updates, pstate[i]->missed_quanta); + } +} + +static long pfair_activate_plugin(void) +{ + int err, i; + struct pfair_state* state; + struct pfair_cluster* cluster ; + quanta_t now; + int cluster_size; + struct cluster_cpu* cpus[NR_CPUS]; + struct scheduling_cluster* clust[NR_CPUS]; + + cluster_size = get_cluster_size(pfair_cluster_level); + + if (cluster_size <= 0 || num_online_cpus() % cluster_size != 0) + return -EINVAL; + + num_pfair_clusters = num_online_cpus() / cluster_size; + + pfair_clusters = kzalloc(num_pfair_clusters * sizeof(struct pfair_cluster), GFP_ATOMIC); + if (!pfair_clusters) { + num_pfair_clusters = 0; + printk(KERN_ERR "Could not allocate Pfair clusters!\n"); + return -ENOMEM; + } + + state = &__get_cpu_var(pfair_state); + now = current_quantum(state); + TRACE("Activating PFAIR at q=%lu\n", now); + + for (i = 0; i < num_pfair_clusters; i++) { + cluster = &pfair_clusters[i]; + pfair_init_cluster(cluster); + cluster->pfair_time = now; + clust[i] = &cluster->topology; +#ifdef CONFIG_RELEASE_MASTER + cluster->pfair.release_master = atomic_read(&release_master_cpu); +#endif + } + + for (i = 0; i < num_online_cpus(); i++) { + state = &per_cpu(pfair_state, i); + state->cur_tick = now; + state->local_tick = now; + state->missed_quanta = 0; + state->missed_updates = 0; + state->offset = cpu_stagger_offset(i); + printk(KERN_ERR "cpus[%d] set; %d\n", i, num_online_cpus()); + cpus[i] = &state->topology; + } + + err = assign_cpus_to_clusters(pfair_cluster_level, clust, num_pfair_clusters, + cpus, num_online_cpus()); + + if (err < 0) + cleanup_clusters(); + + return err; +} + +static long pfair_deactivate_plugin(void) +{ + cleanup_clusters(); + return 0; +} + +/* Plugin object */ +static struct sched_plugin pfair_plugin __cacheline_aligned_in_smp = { + .plugin_name = "PFAIR", + .tick = pfair_tick, + .task_new = pfair_task_new, + .task_exit = pfair_task_exit, + .schedule = pfair_schedule, + .task_wake_up = pfair_task_wake_up, + .task_block = pfair_task_block, + .admit_task = pfair_admit_task, + .release_at = pfair_release_at, + .complete_job = complete_job, + .activate_plugin = pfair_activate_plugin, + .deactivate_plugin = pfair_deactivate_plugin, +}; + + +static struct proc_dir_entry *cluster_file = NULL, *pfair_dir = NULL; + +static int __init init_pfair(void) +{ + int cpu, err, fs; + struct pfair_state *state; + + /* + * initialize short_cut for per-cpu pfair state; + * there may be a problem here if someone removes a cpu + * while we are doing this initialization... and if cpus + * are added / removed later... but we don't support CPU hotplug atm anyway. + */ + pstate = kmalloc(sizeof(struct pfair_state*) * num_online_cpus(), GFP_KERNEL); + + /* initialize CPU state */ + for (cpu = 0; cpu < num_online_cpus(); cpu++) { + state = &per_cpu(pfair_state, cpu); + state->topology.id = cpu; + state->cur_tick = 0; + state->local_tick = 0; + state->linked = NULL; + state->local = NULL; + state->scheduled = NULL; + state->missed_quanta = 0; + state->offset = cpu_stagger_offset(cpu); + pstate[cpu] = state; + } + + pfair_clusters = NULL; + num_pfair_clusters = 0; + + err = register_sched_plugin(&pfair_plugin); + if (!err) { + fs = make_plugin_proc_dir(&pfair_plugin, &pfair_dir); + if (!fs) + cluster_file = create_cluster_file(pfair_dir, &pfair_cluster_level); + else + printk(KERN_ERR "Could not allocate PFAIR procfs dir.\n"); + } + + return err; +} + +static void __exit clean_pfair(void) +{ + kfree(pstate); + + if (cluster_file) + remove_proc_entry("cluster", pfair_dir); + if (pfair_dir) + remove_plugin_proc_dir(&pfair_plugin); +} + +module_init(init_pfair); +module_exit(clean_pfair); diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c new file mode 100644 index 0000000..245e41c --- /dev/null +++ b/litmus/sched_plugin.c @@ -0,0 +1,360 @@ +/* sched_plugin.c -- core infrastructure for the scheduler plugin system + * + * This file includes the initialization of the plugin system, the no-op Linux + * scheduler plugin, some dummy functions, and some helper functions. + */ + +#include +#include +#include + +#include +#include +#include +#include + +#ifdef CONFIG_LITMUS_NVIDIA +#include +#endif + +/* + * Generic function to trigger preemption on either local or remote cpu + * from scheduler plugins. The key feature is that this function is + * non-preemptive section aware and does not invoke the scheduler / send + * IPIs if the to-be-preempted task is actually non-preemptive. + */ +void preempt_if_preemptable(struct task_struct* t, int cpu) +{ + /* t is the real-time task executing on CPU on_cpu If t is NULL, then + * on_cpu is currently scheduling background work. + */ + + int reschedule = 0; + + if (!t) + /* move non-real-time task out of the way */ + reschedule = 1; + else { + if (smp_processor_id() == cpu) { + /* local CPU case */ + /* check if we need to poke userspace */ + if (is_user_np(t)) + /* Yes, poke it. This doesn't have to be atomic since + * the task is definitely not executing. */ + request_exit_np(t); + else if (!is_kernel_np(t)) + /* only if we are allowed to preempt the + * currently-executing task */ + reschedule = 1; + } else { + /* Remote CPU case. Only notify if it's not a kernel + * NP section and if we didn't set the userspace + * flag. */ + reschedule = !(is_kernel_np(t) || request_exit_np_atomic(t)); + } + } + if (likely(reschedule)) + litmus_reschedule(cpu); +} + + +/************************************************************* + * Dummy plugin functions * + *************************************************************/ + +static void litmus_dummy_finish_switch(struct task_struct * prev) +{ +} + +static struct task_struct* litmus_dummy_schedule(struct task_struct * prev) +{ + sched_state_task_picked(); + return NULL; +} + +static void litmus_dummy_tick(struct task_struct* tsk) +{ +} + +static long litmus_dummy_admit_task(struct task_struct* tsk) +{ + printk(KERN_CRIT "LITMUS^RT: Linux plugin rejects %s/%d.\n", + tsk->comm, tsk->pid); + return -EINVAL; +} + +static void litmus_dummy_task_new(struct task_struct *t, int on_rq, int running) +{ +} + +static void litmus_dummy_task_wake_up(struct task_struct *task) +{ +} + +static void litmus_dummy_task_block(struct task_struct *task) +{ +} + +static void litmus_dummy_task_exit(struct task_struct *task) +{ +} + +static long litmus_dummy_complete_job(void) +{ + return -ENOSYS; +} + +static long litmus_dummy_activate_plugin(void) +{ +#ifdef CONFIG_LITMUS_NVIDIA + shutdown_nvidia_info(); +#endif + return 0; +} + +static long litmus_dummy_deactivate_plugin(void) +{ + return 0; +} + +static int litmus_dummy_compare(struct task_struct* a, struct task_struct* b) +{ + TRACE_CUR("WARNING: Dummy compare function called!\n"); + return 0; +} + +#ifdef CONFIG_LITMUS_LOCKING +static long litmus_dummy_allocate_lock(struct litmus_lock **lock, int type, + void* __user config) +{ + return -ENXIO; +} + +static void litmus_dummy_increase_prio(struct task_struct* t, struct task_struct* prio_inh) +{ +} + +static void litmus_dummy_decrease_prio(struct task_struct* t, struct task_struct* prio_inh) +{ +} +#endif + +#ifdef CONFIG_LITMUS_SOFTIRQD +static void litmus_dummy_increase_prio_klitirq(struct task_struct* klitirqd, + struct task_struct* old_owner, + struct task_struct* new_owner) +{ +} + +static void litmus_dummy_decrease_prio_klitirqd(struct task_struct* klitirqd, + struct task_struct* old_owner) +{ +} +#endif + +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD +static int litmus_dummy_enqueue_pai_tasklet(struct tasklet_struct* t) +{ + TRACE("%s: PAI Tasklet unsupported in this plugin!!!!!!\n", __FUNCTION__); + return(0); // failure. +} + +static void litmus_dummy_change_prio_pai_tasklet(struct task_struct *old_prio, + struct task_struct *new_prio) +{ + TRACE("%s: PAI Tasklet unsupported in this plugin!!!!!!\n", __FUNCTION__); +} + +static void litmus_dummy_run_tasklets(struct task_struct* t) +{ + //TRACE("%s: PAI Tasklet unsupported in this plugin!!!!!!\n", __FUNCTION__); +} +#endif + +#ifdef CONFIG_LITMUS_NESTED_LOCKING +static void litmus_dummy_nested_increase_prio(struct task_struct* t, struct task_struct* prio_inh, + raw_spinlock_t *to_unlock, unsigned long irqflags) +{ +} + +static void litmus_dummy_nested_decrease_prio(struct task_struct* t, struct task_struct* prio_inh, + raw_spinlock_t *to_unlock, unsigned long irqflags) +{ +} + +static int litmus_dummy___compare(struct task_struct* a, comparison_mode_t a_mod, + struct task_struct* b, comparison_mode_t b_mode) +{ + TRACE_CUR("WARNING: Dummy compare function called!\n"); + return 0; +} +#endif + +#ifdef CONFIG_LITMUS_DGL_SUPPORT +static raw_spinlock_t* litmus_dummy_get_dgl_spinlock(struct task_struct *t) +{ + return NULL; +} +#endif + +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING +static long litmus_dummy_allocate_aff_obs(struct affinity_observer **aff_obs, + int type, + void* __user config) +{ + return -ENXIO; +} +#endif + + +/* The default scheduler plugin. It doesn't do anything and lets Linux do its + * job. + */ +struct sched_plugin linux_sched_plugin = { + .plugin_name = "Linux", + .tick = litmus_dummy_tick, + .task_new = litmus_dummy_task_new, + .task_exit = litmus_dummy_task_exit, + .task_wake_up = litmus_dummy_task_wake_up, + .task_block = litmus_dummy_task_block, + .complete_job = litmus_dummy_complete_job, + .schedule = litmus_dummy_schedule, + .finish_switch = litmus_dummy_finish_switch, + .activate_plugin = litmus_dummy_activate_plugin, + .deactivate_plugin = litmus_dummy_deactivate_plugin, + .compare = litmus_dummy_compare, +#ifdef CONFIG_LITMUS_LOCKING + .allocate_lock = litmus_dummy_allocate_lock, + .increase_prio = litmus_dummy_increase_prio, + .decrease_prio = litmus_dummy_decrease_prio, +#endif +#ifdef CONFIG_LITMUS_NESTED_LOCKING + .nested_increase_prio = litmus_dummy_nested_increase_prio, + .nested_decrease_prio = litmus_dummy_nested_decrease_prio, + .__compare = litmus_dummy___compare, +#endif +#ifdef CONFIG_LITMUS_SOFTIRQD + .increase_prio_klitirqd = litmus_dummy_increase_prio_klitirqd, + .decrease_prio_klitirqd = litmus_dummy_decrease_prio_klitirqd, +#endif +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + .enqueue_pai_tasklet = litmus_dummy_enqueue_pai_tasklet, + .change_prio_pai_tasklet = litmus_dummy_change_prio_pai_tasklet, + .run_tasklets = litmus_dummy_run_tasklets, +#endif +#ifdef CONFIG_LITMUS_DGL_SUPPORT + .get_dgl_spinlock = litmus_dummy_get_dgl_spinlock, +#endif +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + .allocate_aff_obs = litmus_dummy_allocate_aff_obs, +#endif + + .admit_task = litmus_dummy_admit_task +}; + +/* + * The reference to current plugin that is used to schedule tasks within + * the system. It stores references to actual function implementations + * Should be initialized by calling "init_***_plugin()" + */ +struct sched_plugin *litmus = &linux_sched_plugin; + +/* the list of registered scheduling plugins */ +static LIST_HEAD(sched_plugins); +static DEFINE_RAW_SPINLOCK(sched_plugins_lock); + +#define CHECK(func) {\ + if (!plugin->func) \ + plugin->func = litmus_dummy_ ## func;} + +/* FIXME: get reference to module */ +int register_sched_plugin(struct sched_plugin* plugin) +{ + printk(KERN_INFO "Registering LITMUS^RT plugin %s.\n", + plugin->plugin_name); + + /* make sure we don't trip over null pointers later */ + CHECK(finish_switch); + CHECK(schedule); + CHECK(tick); + CHECK(task_wake_up); + CHECK(task_exit); + CHECK(task_block); + CHECK(task_new); + CHECK(complete_job); + CHECK(activate_plugin); + CHECK(deactivate_plugin); + CHECK(compare); +#ifdef CONFIG_LITMUS_LOCKING + CHECK(allocate_lock); + CHECK(increase_prio); + CHECK(decrease_prio); +#endif +#ifdef CONFIG_LITMUS_NESTED_LOCKING + CHECK(nested_increase_prio); + CHECK(nested_decrease_prio); + CHECK(__compare); +#endif +#ifdef CONFIG_LITMUS_SOFTIRQD + CHECK(increase_prio_klitirqd); + CHECK(decrease_prio_klitirqd); +#endif +#ifdef CONFIG_LITMUS_PAI_SOFTIRQD + CHECK(enqueue_pai_tasklet); + CHECK(change_prio_pai_tasklet); + CHECK(run_tasklets); +#endif +#ifdef CONFIG_LITMUS_DGL_SUPPORT + CHECK(get_dgl_spinlock); +#endif +#ifdef CONFIG_LITMUS_AFFINITY_LOCKING + CHECK(allocate_aff_obs); +#endif + CHECK(admit_task); + + if (!plugin->release_at) + plugin->release_at = release_at; + + raw_spin_lock(&sched_plugins_lock); + list_add(&plugin->list, &sched_plugins); + raw_spin_unlock(&sched_plugins_lock); + + return 0; +} + + +/* FIXME: reference counting, etc. */ +struct sched_plugin* find_sched_plugin(const char* name) +{ + struct list_head *pos; + struct sched_plugin *plugin; + + raw_spin_lock(&sched_plugins_lock); + list_for_each(pos, &sched_plugins) { + plugin = list_entry(pos, struct sched_plugin, list); + if (!strcmp(plugin->plugin_name, name)) + goto out_unlock; + } + plugin = NULL; + +out_unlock: + raw_spin_unlock(&sched_plugins_lock); + return plugin; +} + +int print_sched_plugins(char* buf, int max) +{ + int count = 0; + struct list_head *pos; + struct sched_plugin *plugin; + + raw_spin_lock(&sched_plugins_lock); + list_for_each(pos, &sched_plugins) { + plugin = list_entry(pos, struct sched_plugin, list); + count += snprintf(buf + count, max - count, "%s\n", plugin->plugin_name); + if (max - count <= 0) + break; + } + raw_spin_unlock(&sched_plugins_lock); + return count; +} diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c new file mode 100644 index 0000000..8e4a22d --- /dev/null +++ b/litmus/sched_psn_edf.c @@ -0,0 +1,645 @@ +/* + * kernel/sched_psn_edf.c + * + * Implementation of the PSN-EDF scheduler plugin. + * Based on kern/sched_part_edf.c and kern/sched_gsn_edf.c. + * + * Suspensions and non-preemptable sections are supported. + * Priority inheritance is not supported. + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + rt_domain_t domain; + int cpu; + struct task_struct* scheduled; /* only RT tasks */ +/* + * scheduling lock slock + * protects the domain and serializes scheduling decisions + */ +#define slock domain.ready_lock + +} psnedf_domain_t; + +DEFINE_PER_CPU(psnedf_domain_t, psnedf_domains); + +#define local_edf (&__get_cpu_var(psnedf_domains).domain) +#define local_pedf (&__get_cpu_var(psnedf_domains)) +#define remote_edf(cpu) (&per_cpu(psnedf_domains, cpu).domain) +#define remote_pedf(cpu) (&per_cpu(psnedf_domains, cpu)) +#define task_edf(task) remote_edf(get_partition(task)) +#define task_pedf(task) remote_pedf(get_partition(task)) + + +static void psnedf_domain_init(psnedf_domain_t* pedf, + check_resched_needed_t check, + release_jobs_t release, + int cpu) +{ + edf_domain_init(&pedf->domain, check, release); + pedf->cpu = cpu; + pedf->scheduled = NULL; +} + +static void requeue(struct task_struct* t, rt_domain_t *edf) +{ + if (t->state != TASK_RUNNING) + TRACE_TASK(t, "requeue: !TASK_RUNNING\n"); + + set_rt_flags(t, RT_F_RUNNING); + if (is_released(t, litmus_clock())) + __add_ready(edf, t); + else + add_release(edf, t); /* it has got to wait */ +} + +/* we assume the lock is being held */ +static void preempt(psnedf_domain_t *pedf) +{ + preempt_if_preemptable(pedf->scheduled, pedf->cpu); +} + +#ifdef CONFIG_LITMUS_LOCKING + +static void boost_priority(struct task_struct* t) +{ + unsigned long flags; + psnedf_domain_t* pedf = task_pedf(t); + lt_t now; + + raw_spin_lock_irqsave(&pedf->slock, flags); + now = litmus_clock(); + + TRACE_TASK(t, "priority boosted at %llu\n", now); + + tsk_rt(t)->priority_boosted = 1; + tsk_rt(t)->boost_start_time = now; + + if (pedf->scheduled != t) { + /* holder may be queued: first stop queue changes */ + raw_spin_lock(&pedf->domain.release_lock); + if (is_queued(t) && + /* If it is queued, then we need to re-order. */ + bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node) && + /* If we bubbled to the top, then we need to check for preemptions. */ + edf_preemption_needed(&pedf->domain, pedf->scheduled)) + preempt(pedf); + raw_spin_unlock(&pedf->domain.release_lock); + } /* else: nothing to do since the job is not queued while scheduled */ + + raw_spin_unlock_irqrestore(&pedf->slock, flags); +} + +static void unboost_priority(struct task_struct* t) +{ + unsigned long flags; + psnedf_domain_t* pedf = task_pedf(t); + lt_t now; + + raw_spin_lock_irqsave(&pedf->slock, flags); + now = litmus_clock(); + + /* assumption: this only happens when the job is scheduled */ + BUG_ON(pedf->scheduled != t); + + TRACE_TASK(t, "priority restored at %llu\n", now); + + /* priority boosted jobs must be scheduled */ + BUG_ON(pedf->scheduled != t); + + tsk_rt(t)->priority_boosted = 0; + tsk_rt(t)->boost_start_time = 0; + + /* check if this changes anything */ + if (edf_preemption_needed(&pedf->domain, pedf->scheduled)) + preempt(pedf); + + raw_spin_unlock_irqrestore(&pedf->slock, flags); +} + +#endif + +/* This check is trivial in partioned systems as we only have to consider + * the CPU of the partition. + */ +static int psnedf_check_resched(rt_domain_t *edf) +{ + psnedf_domain_t *pedf = container_of(edf, psnedf_domain_t, domain); + + /* because this is a callback from rt_domain_t we already hold + * the necessary lock for the ready queue + */ + if (edf_preemption_needed(edf, pedf->scheduled)) { + preempt(pedf); + return 1; + } else + return 0; +} + +static void job_completion(struct task_struct* t, int forced) +{ + sched_trace_task_completion(t,forced); + TRACE_TASK(t, "job_completion().\n"); + + set_rt_flags(t, RT_F_SLEEP); + prepare_for_next_period(t); +} + +static void psnedf_tick(struct task_struct *t) +{ + psnedf_domain_t *pedf = local_pedf; + + /* Check for inconsistency. We don't need the lock for this since + * ->scheduled is only changed in schedule, which obviously is not + * executing in parallel on this CPU + */ + BUG_ON(is_realtime(t) && t != pedf->scheduled); + + if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) { + if (!is_np(t)) { + litmus_reschedule_local(); + TRACE("psnedf_scheduler_tick: " + "%d is preemptable " + " => FORCE_RESCHED\n", t->pid); + } else if (is_user_np(t)) { + TRACE("psnedf_scheduler_tick: " + "%d is non-preemptable, " + "preemption delayed.\n", t->pid); + request_exit_np(t); + } + } +} + +static struct task_struct* psnedf_schedule(struct task_struct * prev) +{ + psnedf_domain_t* pedf = local_pedf; + rt_domain_t* edf = &pedf->domain; + struct task_struct* next; + + int out_of_time, sleep, preempt, + np, exists, blocks, resched; + + raw_spin_lock(&pedf->slock); + + /* sanity checking + * differently from gedf, when a task exits (dead) + * pedf->schedule may be null and prev _is_ realtime + */ + BUG_ON(pedf->scheduled && pedf->scheduled != prev); + BUG_ON(pedf->scheduled && !is_realtime(prev)); + + /* (0) Determine state */ + exists = pedf->scheduled != NULL; + blocks = exists && !is_running(pedf->scheduled); + out_of_time = exists && + budget_enforced(pedf->scheduled) && + budget_exhausted(pedf->scheduled); + np = exists && is_np(pedf->scheduled); + sleep = exists && get_rt_flags(pedf->scheduled) == RT_F_SLEEP; + preempt = edf_preemption_needed(edf, prev); + + /* If we need to preempt do so. + * The following checks set resched to 1 in case of special + * circumstances. + */ + resched = preempt; + + /* If a task blocks we have no choice but to reschedule. + */ + if (blocks) + resched = 1; + + /* Request a sys_exit_np() call if we would like to preempt but cannot. + * Multiple calls to request_exit_np() don't hurt. + */ + if (np && (out_of_time || preempt || sleep)) + request_exit_np(pedf->scheduled); + + /* Any task that is preemptable and either exhausts its execution + * budget or wants to sleep completes. We may have to reschedule after + * this. + */ + if (!np && (out_of_time || sleep) && !blocks) { + job_completion(pedf->scheduled, !sleep); + resched = 1; + } + + /* The final scheduling decision. Do we need to switch for some reason? + * Switch if we are in RT mode and have no task or if we need to + * resched. + */ + next = NULL; + if ((!np || blocks) && (resched || !exists)) { + /* When preempting a task that does not block, then + * re-insert it into either the ready queue or the + * release queue (if it completed). requeue() picks + * the appropriate queue. + */ + if (pedf->scheduled && !blocks) + requeue(pedf->scheduled, edf); + next = __take_ready(edf); + } else + /* Only override Linux scheduler if we have a real-time task + * scheduled that needs to continue. + */ + if (exists) + next = prev; + + if (next) { + TRACE_TASK(next, "scheduled at %llu\n", litmus_clock()); + set_rt_flags(next, RT_F_RUNNING); + } else { + TRACE("becoming idle at %llu\n", litmus_clock()); + } + + pedf->scheduled = next; + sched_state_task_picked(); + raw_spin_unlock(&pedf->slock); + + return next; +} + + +/* Prepare a task for running in RT mode + */ +static void psnedf_task_new(struct task_struct * t, int on_rq, int running) +{ + rt_domain_t* edf = task_edf(t); + psnedf_domain_t* pedf = task_pedf(t); + unsigned long flags; + + TRACE_TASK(t, "psn edf: task new, cpu = %d\n", + t->rt_param.task_params.cpu); + + /* setup job parameters */ + release_at(t, litmus_clock()); + + /* The task should be running in the queue, otherwise signal + * code will try to wake it up with fatal consequences. + */ + raw_spin_lock_irqsave(&pedf->slock, flags); + if (running) { + /* there shouldn't be anything else running at the time */ + BUG_ON(pedf->scheduled); + pedf->scheduled = t; + } else { + requeue(t, edf); + /* maybe we have to reschedule */ + preempt(pedf); + } + raw_spin_unlock_irqrestore(&pedf->slock, flags); +} + +static void psnedf_task_wake_up(struct task_struct *task) +{ + unsigned long flags; + psnedf_domain_t* pedf = task_pedf(task); + rt_domain_t* edf = task_edf(task); + lt_t now; + + TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); + raw_spin_lock_irqsave(&pedf->slock, flags); + BUG_ON(is_queued(task)); + now = litmus_clock(); + if (is_tardy(task, now) +#ifdef CONFIG_LITMUS_LOCKING + /* We need to take suspensions because of semaphores into + * account! If a job resumes after being suspended due to acquiring + * a semaphore, it should never be treated as a new job release. + */ + && !is_priority_boosted(task) +#endif + ) { + /* new sporadic release */ + release_at(task, now); + sched_trace_task_release(task); + } + + /* Only add to ready queue if it is not the currently-scheduled + * task. This could be the case if a task was woken up concurrently + * on a remote CPU before the executing CPU got around to actually + * de-scheduling the task, i.e., wake_up() raced with schedule() + * and won. + */ + if (pedf->scheduled != task) + requeue(task, edf); + + raw_spin_unlock_irqrestore(&pedf->slock, flags); + TRACE_TASK(task, "wake up done\n"); +} + +static void psnedf_task_block(struct task_struct *t) +{ + /* only running tasks can block, thus t is in no queue */ + TRACE_TASK(t, "block at %llu, state=%d\n", litmus_clock(), t->state); + + BUG_ON(!is_realtime(t)); + BUG_ON(is_queued(t)); +} + +static void psnedf_task_exit(struct task_struct * t) +{ + unsigned long flags; + psnedf_domain_t* pedf = task_pedf(t); + rt_domain_t* edf; + + raw_spin_lock_irqsave(&pedf->slock, flags); + if (is_queued(t)) { + /* dequeue */ + edf = task_edf(t); + remove(edf, t); + } + if (pedf->scheduled == t) + pedf->scheduled = NULL; + + TRACE_TASK(t, "RIP, now reschedule\n"); + + preempt(pedf); + raw_spin_unlock_irqrestore(&pedf->slock, flags); +} + +#ifdef CONFIG_LITMUS_LOCKING + +#include +#include + +/* ******************** SRP support ************************ */ + +static unsigned int psnedf_get_srp_prio(struct task_struct* t) +{ + /* assumes implicit deadlines */ + return get_rt_period(t); +} + +/* ******************** FMLP support ********************** */ + +/* struct for semaphore with priority inheritance */ +struct fmlp_semaphore { + struct litmus_lock litmus_lock; + + /* current resource holder */ + struct task_struct *owner; + + /* FIFO queue of waiting tasks */ + wait_queue_head_t wait; +}; + +static inline struct fmlp_semaphore* fmlp_from_lock(struct litmus_lock* lock) +{ + return container_of(lock, struct fmlp_semaphore, litmus_lock); +} +int psnedf_fmlp_lock(struct litmus_lock* l) +{ + struct task_struct* t = current; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + wait_queue_t wait; + unsigned long flags; + + if (!is_realtime(t)) + return -EPERM; + + spin_lock_irqsave(&sem->wait.lock, flags); + + if (sem->owner) { + /* resource is not free => must suspend and wait */ + + init_waitqueue_entry(&wait, t); + + /* FIXME: interruptible would be nice some day */ + set_task_state(t, TASK_UNINTERRUPTIBLE); + + __add_wait_queue_tail_exclusive(&sem->wait, &wait); + + TS_LOCK_SUSPEND; + + /* release lock before sleeping */ + spin_unlock_irqrestore(&sem->wait.lock, flags); + + /* We depend on the FIFO order. Thus, we don't need to recheck + * when we wake up; we are guaranteed to have the lock since + * there is only one wake up per release. + */ + + schedule(); + + TS_LOCK_RESUME; + + /* Since we hold the lock, no other task will change + * ->owner. We can thus check it without acquiring the spin + * lock. */ + BUG_ON(sem->owner != t); + } else { + /* it's ours now */ + sem->owner = t; + + /* mark the task as priority-boosted. */ + boost_priority(t); + + spin_unlock_irqrestore(&sem->wait.lock, flags); + } + + return 0; +} + +int psnedf_fmlp_unlock(struct litmus_lock* l) +{ + struct task_struct *t = current, *next; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + unsigned long flags; + int err = 0; + + spin_lock_irqsave(&sem->wait.lock, flags); + + if (sem->owner != t) { + err = -EINVAL; + goto out; + } + + /* we lose the benefit of priority boosting */ + + unboost_priority(t); + + /* check if there are jobs waiting for this resource */ + next = __waitqueue_remove_first(&sem->wait); + if (next) { + /* boost next job */ + boost_priority(next); + + /* next becomes the resouce holder */ + sem->owner = next; + + /* wake up next */ + wake_up_process(next); + } else + /* resource becomes available */ + sem->owner = NULL; + +out: + spin_unlock_irqrestore(&sem->wait.lock, flags); + return err; +} + +int psnedf_fmlp_close(struct litmus_lock* l) +{ + struct task_struct *t = current; + struct fmlp_semaphore *sem = fmlp_from_lock(l); + unsigned long flags; + + int owner; + + spin_lock_irqsave(&sem->wait.lock, flags); + + owner = sem->owner == t; + + spin_unlock_irqrestore(&sem->wait.lock, flags); + + if (owner) + psnedf_fmlp_unlock(l); + + return 0; +} + +void psnedf_fmlp_free(struct litmus_lock* lock) +{ + kfree(fmlp_from_lock(lock)); +} + +static struct litmus_lock_ops psnedf_fmlp_lock_ops = { + .close = psnedf_fmlp_close, + .lock = psnedf_fmlp_lock, + .unlock = psnedf_fmlp_unlock, + .deallocate = psnedf_fmlp_free, +}; + +static struct litmus_lock* psnedf_new_fmlp(void) +{ + struct fmlp_semaphore* sem; + + sem = kmalloc(sizeof(*sem), GFP_KERNEL); + if (!sem) + return NULL; + + sem->owner = NULL; + init_waitqueue_head(&sem->wait); + sem->litmus_lock.ops = &psnedf_fmlp_lock_ops; + + return &sem->litmus_lock; +} + +/* **** lock constructor **** */ + + +static long psnedf_allocate_lock(struct litmus_lock **lock, int type, + void* __user unused) +{ + int err = -ENXIO; + struct srp_semaphore* srp; + + /* PSN-EDF currently supports the SRP for local resources and the FMLP + * for global resources. */ + switch (type) { + case FMLP_SEM: + /* Flexible Multiprocessor Locking Protocol */ + *lock = psnedf_new_fmlp(); + if (*lock) + err = 0; + else + err = -ENOMEM; + break; + + case SRP_SEM: + /* Baker's Stack Resource Policy */ + srp = allocate_srp_semaphore(); + if (srp) { + *lock = &srp->litmus_lock; + err = 0; + } else + err = -ENOMEM; + break; + }; + + return err; +} + +#endif + + +static long psnedf_activate_plugin(void) +{ +#ifdef CONFIG_RELEASE_MASTER + int cpu; + + for_each_online_cpu(cpu) { + remote_edf(cpu)->release_master = atomic_read(&release_master_cpu); + } +#endif + +#ifdef CONFIG_LITMUS_LOCKING + get_srp_prio = psnedf_get_srp_prio; +#endif + + return 0; +} + +static long psnedf_admit_task(struct task_struct* tsk) +{ + if (task_cpu(tsk) == tsk->rt_param.task_params.cpu +#ifdef CONFIG_RELEASE_MASTER + /* don't allow tasks on release master CPU */ + && task_cpu(tsk) != remote_edf(task_cpu(tsk))->release_master +#endif + ) + return 0; + else + return -EINVAL; +} + +/* Plugin object */ +static struct sched_plugin psn_edf_plugin __cacheline_aligned_in_smp = { + .plugin_name = "PSN-EDF", + .tick = psnedf_tick, + .task_new = psnedf_task_new, + .complete_job = complete_job, + .task_exit = psnedf_task_exit, + .schedule = psnedf_schedule, + .task_wake_up = psnedf_task_wake_up, + .task_block = psnedf_task_block, + .admit_task = psnedf_admit_task, + .activate_plugin = psnedf_activate_plugin, +#ifdef CONFIG_LITMUS_LOCKING + .allocate_lock = psnedf_allocate_lock, +#endif +}; + + +static int __init init_psn_edf(void) +{ + int i; + + /* We do not really want to support cpu hotplug, do we? ;) + * However, if we are so crazy to do so, + * we cannot use num_online_cpu() + */ + for (i = 0; i < num_online_cpus(); i++) { + psnedf_domain_init(remote_pedf(i), + psnedf_check_resched, + NULL, i); + } + return register_sched_plugin(&psn_edf_plugin); +} + +module_init(init_psn_edf); + diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c new file mode 100644 index 0000000..f7f5753 --- /dev/null +++ b/litmus/sched_task_trace.c @@ -0,0 +1,509 @@ +/* + * sched_task_trace.c -- record scheduling events to a byte stream + */ + +#define NO_TASK_TRACE_DECLS + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + + +#define NUM_EVENTS (1 << (CONFIG_SCHED_TASK_TRACE_SHIFT+11)) + +#define now() litmus_clock() + +struct local_buffer { + struct st_event_record record[NUM_EVENTS]; + char flag[NUM_EVENTS]; + struct ft_buffer ftbuf; +}; + +DEFINE_PER_CPU(struct local_buffer, st_event_buffer); + +static struct ftdev st_dev; + +static int st_dev_can_open(struct ftdev *dev, unsigned int cpu) +{ + return cpu_online(cpu) ? 0 : -ENODEV; +} + +static int __init init_sched_task_trace(void) +{ + struct local_buffer* buf; + int i, ok = 0, err; + printk("Allocated %u sched_trace_xxx() events per CPU " + "(buffer size: %d bytes)\n", + NUM_EVENTS, (int) sizeof(struct local_buffer)); + + err = ftdev_init(&st_dev, THIS_MODULE, + num_online_cpus(), "sched_trace"); + if (err) + goto err_out; + + for (i = 0; i < st_dev.minor_cnt; i++) { + buf = &per_cpu(st_event_buffer, i); + ok += init_ft_buffer(&buf->ftbuf, NUM_EVENTS, + sizeof(struct st_event_record), + buf->flag, + buf->record); + st_dev.minor[i].buf = &buf->ftbuf; + } + if (ok == st_dev.minor_cnt) { + st_dev.can_open = st_dev_can_open; + err = register_ftdev(&st_dev); + if (err) + goto err_dealloc; + } else { + err = -EINVAL; + goto err_dealloc; + } + + return 0; + +err_dealloc: + ftdev_exit(&st_dev); +err_out: + printk(KERN_WARNING "Could not register sched_trace module\n"); + return err; +} + +static void __exit exit_sched_task_trace(void) +{ + ftdev_exit(&st_dev); +} + +module_init(init_sched_task_trace); +module_exit(exit_sched_task_trace); + + +static inline struct st_event_record* get_record(u8 type, struct task_struct* t) +{ + struct st_event_record* rec = NULL; + struct local_buffer* buf; + + buf = &get_cpu_var(st_event_buffer); + if (ft_buffer_start_write(&buf->ftbuf, (void**) &rec)) { + rec->hdr.type = type; + rec->hdr.cpu = smp_processor_id(); + rec->hdr.pid = t ? t->pid : 0; + rec->hdr.job = t ? t->rt_param.job_params.job_no : 0; + } else { + put_cpu_var(st_event_buffer); + } + /* rec will be NULL if it failed */ + return rec; +} + +static inline void put_record(struct st_event_record* rec) +{ + struct local_buffer* buf; + buf = &__get_cpu_var(st_event_buffer); + ft_buffer_finish_write(&buf->ftbuf, rec); + put_cpu_var(st_event_buffer); +} + +feather_callback void do_sched_trace_task_name(unsigned long id, unsigned long _task) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_NAME, t); + int i; + if (rec) { + for (i = 0; i < min(TASK_COMM_LEN, ST_NAME_LEN); i++) + rec->data.name.cmd[i] = t->comm[i]; + put_record(rec); + } +} + +feather_callback void do_sched_trace_task_param(unsigned long id, unsigned long _task) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_PARAM, t); + if (rec) { + rec->data.param.wcet = get_exec_cost(t); + rec->data.param.period = get_rt_period(t); + rec->data.param.phase = get_rt_phase(t); + rec->data.param.partition = get_partition(t); + rec->data.param.class = get_class(t); + put_record(rec); + } +} + +feather_callback void do_sched_trace_task_release(unsigned long id, unsigned long _task) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_RELEASE, t); + if (rec) { + rec->data.release.release = get_release(t); + rec->data.release.deadline = get_deadline(t); + put_record(rec); + } +} + +/* skipped: st_assigned_data, we don't use it atm */ + +feather_callback void do_sched_trace_task_switch_to(unsigned long id, + unsigned long _task) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec; + //if (is_realtime(t)) /* comment out to trace EVERYTHING */ + { + rec = get_record(ST_SWITCH_TO, t); + if (rec) { + rec->data.switch_to.when = now(); + rec->data.switch_to.exec_time = get_exec_time(t); + put_record(rec); + } + } +} + +feather_callback void do_sched_trace_task_switch_away(unsigned long id, + unsigned long _task) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec; + //if (is_realtime(t)) /* comment out to trace EVERYTHING */ + { + rec = get_record(ST_SWITCH_AWAY, t); + if (rec) { + rec->data.switch_away.when = now(); + rec->data.switch_away.exec_time = get_exec_time(t); + put_record(rec); + } + } +} + +feather_callback void do_sched_trace_task_completion(unsigned long id, + unsigned long _task, + unsigned long forced) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_COMPLETION, t); + if (rec) { + rec->data.completion.when = now(); + rec->data.completion.forced = forced; +#ifdef LITMUS_NVIDIA + rec->data.completion.nv_int_count = (u16)atomic_read(&tsk_rt(t)->nv_int_count); +#endif + put_record(rec); + } +} + +feather_callback void do_sched_trace_task_block(unsigned long id, + unsigned long _task) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_BLOCK, t); + if (rec) { + rec->data.block.when = now(); + put_record(rec); + } +} + +feather_callback void do_sched_trace_task_resume(unsigned long id, + unsigned long _task) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_RESUME, t); + if (rec) { + rec->data.resume.when = now(); + put_record(rec); + } +} + +feather_callback void do_sched_trace_sys_release(unsigned long id, + unsigned long _start) +{ + lt_t *start = (lt_t*) _start; + struct st_event_record* rec = get_record(ST_SYS_RELEASE, NULL); + if (rec) { + rec->data.sys_release.when = now(); + rec->data.sys_release.release = *start; + put_record(rec); + } +} + +feather_callback void do_sched_trace_action(unsigned long id, + unsigned long _task, + unsigned long action) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record* rec = get_record(ST_ACTION, t); + + if (rec) { + rec->data.action.when = now(); + rec->data.action.action = action; + put_record(rec); + } +} + + + + +feather_callback void do_sched_trace_prediction_err(unsigned long id, + unsigned long _task, + unsigned long _distance, + unsigned long _rel_err) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record *rec = get_record(ST_PREDICTION_ERR, t); + + if (rec) { + gpu_migration_dist_t* distance = (gpu_migration_dist_t*) _distance; + fp_t* rel_err = (fp_t*) _rel_err; + + rec->data.prediction_err.distance = *distance; + rec->data.prediction_err.rel_err = rel_err->val; + put_record(rec); + } +} + + +feather_callback void do_sched_trace_migration(unsigned long id, + unsigned long _task, + unsigned long _mig_info) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record *rec = get_record(ST_MIGRATION, t); + + if (rec) { + struct migration_info* mig_info = (struct migration_info*) _mig_info; + + rec->hdr.extra = mig_info->distance; + rec->data.migration.observed = mig_info->observed; + rec->data.migration.estimated = mig_info->estimated; + + put_record(rec); + } +} + + + + + + + + + +feather_callback void do_sched_trace_tasklet_release(unsigned long id, + unsigned long _owner) +{ + struct task_struct *t = (struct task_struct*) _owner; + struct st_event_record *rec = get_record(ST_TASKLET_RELEASE, t); + + if (rec) { + rec->data.tasklet_release.when = now(); + put_record(rec); + } +} + + +feather_callback void do_sched_trace_tasklet_begin(unsigned long id, + unsigned long _owner) +{ + struct task_struct *t = (struct task_struct*) _owner; + struct st_event_record *rec = get_record(ST_TASKLET_BEGIN, t); + + if (rec) { + rec->data.tasklet_begin.when = now(); + + if(!in_interrupt()) + rec->data.tasklet_begin.exe_pid = current->pid; + else + rec->data.tasklet_begin.exe_pid = 0; + + put_record(rec); + } +} +EXPORT_SYMBOL(do_sched_trace_tasklet_begin); + + +feather_callback void do_sched_trace_tasklet_end(unsigned long id, + unsigned long _owner, + unsigned long _flushed) +{ + struct task_struct *t = (struct task_struct*) _owner; + struct st_event_record *rec = get_record(ST_TASKLET_END, t); + + if (rec) { + rec->data.tasklet_end.when = now(); + rec->data.tasklet_end.flushed = _flushed; + + if(!in_interrupt()) + rec->data.tasklet_end.exe_pid = current->pid; + else + rec->data.tasklet_end.exe_pid = 0; + + put_record(rec); + } +} +EXPORT_SYMBOL(do_sched_trace_tasklet_end); + + +feather_callback void do_sched_trace_work_release(unsigned long id, + unsigned long _owner) +{ + struct task_struct *t = (struct task_struct*) _owner; + struct st_event_record *rec = get_record(ST_WORK_RELEASE, t); + + if (rec) { + rec->data.work_release.when = now(); + put_record(rec); + } +} + + +feather_callback void do_sched_trace_work_begin(unsigned long id, + unsigned long _owner, + unsigned long _exe) +{ + struct task_struct *t = (struct task_struct*) _owner; + struct st_event_record *rec = get_record(ST_WORK_BEGIN, t); + + if (rec) { + struct task_struct *exe = (struct task_struct*) _exe; + rec->data.work_begin.exe_pid = exe->pid; + rec->data.work_begin.when = now(); + put_record(rec); + } +} +EXPORT_SYMBOL(do_sched_trace_work_begin); + + +feather_callback void do_sched_trace_work_end(unsigned long id, + unsigned long _owner, + unsigned long _exe, + unsigned long _flushed) +{ + struct task_struct *t = (struct task_struct*) _owner; + struct st_event_record *rec = get_record(ST_WORK_END, t); + + if (rec) { + struct task_struct *exe = (struct task_struct*) _exe; + rec->data.work_end.exe_pid = exe->pid; + rec->data.work_end.flushed = _flushed; + rec->data.work_end.when = now(); + put_record(rec); + } +} +EXPORT_SYMBOL(do_sched_trace_work_end); + + +feather_callback void do_sched_trace_eff_prio_change(unsigned long id, + unsigned long _task, + unsigned long _inh) +{ + struct task_struct *t = (struct task_struct*) _task; + struct st_event_record *rec = get_record(ST_EFF_PRIO_CHANGE, t); + + if (rec) { + struct task_struct *inh = (struct task_struct*) _inh; + rec->data.effective_priority_change.when = now(); + rec->data.effective_priority_change.inh_pid = (inh != NULL) ? + inh->pid : + 0xffff; + + put_record(rec); + } +} + +/* pray for no nesting of nv interrupts on same CPU... */ +struct tracing_interrupt_map +{ + int active; + int count; + unsigned long data[128]; // assume nesting less than 128... + unsigned long serial[128]; +}; +DEFINE_PER_CPU(struct tracing_interrupt_map, active_interrupt_tracing); + + +DEFINE_PER_CPU(u32, intCounter); + +feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id, + unsigned long _device) +{ + struct st_event_record *rec; + u32 serialNum; + + { + u32* serial; + struct tracing_interrupt_map* int_map = &per_cpu(active_interrupt_tracing, smp_processor_id()); + if(!int_map->active == 0xcafebabe) + { + int_map->count++; + } + else + { + int_map->active = 0xcafebabe; + int_map->count = 1; + } + //int_map->data[int_map->count-1] = _device; + + serial = &per_cpu(intCounter, smp_processor_id()); + *serial += num_online_cpus(); + serialNum = *serial; + int_map->serial[int_map->count-1] = serialNum; + } + + rec = get_record(ST_NV_INTERRUPT_BEGIN, NULL); + if(rec) { + u32 device = _device; + rec->data.nv_interrupt_begin.when = now(); + rec->data.nv_interrupt_begin.device = device; + rec->data.nv_interrupt_begin.serialNumber = serialNum; + put_record(rec); + } +} +EXPORT_SYMBOL(do_sched_trace_nv_interrupt_begin); + +/* +int is_interrupt_tracing_active(void) +{ + struct tracing_interrupt_map* int_map = &per_cpu(active_interrupt_tracing, smp_processor_id()); + if(int_map->active == 0xcafebabe) + return 1; + return 0; +} +*/ + +feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id, unsigned long _device) +{ + struct tracing_interrupt_map* int_map = &per_cpu(active_interrupt_tracing, smp_processor_id()); + if(int_map->active == 0xcafebabe) + { + struct st_event_record *rec = get_record(ST_NV_INTERRUPT_END, NULL); + + int_map->count--; + if(int_map->count == 0) + int_map->active = 0; + + if(rec) { + u32 device = _device; + rec->data.nv_interrupt_end.when = now(); + //rec->data.nv_interrupt_end.device = int_map->data[int_map->count]; + rec->data.nv_interrupt_end.device = device; + rec->data.nv_interrupt_end.serialNumber = int_map->serial[int_map->count]; + put_record(rec); + } + } +} +EXPORT_SYMBOL(do_sched_trace_nv_interrupt_end); + + + + + + + + + diff --git a/litmus/sched_trace.c b/litmus/sched_trace.c new file mode 100644 index 0000000..f4171fd --- /dev/null +++ b/litmus/sched_trace.c @@ -0,0 +1,252 @@ +/* + * sched_trace.c -- record scheduling events to a byte stream. + */ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#define SCHED_TRACE_NAME "litmus/log" + +/* Compute size of TRACE() buffer */ +#define LITMUS_TRACE_BUF_SIZE (1 << CONFIG_SCHED_DEBUG_TRACE_SHIFT) + +/* Max length of one read from the buffer */ +#define MAX_READ_LEN (64 * 1024) + +/* Max length for one write --- by TRACE() --- to the buffer. This is used to + * allocate a per-cpu buffer for printf() formatting. */ +#define MSG_SIZE 255 + + +static DEFINE_MUTEX(reader_mutex); +static atomic_t reader_cnt = ATOMIC_INIT(0); +static DEFINE_KFIFO(debug_buffer, char, LITMUS_TRACE_BUF_SIZE); + + +static DEFINE_RAW_SPINLOCK(log_buffer_lock); +static DEFINE_PER_CPU(char[MSG_SIZE], fmt_buffer); + +/* + * sched_trace_log_message - Write to the trace buffer (log_buffer) + * + * This is the only function accessing the log_buffer from inside the + * kernel for writing. + * Concurrent access to sched_trace_log_message must be serialized using + * log_buffer_lock + * The maximum length of a formatted message is 255 + */ +void sched_trace_log_message(const char* fmt, ...) +{ + unsigned long flags; + va_list args; + size_t len; + char* buf; + + if (!atomic_read(&reader_cnt)) + /* early exit if nobody is listening */ + return; + + va_start(args, fmt); + local_irq_save(flags); + + /* format message */ + buf = __get_cpu_var(fmt_buffer); + len = vscnprintf(buf, MSG_SIZE, fmt, args); + + raw_spin_lock(&log_buffer_lock); + /* Don't copy the trailing null byte, we don't want null bytes in a + * text file. + */ + kfifo_in(&debug_buffer, buf, len); + raw_spin_unlock(&log_buffer_lock); + + local_irq_restore(flags); + va_end(args); +} + + +/* + * log_read - Read the trace buffer + * + * This function is called as a file operation from userspace. + * Readers can sleep. Access is serialized through reader_mutex + */ +static ssize_t log_read(struct file *filp, + char __user *to, size_t len, + loff_t *f_pos) +{ + /* we ignore f_pos, this is strictly sequential */ + + ssize_t error = -EINVAL; + char* mem; + + if (mutex_lock_interruptible(&reader_mutex)) { + error = -ERESTARTSYS; + goto out; + } + + if (len > MAX_READ_LEN) + len = MAX_READ_LEN; + + mem = kmalloc(len, GFP_KERNEL); + if (!mem) { + error = -ENOMEM; + goto out_unlock; + } + + error = kfifo_out(&debug_buffer, mem, len); + while (!error) { + set_current_state(TASK_INTERRUPTIBLE); + schedule_timeout(110); + if (signal_pending(current)) + error = -ERESTARTSYS; + else + error = kfifo_out(&debug_buffer, mem, len); + } + + if (error > 0 && copy_to_user(to, mem, error)) + error = -EFAULT; + + kfree(mem); + out_unlock: + mutex_unlock(&reader_mutex); + out: + return error; +} + +/* + * Enable redirection of printk() messages to the trace buffer. + * Defined in kernel/printk.c + */ +extern int trace_override; +extern int trace_recurse; + +/* + * log_open - open the global log message ring buffer. + */ +static int log_open(struct inode *in, struct file *filp) +{ + int error = -EINVAL; + + if (mutex_lock_interruptible(&reader_mutex)) { + error = -ERESTARTSYS; + goto out; + } + + atomic_inc(&reader_cnt); + error = 0; + + printk(KERN_DEBUG + "sched_trace kfifo with buffer starting at: 0x%p\n", + debug_buffer.buf); + + /* override printk() */ + trace_override++; + + mutex_unlock(&reader_mutex); + out: + return error; +} + +static int log_release(struct inode *in, struct file *filp) +{ + int error = -EINVAL; + + if (mutex_lock_interruptible(&reader_mutex)) { + error = -ERESTARTSYS; + goto out; + } + + atomic_dec(&reader_cnt); + + /* release printk() overriding */ + trace_override--; + + printk(KERN_DEBUG "sched_trace kfifo released\n"); + + mutex_unlock(&reader_mutex); + out: + return error; +} + +/* + * log_fops - The file operations for accessing the global LITMUS log message + * buffer. + * + * Except for opening the device file it uses the same operations as trace_fops. + */ +static struct file_operations log_fops = { + .owner = THIS_MODULE, + .open = log_open, + .release = log_release, + .read = log_read, +}; + +static struct miscdevice litmus_log_dev = { + .name = SCHED_TRACE_NAME, + .minor = MISC_DYNAMIC_MINOR, + .fops = &log_fops, +}; + +#ifdef CONFIG_MAGIC_SYSRQ +void dump_trace_buffer(int max) +{ + char line[80]; + int len; + int count = 0; + + /* potential, but very unlikely, race... */ + trace_recurse = 1; + while ((max == 0 || count++ < max) && + (len = kfifo_out(&debug_buffer, line, sizeof(line - 1))) > 0) { + line[len] = '\0'; + printk("%s", line); + } + trace_recurse = 0; +} + +static void sysrq_dump_trace_buffer(int key) +{ + dump_trace_buffer(100); +} + +static struct sysrq_key_op sysrq_dump_trace_buffer_op = { + .handler = sysrq_dump_trace_buffer, + .help_msg = "dump-trace-buffer(Y)", + .action_msg = "writing content of TRACE() buffer", +}; +#endif + +static int __init init_sched_trace(void) +{ + printk("Initializing TRACE() device\n"); + +#ifdef CONFIG_MAGIC_SYSRQ + /* offer some debugging help */ + if (!register_sysrq_key('y', &sysrq_dump_trace_buffer_op)) + printk("Registered dump-trace-buffer(Y) magic sysrq.\n"); + else + printk("Could not register dump-trace-buffer(Y) magic sysrq.\n"); +#endif + + return misc_register(&litmus_log_dev); +} + +static void __exit exit_sched_trace(void) +{ + misc_deregister(&litmus_log_dev); +} + +module_init(init_sched_trace); +module_exit(exit_sched_trace); diff --git a/litmus/sched_trace_external.c b/litmus/sched_trace_external.c new file mode 100644 index 0000000..cf8e1d7 --- /dev/null +++ b/litmus/sched_trace_external.c @@ -0,0 +1,64 @@ +#include + +#include +#include +#include + +void __sched_trace_tasklet_begin_external(struct task_struct* t) +{ + sched_trace_tasklet_begin(t); +} +EXPORT_SYMBOL(__sched_trace_tasklet_begin_external); + +void __sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed) +{ + sched_trace_tasklet_end(t, flushed); +} +EXPORT_SYMBOL(__sched_trace_tasklet_end_external); + + + +void __sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e) +{ + sched_trace_work_begin(t, e); +} +EXPORT_SYMBOL(__sched_trace_work_begin_external); + +void __sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f) +{ + sched_trace_work_end(t, e, f); +} +EXPORT_SYMBOL(__sched_trace_work_end_external); + + + +void __sched_trace_nv_interrupt_begin_external(u32 device) +{ + //unsigned long _device = device; + sched_trace_nv_interrupt_begin((unsigned long)device); +} +EXPORT_SYMBOL(__sched_trace_nv_interrupt_begin_external); + +void __sched_trace_nv_interrupt_end_external(u32 device) +{ + //unsigned long _device = device; + sched_trace_nv_interrupt_end((unsigned long)device); +} +EXPORT_SYMBOL(__sched_trace_nv_interrupt_end_external); + + +#ifdef CONFIG_LITMUS_NVIDIA + +#define EXX_TS(evt) \ +void __##evt(void) { evt; } \ +EXPORT_SYMBOL(__##evt); + +EXX_TS(TS_NV_TOPISR_START) +EXX_TS(TS_NV_TOPISR_END) +EXX_TS(TS_NV_BOTISR_START) +EXX_TS(TS_NV_BOTISR_END) +EXX_TS(TS_NV_RELEASE_BOTISR_START) +EXX_TS(TS_NV_RELEASE_BOTISR_END) + +#endif + diff --git a/litmus/srp.c b/litmus/srp.c new file mode 100644 index 0000000..2ed4ec1 --- /dev/null +++ b/litmus/srp.c @@ -0,0 +1,295 @@ +/* ************************************************************************** */ +/* STACK RESOURCE POLICY */ +/* ************************************************************************** */ + +#include +#include +#include + +#include +#include +#include +#include + + +#ifdef CONFIG_LITMUS_LOCKING + +#include + +srp_prioritization_t get_srp_prio; + +struct srp { + struct list_head ceiling; + wait_queue_head_t ceiling_blocked; +}; +#define system_ceiling(srp) list2prio(srp->ceiling.next) +#define ceiling2sem(c) container_of(c, struct srp_semaphore, ceiling) + +#define UNDEF_SEM -2 + +atomic_t srp_objects_in_use = ATOMIC_INIT(0); + +DEFINE_PER_CPU(struct srp, srp); + +/* Initialize SRP semaphores at boot time. */ +static int __init srp_init(void) +{ + int i; + + printk("Initializing SRP per-CPU ceilings..."); + for (i = 0; i < NR_CPUS; i++) { + init_waitqueue_head(&per_cpu(srp, i).ceiling_blocked); + INIT_LIST_HEAD(&per_cpu(srp, i).ceiling); + } + printk(" done!\n"); + + return 0; +} +module_init(srp_init); + +/* SRP task priority comparison function. Smaller numeric values have higher + * priority, tie-break is PID. Special case: priority == 0 <=> no priority + */ +static int srp_higher_prio(struct srp_priority* first, + struct srp_priority* second) +{ + if (!first->priority) + return 0; + else + return !second->priority || + first->priority < second->priority || ( + first->priority == second->priority && + first->pid < second->pid); +} + + +static int srp_exceeds_ceiling(struct task_struct* first, + struct srp* srp) +{ + struct srp_priority prio; + + if (list_empty(&srp->ceiling)) + return 1; + else { + prio.pid = first->pid; + prio.priority = get_srp_prio(first); + return srp_higher_prio(&prio, system_ceiling(srp)) || + ceiling2sem(system_ceiling(srp))->owner == first; + } +} + +static void srp_add_prio(struct srp* srp, struct srp_priority* prio) +{ + struct list_head *pos; + if (in_list(&prio->list)) { + printk(KERN_CRIT "WARNING: SRP violation detected, prio is already in " + "ceiling list! cpu=%d, srp=%p\n", smp_processor_id(), ceiling2sem(prio)); + return; + } + list_for_each(pos, &srp->ceiling) + if (unlikely(srp_higher_prio(prio, list2prio(pos)))) { + __list_add(&prio->list, pos->prev, pos); + return; + } + + list_add_tail(&prio->list, &srp->ceiling); +} + + +static int lock_srp_semaphore(struct litmus_lock* l) +{ + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); + + if (!is_realtime(current)) + return -EPERM; + + preempt_disable(); + + /* Update ceiling. */ + srp_add_prio(&__get_cpu_var(srp), &sem->ceiling); + + /* SRP invariant: all resources available */ + BUG_ON(sem->owner != NULL); + + sem->owner = current; + TRACE_CUR("acquired srp 0x%p\n", sem); + + preempt_enable(); + + return 0; +} + +static int unlock_srp_semaphore(struct litmus_lock* l) +{ + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); + int err = 0; + + preempt_disable(); + + if (sem->owner != current) { + err = -EINVAL; + } else { + /* Determine new system priority ceiling for this CPU. */ + BUG_ON(!in_list(&sem->ceiling.list)); + + list_del(&sem->ceiling.list); + sem->owner = NULL; + + /* Wake tasks on this CPU, if they exceed current ceiling. */ + TRACE_CUR("released srp 0x%p\n", sem); + wake_up_all(&__get_cpu_var(srp).ceiling_blocked); + } + + preempt_enable(); + return err; +} + +static int open_srp_semaphore(struct litmus_lock* l, void* __user arg) +{ + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); + int err = 0; + struct task_struct* t = current; + struct srp_priority t_prio; + + if (!is_realtime(t)) + return -EPERM; + + TRACE_CUR("opening SRP semaphore %p, cpu=%d\n", sem, sem->cpu); + + preempt_disable(); + + if (sem->owner != NULL) + err = -EBUSY; + + if (err == 0) { + if (sem->cpu == UNDEF_SEM) + sem->cpu = get_partition(t); + else if (sem->cpu != get_partition(t)) + err = -EPERM; + } + + if (err == 0) { + t_prio.priority = get_srp_prio(t); + t_prio.pid = t->pid; + if (srp_higher_prio(&t_prio, &sem->ceiling)) { + sem->ceiling.priority = t_prio.priority; + sem->ceiling.pid = t_prio.pid; + } + } + + preempt_enable(); + + return err; +} + +static int close_srp_semaphore(struct litmus_lock* l) +{ + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); + int err = 0; + + preempt_disable(); + + if (sem->owner == current) + unlock_srp_semaphore(l); + + preempt_enable(); + + return err; +} + +static void deallocate_srp_semaphore(struct litmus_lock* l) +{ + struct srp_semaphore* sem = container_of(l, struct srp_semaphore, litmus_lock); + atomic_dec(&srp_objects_in_use); + kfree(sem); +} + +static struct litmus_lock_ops srp_lock_ops = { + .open = open_srp_semaphore, + .close = close_srp_semaphore, + .lock = lock_srp_semaphore, + .unlock = unlock_srp_semaphore, + .deallocate = deallocate_srp_semaphore, +}; + +struct srp_semaphore* allocate_srp_semaphore(void) +{ + struct srp_semaphore* sem; + + sem = kmalloc(sizeof(*sem), GFP_KERNEL); + if (!sem) + return NULL; + + INIT_LIST_HEAD(&sem->ceiling.list); + sem->ceiling.priority = 0; + sem->cpu = UNDEF_SEM; + sem->owner = NULL; + + sem->litmus_lock.ops = &srp_lock_ops; + + atomic_inc(&srp_objects_in_use); + return sem; +} + +static int srp_wake_up(wait_queue_t *wait, unsigned mode, int sync, + void *key) +{ + int cpu = smp_processor_id(); + struct task_struct *tsk = wait->private; + if (cpu != get_partition(tsk)) + TRACE_TASK(tsk, "srp_wake_up on wrong cpu, partition is %d\b", + get_partition(tsk)); + else if (srp_exceeds_ceiling(tsk, &__get_cpu_var(srp))) + return default_wake_function(wait, mode, sync, key); + return 0; +} + +static void do_ceiling_block(struct task_struct *tsk) +{ + wait_queue_t wait = { + .private = tsk, + .func = srp_wake_up, + .task_list = {NULL, NULL} + }; + + tsk->state = TASK_UNINTERRUPTIBLE; + add_wait_queue(&__get_cpu_var(srp).ceiling_blocked, &wait); + tsk->rt_param.srp_non_recurse = 1; + preempt_enable_no_resched(); + schedule(); + preempt_disable(); + tsk->rt_param.srp_non_recurse = 0; + remove_wait_queue(&__get_cpu_var(srp).ceiling_blocked, &wait); +} + +/* Wait for current task priority to exceed system-wide priority ceiling. + * FIXME: the hotpath should be inline. + */ +void srp_ceiling_block(void) +{ + struct task_struct *tsk = current; + + /* Only applies to real-time tasks, but optimize for RT tasks. */ + if (unlikely(!is_realtime(tsk))) + return; + + /* Avoid recursive ceiling blocking. */ + if (unlikely(tsk->rt_param.srp_non_recurse)) + return; + + /* Bail out early if there aren't any SRP resources around. */ + if (likely(!atomic_read(&srp_objects_in_use))) + return; + + preempt_disable(); + if (!srp_exceeds_ceiling(tsk, &__get_cpu_var(srp))) { + TRACE_CUR("is priority ceiling blocked.\n"); + while (!srp_exceeds_ceiling(tsk, &__get_cpu_var(srp))) + do_ceiling_block(tsk); + TRACE_CUR("finally exceeds system ceiling.\n"); + } else + TRACE_CUR("is not priority ceiling blocked\n"); + preempt_enable(); +} + +#endif diff --git a/litmus/sync.c b/litmus/sync.c new file mode 100644 index 0000000..bf75fde --- /dev/null +++ b/litmus/sync.c @@ -0,0 +1,104 @@ +/* litmus/sync.c - Support for synchronous and asynchronous task system releases. + * + * + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +static DECLARE_COMPLETION(ts_release); + +static long do_wait_for_ts_release(void) +{ + long ret = 0; + + /* If the interruption races with a release, the completion object + * may have a non-zero counter. To avoid this problem, this should + * be replaced by wait_for_completion(). + * + * For debugging purposes, this is interruptible for now. + */ + ret = wait_for_completion_interruptible(&ts_release); + + return ret; +} + +int count_tasks_waiting_for_release(void) +{ + unsigned long flags; + int task_count = 0; + struct list_head *pos; + + spin_lock_irqsave(&ts_release.wait.lock, flags); + list_for_each(pos, &ts_release.wait.task_list) { + task_count++; + } + spin_unlock_irqrestore(&ts_release.wait.lock, flags); + + return task_count; +} + +static long do_release_ts(lt_t start) +{ + int task_count = 0; + unsigned long flags; + struct list_head *pos; + struct task_struct *t; + + + spin_lock_irqsave(&ts_release.wait.lock, flags); + TRACE("<<<<<< synchronous task system release >>>>>>\n"); + + sched_trace_sys_release(&start); + list_for_each(pos, &ts_release.wait.task_list) { + t = (struct task_struct*) list_entry(pos, + struct __wait_queue, + task_list)->private; + task_count++; + litmus->release_at(t, start + t->rt_param.task_params.phase); + sched_trace_task_release(t); + } + + spin_unlock_irqrestore(&ts_release.wait.lock, flags); + + complete_n(&ts_release, task_count); + + return task_count; +} + + +asmlinkage long sys_wait_for_ts_release(void) +{ + long ret = -EPERM; + struct task_struct *t = current; + + if (is_realtime(t)) + ret = do_wait_for_ts_release(); + + return ret; +} + + +asmlinkage long sys_release_ts(lt_t __user *__delay) +{ + long ret; + lt_t delay; + + /* FIXME: check capabilities... */ + + ret = copy_from_user(&delay, __delay, sizeof(delay)); + if (ret == 0) + ret = do_release_ts(litmus_clock() + delay); + + return ret; +} diff --git a/litmus/trace.c b/litmus/trace.c new file mode 100644 index 0000000..3c35c52 --- /dev/null +++ b/litmus/trace.c @@ -0,0 +1,225 @@ +#include +#include +#include + +#include +#include +#include + +/******************************************************************************/ +/* Allocation */ +/******************************************************************************/ + +static struct ftdev overhead_dev; + +#define trace_ts_buf overhead_dev.minor[0].buf + +static unsigned int ts_seq_no = 0; + +DEFINE_PER_CPU(atomic_t, irq_fired_count); + +static inline void clear_irq_fired(void) +{ + atomic_set(&__raw_get_cpu_var(irq_fired_count), 0); +} + +static inline unsigned int get_and_clear_irq_fired(void) +{ + /* This is potentially not atomic since we might migrate if + * preemptions are not disabled. As a tradeoff between + * accuracy and tracing overheads, this seems acceptable. + * If it proves to be a problem, then one could add a callback + * from the migration code to invalidate irq_fired_count. + */ + return atomic_xchg(&__raw_get_cpu_var(irq_fired_count), 0); +} + +static inline void __save_irq_flags(struct timestamp *ts) +{ + unsigned int irq_count; + + irq_count = get_and_clear_irq_fired(); + /* Store how many interrupts occurred. */ + ts->irq_count = irq_count; + /* Extra flag because ts->irq_count overflows quickly. */ + ts->irq_flag = irq_count > 0; +} + +static inline void __save_timestamp_cpu(unsigned long event, + uint8_t type, uint8_t cpu) +{ + unsigned int seq_no; + struct timestamp *ts; + seq_no = fetch_and_inc((int *) &ts_seq_no); + if (ft_buffer_start_write(trace_ts_buf, (void**) &ts)) { + ts->event = event; + ts->seq_no = seq_no; + ts->cpu = cpu; + ts->task_type = type; + __save_irq_flags(ts); + barrier(); + /* prevent re-ordering of ft_timestamp() */ + ts->timestamp = ft_timestamp(); + ft_buffer_finish_write(trace_ts_buf, ts); + } +} + +static void __add_timestamp_user(struct timestamp *pre_recorded) +{ + unsigned int seq_no; + struct timestamp *ts; + seq_no = fetch_and_inc((int *) &ts_seq_no); + if (ft_buffer_start_write(trace_ts_buf, (void**) &ts)) { + *ts = *pre_recorded; + ts->seq_no = seq_no; + __save_irq_flags(ts); + ft_buffer_finish_write(trace_ts_buf, ts); + } +} + +static inline void __save_timestamp(unsigned long event, + uint8_t type) +{ + __save_timestamp_cpu(event, type, raw_smp_processor_id()); +} + +feather_callback void save_timestamp(unsigned long event) +{ + __save_timestamp(event, TSK_UNKNOWN); +} + +feather_callback void save_timestamp_def(unsigned long event, + unsigned long type) +{ + __save_timestamp(event, (uint8_t) type); +} + +feather_callback void save_timestamp_task(unsigned long event, + unsigned long t_ptr) +{ + int rt = is_realtime((struct task_struct *) t_ptr); + __save_timestamp(event, rt ? TSK_RT : TSK_BE); +} + +feather_callback void save_timestamp_cpu(unsigned long event, + unsigned long cpu) +{ + __save_timestamp_cpu(event, TSK_UNKNOWN, cpu); +} + +feather_callback void save_task_latency(unsigned long event, + unsigned long when_ptr) +{ + lt_t now = litmus_clock(); + lt_t *when = (lt_t*) when_ptr; + unsigned int seq_no; + int cpu = raw_smp_processor_id(); + struct timestamp *ts; + + seq_no = fetch_and_inc((int *) &ts_seq_no); + if (ft_buffer_start_write(trace_ts_buf, (void**) &ts)) { + ts->event = event; + ts->timestamp = now - *when; + ts->seq_no = seq_no; + ts->cpu = cpu; + ts->task_type = TSK_RT; + __save_irq_flags(ts); + ft_buffer_finish_write(trace_ts_buf, ts); + } +} + +/******************************************************************************/ +/* DEVICE FILE DRIVER */ +/******************************************************************************/ + +/* + * should be 8M; it is the max we can ask to buddy system allocator (MAX_ORDER) + * and we might not get as much + */ +#define NO_TIMESTAMPS (2 << 16) + +static int alloc_timestamp_buffer(struct ftdev* ftdev, unsigned int idx) +{ + unsigned int count = NO_TIMESTAMPS; + + /* An overhead-tracing timestamp should be exactly 16 bytes long. */ + BUILD_BUG_ON(sizeof(struct timestamp) != 16); + + while (count && !trace_ts_buf) { + printk("time stamp buffer: trying to allocate %u time stamps.\n", count); + ftdev->minor[idx].buf = alloc_ft_buffer(count, sizeof(struct timestamp)); + count /= 2; + } + return ftdev->minor[idx].buf ? 0 : -ENOMEM; +} + +static void free_timestamp_buffer(struct ftdev* ftdev, unsigned int idx) +{ + free_ft_buffer(ftdev->minor[idx].buf); + ftdev->minor[idx].buf = NULL; +} + +static ssize_t write_timestamp_from_user(struct ft_buffer* buf, size_t len, + const char __user *from) +{ + ssize_t consumed = 0; + struct timestamp ts; + + /* don't give us partial timestamps */ + if (len % sizeof(ts)) + return -EINVAL; + + while (len >= sizeof(ts)) { + if (copy_from_user(&ts, from, sizeof(ts))) { + consumed = -EFAULT; + goto out; + } + len -= sizeof(ts); + from += sizeof(ts); + consumed += sizeof(ts); + + __add_timestamp_user(&ts); + } + +out: + return consumed; +} + +static int __init init_ft_overhead_trace(void) +{ + int err, cpu; + + printk("Initializing Feather-Trace overhead tracing device.\n"); + err = ftdev_init(&overhead_dev, THIS_MODULE, 1, "ft_trace"); + if (err) + goto err_out; + + overhead_dev.alloc = alloc_timestamp_buffer; + overhead_dev.free = free_timestamp_buffer; + overhead_dev.write = write_timestamp_from_user; + + err = register_ftdev(&overhead_dev); + if (err) + goto err_dealloc; + + /* initialize IRQ flags */ + for (cpu = 0; cpu < NR_CPUS; cpu++) { + clear_irq_fired(); + } + + return 0; + +err_dealloc: + ftdev_exit(&overhead_dev); +err_out: + printk(KERN_WARNING "Could not register ft_trace module.\n"); + return err; +} + +static void __exit exit_ft_overhead_trace(void) +{ + ftdev_exit(&overhead_dev); +} + +module_init(init_ft_overhead_trace); +module_exit(exit_ft_overhead_trace); -- 1.7.10.4