Stripped the schedule even further. Booting with FW.
authorJens Krieg <jkrieg@mailbox.tu-berlin.de>
Wed, 8 Jan 2014 18:27:07 +0000 (19:27 +0100)
committerJens Krieg <jkrieg@mailbox.tu-berlin.de>
Wed, 8 Jan 2014 18:27:07 +0000 (19:27 +0100)
sys/kern/sched_ule.c

index 0894425..edb72d8 100644 (file)
@@ -322,7 +322,7 @@ static int sched_pickcpu(struct thread *, int);
 //static int sched_balance_pair(struct tdq *, struct tdq *);
 static inline struct tdq *sched_setcpu(struct thread *, int, int);
 static inline void thread_unblock_switch(struct thread *, struct mtx *);
-static struct mtx *sched_switch_migrate(struct tdq *, struct thread *, int);
+//static struct mtx *sched_switch_migrate(struct tdq *, struct thread *, int);
 static int sysctl_kern_sched_topology_spec(SYSCTL_HANDLER_ARGS);
 static int sysctl_kern_sched_topology_spec_internal(struct sbuf *sb,
     struct cpu_group *cg, int indent);
@@ -450,75 +450,74 @@ sched_shouldpreempt(int pri, int cpri, int remote)
 static __inline void
 tdq_runq_add(struct tdq *tdq, struct thread *td, int flags)
 {
-       struct td_sched *ts;
-       struct fw_queues *q = cpu_queue(0);
-       struct fw_task *p = &td->td_sched->fw_task;
-//     u_char pri;
-
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED);
-       THREAD_LOCK_ASSERT(td, MA_OWNED);
-
-//     pri = td->td_priority;
-//     printf("THREAD PRIORITY %d\n", pri);
-       ts = td->td_sched;
-       TD_SET_RUNQ(td);
-       if (THREAD_CAN_MIGRATE(td)) {
-               tdq->tdq_transferable++;
-               ts->ts_flags |= TSF_XFERABLE;
-       }
-
-//     if (pri < PRI_MIN_BATCH) {
-//             ts->ts_runq = &tdq->tdq_realtime;
-//     } else if (pri <= PRI_MAX_BATCH) {
-//             ts->ts_runq = &tdq->tdq_timeshare;
-//             KASSERT(pri <= PRI_MAX_BATCH && pri >= PRI_MIN_BATCH,
-//                     ("Invalid priority %d on timeshare runq", pri));
+//     struct td_sched *ts;
+//     struct fw_queues *q = cpu_queue(0);
+//     struct fw_task *p = &td->td_sched->fw_task;
+////   u_char pri;
+//
+//     TDQ_LOCK_ASSERT(tdq, MA_OWNED);
+//     THREAD_LOCK_ASSERT(td, MA_OWNED);
+//
+////   pri = td->td_priority;
+////   printf("THREAD PRIORITY %d\n", pri);
+//     ts = td->td_sched;
+//     TD_SET_RUNQ(td);
+//     if (THREAD_CAN_MIGRATE(td)) {
+//             tdq->tdq_transferable++;
+//             ts->ts_flags |= TSF_XFERABLE;
+//     }
+//
+////   if (pri < PRI_MIN_BATCH) {
+////           ts->ts_runq = &tdq->tdq_realtime;
+////   } else if (pri <= PRI_MAX_BATCH) {
+////           ts->ts_runq = &tdq->tdq_timeshare;
+////           KASSERT(pri <= PRI_MAX_BATCH && pri >= PRI_MIN_BATCH,
+////                   ("Invalid priority %d on timeshare runq", pri));
+////           /*
+////            * This queue contains only priorities between MIN and MAX
+////            * realtime.  Use the whole queue to represent these values.
+////            */
+////           if ((flags & (SRQ_BORROWING|SRQ_PREEMPTED)) == 0) {
+////                   pri = RQ_NQS * (pri - PRI_MIN_BATCH) / PRI_BATCH_RANGE;
+////                   pri = (pri + tdq->tdq_idx) % RQ_NQS;
+////                   /*
+////                    * This effectively shortens the queue by one so we
+////                    * can have a one slot difference between idx and
+////                    * ridx while we wait for threads to drain.
+////                    */
+////                   if (tdq->tdq_ridx != tdq->tdq_idx &&
+////                       pri == tdq->tdq_ridx)
+////                           pri = (unsigned char)(pri - 1) % RQ_NQS;
+////           } else
+////                   pri = tdq->tdq_ridx;
+////           runq_add_pri(ts->ts_runq, td, pri, flags);
+////           return;
+////   } else
+////           ts->ts_runq = &tdq->tdq_idle;
+////   runq_add(ts->ts_runq, td, flags);
+//
+//     /* Insert thread at the tail. We have no preemption anyway. */
+////   flags &= ~SRQ_PREEMPTED;
+//
+////   printf("tdq_runq_add\n");
+////   tdq_print(TDQ_ID(tdq));
+//
+//     if (td->td_flags & TDF_IDLETD) {
+//             ts->ts_runq = &tdq->tdq_idle;
+//             runq_add_pri(ts->ts_runq, td, 0, flags);
+//     }
+//     else {
+//
 //             /*
-//              * This queue contains only priorities between MIN and MAX
-//              * realtime.  Use the whole queue to represent these values.
+//              * Announce task to framework.
 //              */
-//             if ((flags & (SRQ_BORROWING|SRQ_PREEMPTED)) == 0) {
-//                     pri = RQ_NQS * (pri - PRI_MIN_BATCH) / PRI_BATCH_RANGE;
-//                     pri = (pri + tdq->tdq_idx) % RQ_NQS;
-//                     /*
-//                      * This effectively shortens the queue by one so we
-//                      * can have a one slot difference between idx and
-//                      * ridx while we wait for threads to drain.
-//                      */
-//                     if (tdq->tdq_ridx != tdq->tdq_idx &&
-//                         pri == tdq->tdq_ridx)
-//                             pri = (unsigned char)(pri - 1) % RQ_NQS;
-//             } else
-//                     pri = tdq->tdq_ridx;
-//             runq_add_pri(ts->ts_runq, td, pri, flags);
-//             return;
-//     } else
-//             ts->ts_runq = &tdq->tdq_idle;
-//     runq_add(ts->ts_runq, td, flags);
-
-       /* Insert thread at the tail. We have no preemption anyway. */
-//     flags &= ~SRQ_PREEMPTED;
-
-//     printf("tdq_runq_add\n");
-//     tdq_print(TDQ_ID(tdq));
-
-       if (td->td_flags & TDF_IDLETD) {
-               ts->ts_runq = &tdq->tdq_idle;
-               runq_add_pri(ts->ts_runq, td, 0, flags);
-       }
-       else {
-
-               /*
-                * Announce task to framework.
-                */
-               p->real_task = td;
-               p->state = FW_READY;
-               fw_notify(FW_ADMIT, (void*)p);
-
-               ts->ts_runq = &tdq->tdq_timeshare;
-               runq_add_pri(ts->ts_runq, td, 0, flags);
-       }
-
+//             p->real_task = td;
+//             p->state = FW_READY;
+//             fw_notify(FW_ADMIT, (void*)p);
+//
+//             ts->ts_runq = &tdq->tdq_timeshare;
+//             runq_add_pri(ts->ts_runq, td, 0, flags);
+//     }
 }
 
 /*
@@ -529,30 +528,24 @@ tdq_runq_add(struct tdq *tdq, struct thread *td, int flags)
 static __inline void
 tdq_runq_rem(struct tdq *tdq, struct thread *td)
 {
-       struct td_sched *ts;
-
-       ts = td->td_sched;
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED);
-       KASSERT(ts->ts_runq != NULL,
-           ("tdq_runq_remove: thread %p null ts_runq", td));
-       if (ts->ts_flags & TSF_XFERABLE) {
-               tdq->tdq_transferable--;
-               ts->ts_flags &= ~TSF_XFERABLE;
-       }
-
-       /**
-        * Remove framework task.
-        */
-//     fw_notify(FW_RELEASE, (void*)&td->td_sched->fw_task);
-//     fw_notify(FW_EVENT_WAIT, (void*)&td->td_sched->fw_task);
-
-       if (ts->ts_runq == &tdq->tdq_timeshare) {
-               if (tdq->tdq_idx != tdq->tdq_ridx)
-                       runq_remove_idx(ts->ts_runq, td, &tdq->tdq_ridx);
-               else
-                       runq_remove_idx(ts->ts_runq, td, NULL);
-       } else
-               runq_remove(ts->ts_runq, td);
+//     struct td_sched *ts;
+//
+//     ts = td->td_sched;
+//     TDQ_LOCK_ASSERT(tdq, MA_OWNED);
+//     KASSERT(ts->ts_runq != NULL,
+//         ("tdq_runq_remove: thread %p null ts_runq", td));
+//     if (ts->ts_flags & TSF_XFERABLE) {
+//             tdq->tdq_transferable--;
+//             ts->ts_flags &= ~TSF_XFERABLE;
+//     }
+//
+//     if (ts->ts_runq == &tdq->tdq_timeshare) {
+//             if (tdq->tdq_idx != tdq->tdq_ridx)
+//                     runq_remove_idx(ts->ts_runq, td, &tdq->tdq_ridx);
+//             else
+//                     runq_remove_idx(ts->ts_runq, td, NULL);
+//     } else
+//             runq_remove(ts->ts_runq, td);
 }
 
 /*
@@ -562,11 +555,10 @@ tdq_runq_rem(struct tdq *tdq, struct thread *td)
 static void
 tdq_load_add(struct tdq *tdq, struct thread *td)
 {
-
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED);
-       THREAD_LOCK_ASSERT(td, MA_OWNED);
-
-       tdq->tdq_load++;
+//     TDQ_LOCK_ASSERT(tdq, MA_OWNED);
+//     THREAD_LOCK_ASSERT(td, MA_OWNED);
+//
+//     tdq->tdq_load++;
 //     if ((td->td_flags & TDF_NOLOAD) == 0)
 //             tdq->tdq_sysload++;
 //     KTR_COUNTER0(KTR_SCHED, "load", tdq->tdq_loadname, tdq->tdq_load);
@@ -580,13 +572,13 @@ tdq_load_add(struct tdq *tdq, struct thread *td)
 static void
 tdq_load_rem(struct tdq *tdq, struct thread *td)
 {
-
-       THREAD_LOCK_ASSERT(td, MA_OWNED);
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED);
-       KASSERT(tdq->tdq_load != 0,
-           ("tdq_load_rem: Removing with 0 load on queue %d", TDQ_ID(tdq)));
-
-       tdq->tdq_load--;
+//
+//     THREAD_LOCK_ASSERT(td, MA_OWNED);
+//     TDQ_LOCK_ASSERT(tdq, MA_OWNED);
+//     KASSERT(tdq->tdq_load != 0,
+//         ("tdq_load_rem: Removing with 0 load on queue %d", TDQ_ID(tdq)));
+//
+//     tdq->tdq_load--;
 //     if ((td->td_flags & TDF_NOLOAD) == 0)
 //             tdq->tdq_sysload--;
 //     KTR_COUNTER0(KTR_SCHED, "load", tdq->tdq_loadname, tdq->tdq_load);
@@ -1326,28 +1318,20 @@ sched_pickcpu(struct thread *td, int flags)
 static struct thread *
 tdq_choose(struct tdq *tdq)
 {
-       struct thread *td;
-
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED);
-//     td = runq_choose(&tdq->tdq_realtime);
-//     if (td != NULL)
+//     struct thread *td;
+//
+//     TDQ_LOCK_ASSERT(tdq, MA_OWNED);
+//
+//     td = runq_choose_from(&tdq->tdq_timeshare, tdq->tdq_ridx);
+//     if (td != NULL) {
 //             return (td);
-       td = runq_choose_from(&tdq->tdq_timeshare, tdq->tdq_ridx);
-       if (td != NULL) {
-//             KASSERT(td->td_priority >= PRI_MIN_BATCH,
-//                 ("tdq_choose: Invalid priority on timeshare queue %d",
-//                 td->td_priority));
-               return (td);
-       }
-       td = runq_choose(&tdq->tdq_idle);
-       if (td != NULL) {
-//             KASSERT(td->td_priority >= PRI_MIN_IDLE,
-//                 ("tdq_choose: Invalid priority on idle queue %d",
-//                 td->td_priority));
-               return (td);
-       }
-
-       return (NULL);
+//     }
+//     td = runq_choose(&tdq->tdq_idle);
+//     if (td != NULL) {
+//             return (td);
+//     }
+//
+//     return (NULL);
 }
 
 /*
@@ -1356,19 +1340,19 @@ tdq_choose(struct tdq *tdq)
 static void
 tdq_setup(struct tdq *tdq)
 {
-       if (bootverbose)
-               printf("ULE: setup cpu %d\n", TDQ_ID(tdq));
+//     if (bootverbose)
+//             printf("ULE: setup cpu %d\n", TDQ_ID(tdq));
 //     runq_init(&tdq->tdq_realtime);
-       runq_init(&tdq->tdq_timeshare);
-       runq_init(&tdq->tdq_idle);
-       snprintf(tdq->tdq_name, sizeof(tdq->tdq_name),
-           "sched lock %d", (int)TDQ_ID(tdq));
-       mtx_init(&tdq->tdq_lock, tdq->tdq_name, "sched lock",
-           MTX_SPIN | MTX_RECURSE);
-#ifdef KTR
-       snprintf(tdq->tdq_loadname, sizeof(tdq->tdq_loadname),
-           "CPU %d load", (int)TDQ_ID(tdq));
-#endif
+//     runq_init(&tdq->tdq_timeshare);
+//     runq_init(&tdq->tdq_idle);
+//     snprintf(tdq->tdq_name, sizeof(tdq->tdq_name),
+//         "sched lock %d", (int)TDQ_ID(tdq));
+//     mtx_init(&tdq->tdq_lock, tdq->tdq_name, "sched lock",
+//         MTX_SPIN | MTX_RECURSE);
+//#ifdef KTR
+//     snprintf(tdq->tdq_loadname, sizeof(tdq->tdq_loadname),
+//         "CPU %d load", (int)TDQ_ID(tdq));
+//#endif
 }
 
 #ifdef SMP
@@ -1401,11 +1385,7 @@ sched_setup(void *dummy)
        struct tdq *tdq;
 
        tdq = TDQ_SELF();
-#ifdef SMP
-       sched_setup_smp();
-#else
-       tdq_setup(tdq);
-#endif
+
        /*
         * To avoid divide-by-zero, we set realstathz a dummy value
         * in case which sched_clock() called before sched_initticks().
@@ -1417,8 +1397,7 @@ sched_setup(void *dummy)
        /* Add thread0's load since it's running. */
        TDQ_LOCK(tdq);
        thread0.td_lock = TDQ_LOCKPTR(TDQ_SELF());
-       tdq_load_add(tdq, &thread0);
-//     tdq->tdq_lowpri = thread0.td_priority;
+       tdq->tdq_load++;
        TDQ_UNLOCK(tdq);
 }
 
@@ -1649,7 +1628,6 @@ schedinit(void)
 int
 sched_rr_interval(void)
 {
-
        /* Convert sched_slice to hz */
        return (hz/(realstathz/sched_slice));
 }
@@ -1842,38 +1820,38 @@ sched_lend_user_prio(struct thread *td, u_char prio)
  * Handle migration from sched_switch().  This happens only for
  * cpu binding.
  */
-static struct mtx *
-sched_switch_migrate(struct tdq *tdq, struct thread *td, int flags)
-{
-       struct tdq *tdn;
-
-       tdn = TDQ_CPU(td->td_sched->ts_cpu);
-#ifdef SMP
-       tdq_load_rem(tdq, td);
-       /*
-        * Do the lock dance required to avoid LOR.  We grab an extra
-        * spinlock nesting to prevent preemption while we're
-        * not holding either run-queue lock.
-        */
-       spinlock_enter();
-       thread_lock_block(td);  /* This releases the lock on tdq. */
-
-       /*
-        * Acquire both run-queue locks before placing the thread on the new
-        * run-queue to avoid deadlocks created by placing a thread with a
-        * blocked lock on the run-queue of a remote processor.  The deadlock
-        * occurs when a third processor attempts to lock the two queues in
-        * question while the target processor is spinning with its own
-        * run-queue lock held while waiting for the blocked lock to clear.
-        */
-       tdq_lock_pair(tdn, tdq);
-       tdq_add(tdn, td, flags);
-       tdq_notify(tdn, td);
-       TDQ_UNLOCK(tdn);
-       spinlock_exit();
-#endif
-       return (TDQ_LOCKPTR(tdn));
-}
+//static struct mtx *
+//sched_switch_migrate(struct tdq *tdq, struct thread *td, int flags)
+//{
+//     struct tdq *tdn;
+//
+//     tdn = TDQ_CPU(td->td_sched->ts_cpu);
+//#ifdef SMP
+//     tdq_load_rem(tdq, td);
+//     /*
+//      * Do the lock dance required to avoid LOR.  We grab an extra
+//      * spinlock nesting to prevent preemption while we're
+//      * not holding either run-queue lock.
+//      */
+//     spinlock_enter();
+//     thread_lock_block(td);  /* This releases the lock on tdq. */
+//
+//     /*
+//      * Acquire both run-queue locks before placing the thread on the new
+//      * run-queue to avoid deadlocks created by placing a thread with a
+//      * blocked lock on the run-queue of a remote processor.  The deadlock
+//      * occurs when a third processor attempts to lock the two queues in
+//      * question while the target processor is spinning with its own
+//      * run-queue lock held while waiting for the blocked lock to clear.
+//      */
+//     tdq_lock_pair(tdn, tdq);
+//     tdq_add(tdn, td, flags);
+//     tdq_notify(tdn, td);
+//     TDQ_UNLOCK(tdn);
+//     spinlock_exit();
+//#endif
+//     return (TDQ_LOCKPTR(tdn));
+//}
 
 /*
  * Variadic version of thread_lock_unblock() that does not assume td_lock
@@ -1896,128 +1874,41 @@ void
 sched_switch(struct thread *td, struct thread *newtd, int flags)
 {
        struct tdq *tdq;
-       struct td_sched *ts;
        struct mtx *mtx;
        int srqflag;
-       int cpuid;
-
        struct fw_queues *q = cpu_queue(0);
 
 
-       THREAD_LOCK_ASSERT(td, MA_OWNED);
-       KASSERT(newtd == NULL, ("sched_switch: Unsupported newtd argument"));
-
-       cpuid = PCPU_GET(cpuid);
        tdq = TDQ_CPU(cpuid);
-       ts = td->td_sched;
        mtx = td->td_lock;
 
-//     tdq_print(tdq);
-//     sched_pctcpu_update(ts, 1);
-       ts->ts_rltick = ticks;
-       td->td_lastcpu = td->td_oncpu;
-       td->td_oncpu = NOCPU;
+
        if (!(flags & SW_PREEMPT))
                td->td_flags &= ~TDF_NEEDRESCHED;
        td->td_owepreempt = 0;
-       tdq->tdq_switchcnt++;
+
        /*
         * The lock pointer in an idle thread should never change.  Reset it
         * to CAN_RUN as well.
         */
        if (TD_IS_IDLETHREAD(td)) {
-               MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
                TD_SET_CAN_RUN(td);
-       } else if (TD_IS_RUNNING(td)) {
-               MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
-               srqflag = (flags & SW_PREEMPT) ?
-                   SRQ_OURSELF|SRQ_YIELDING|SRQ_PREEMPTED :
-                   SRQ_OURSELF|SRQ_YIELDING;
-#ifdef SMP
-               if (THREAD_CAN_MIGRATE(td) && !THREAD_CAN_SCHED(td, ts->ts_cpu))
-                       ts->ts_cpu = sched_pickcpu(td, 0);
-#endif
-               if (ts->ts_cpu == cpuid)
-                       tdq_runq_add(tdq, td, srqflag);
-//                     printf("");
-               else {
-                       KASSERT(THREAD_CAN_MIGRATE(td) ||
-                           (ts->ts_flags & TSF_BOUND) != 0,
-                           ("Thread %p shouldn't migrate", td));
-                       mtx = sched_switch_migrate(tdq, td, srqflag);
-               }
-       } else {
+       }
+       else if (TD_IS_RUNNING(td)) {
+               td->td_sched->fw_task.real_task = td;
+               fw_notify(FW_ADMIT, (void*)&td->td_sched->fw_task);
+       }
+       else {
                /* This thread must be going to sleep. */
                TDQ_LOCK(tdq);
                mtx = thread_lock_block(td);
-               tdq_load_rem(tdq, td);
-
+               tdq->tdq_load--;
        }
-       /*
-        * We enter here with the thread blocked and assigned to the
-        * appropriate cpu run-queue or sleep-queue and with the current
-        * thread-queue locked.
-        */
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED | MA_NOTRECURSED);
-
-//     tdq_print(0);
-//     rr_print_ready_list();
-
-       fw_notify(FW_DISPATCH, NULL);
-
-//     newtd = choosethread();
-       newtd = (struct thread*)q->fw_current->real_task;
-       TD_SET_RUNNING(newtd);
 
-       /*
-        * Call the MD code to switch contexts if necessary.
-        */
-       if (td != newtd) {
-#ifdef HWPMC_HOOKS
-               if (PMC_PROC_IS_USING_PMCS(td->td_proc))
-                       PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_OUT);
-#endif
-               SDT_PROBE2(sched, , , off_cpu, td, td->td_proc);
-               lock_profile_release_lock(&TDQ_LOCKPTR(tdq)->lock_object);
-               TDQ_LOCKPTR(tdq)->mtx_lock = (uintptr_t)newtd;
-//             sched_pctcpu_update(newtd->td_sched, 0);
+       newtd = sched_choose();
 
-#ifdef KDTRACE_HOOKS
-               /*
-                * If DTrace has set the active vtime enum to anything
-                * other than INACTIVE (0), then it should have set the
-                * function to call.
-                */
-               if (dtrace_vtime_active)
-                       (*dtrace_vtime_switch_func)(newtd);
-#endif
-
-               cpu_switch(td, newtd, mtx);
-               /*
-                * We may return from cpu_switch on a different cpu.  However,
-                * we always return with td_lock pointing to the current cpu's
-                * run queue lock.
-                */
-               cpuid = PCPU_GET(cpuid);
-               tdq = TDQ_CPU(cpuid);
-               lock_profile_obtain_lock_success(
-                   &TDQ_LOCKPTR(tdq)->lock_object, 0, 0, __FILE__, __LINE__);
-
-               SDT_PROBE0(sched, , , on_cpu);
-#ifdef HWPMC_HOOKS
-               if (PMC_PROC_IS_USING_PMCS(td->td_proc))
-                       PMC_SWITCH_CONTEXT(td, PMC_FN_CSW_IN);
-#endif
-       } else {
-               thread_unblock_switch(td, mtx);
-               SDT_PROBE0(sched, , , remain_cpu);
-       }
-       /*
-        * Assert that all went well and return.
-        */
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED|MA_NOTRECURSED);
-       MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
-       td->td_oncpu = cpuid;
+       TDQ_LOCKPTR(tdq)->mtx_lock = (uintptr_t)newtd;
+       cpu_switch(td, newtd, mtx);
 }
 
 /*
@@ -2067,22 +1958,11 @@ void
 sched_wakeup(struct thread *td)
 {
        struct td_sched *ts;
-//     int slptick;
 
        THREAD_LOCK_ASSERT(td, MA_OWNED);
        ts = td->td_sched;
        td->td_flags &= ~TDF_CANSWAP;
-//     /*
-//      * If we slept for more than a tick update our interactivity and
-//      * priority.
-//      */
-//     slptick = td->td_slptick;
-//     td->td_slptick = 0;
-//     if (slptick && slptick != ticks) {
-//             ts->ts_slptime += (ticks - slptick) << SCHED_TICK_SHIFT;
-//             sched_interact_update(td);
-//             sched_pctcpu_update(ts, 0);
-//     }
+
        /* Reset the slice value after we sleep. */
        ts->ts_slice = sched_slice;
 
@@ -2172,6 +2052,7 @@ sched_class(struct thread *td, int class)
 
        if (td->td_pri_class == class)
                return;
+
        td->td_pri_class = class;
 }
 
@@ -2373,20 +2254,14 @@ out:
 struct thread *
 sched_choose(void)
 {
-       struct thread *td;
-       struct tdq *tdq;
+       struct thread *newtd;
+       struct fw_queues *q = cpu_queue(0);
 
-       tdq = TDQ_SELF();
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED);
 
-       td = tdq_choose(tdq);
-       if (td) {
-               tdq_runq_rem(tdq, td);
-//             tdq->tdq_lowpri = td->td_priority;
-               return (td);
-       }
-//     tdq->tdq_lowpri = PRI_MAX_IDLE;
-       return (PCPU_GET(idlethread));
+       fw_notify(FW_DISPATCH, NULL);
+
+       newtd = (struct thread*)q->fw_current->real_task;
+       TD_SET_RUNNING(newtd);
 }
 
 /*
@@ -2423,18 +2298,20 @@ void
 tdq_add(struct tdq *tdq, struct thread *td, int flags)
 {
 
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED);
-       KASSERT((td->td_inhibitors == 0),
-           ("sched_add: trying to run inhibited thread"));
-       KASSERT((TD_CAN_RUN(td) || TD_IS_RUNNING(td)),
-           ("sched_add: bad thread state"));
-       KASSERT(td->td_flags & TDF_INMEM,
-           ("sched_add: thread swapped out"));
-
-//     if (td->td_priority < tdq->tdq_lowpri)
-//             tdq->tdq_lowpri = td->td_priority;
-       tdq_runq_add(tdq, td, flags);
-       tdq_load_add(tdq, td);
+//     TDQ_LOCK_ASSERT(tdq, MA_OWNED);
+//     KASSERT((td->td_inhibitors == 0),
+//         ("sched_add: trying to run inhibited thread"));
+//     KASSERT((TD_CAN_RUN(td) || TD_IS_RUNNING(td)),
+//         ("sched_add: bad thread state"));
+//     KASSERT(td->td_flags & TDF_INMEM,
+//         ("sched_add: thread swapped out"));
+//
+////   if (td->td_priority < tdq->tdq_lowpri)
+////           tdq->tdq_lowpri = td->td_priority;
+////   tdq_runq_add(tdq, td, flags);
+//
+//     td->td_sched->fw_task.real_task = td;
+//     fw_notify(FW_ADMIT, (void*)&td->td_sched->fw_task);
 }
 
 /*
@@ -2446,48 +2323,17 @@ sched_add(struct thread *td, int flags)
 {
        struct tdq *tdq;
 
-#ifdef SMP
-       int cpu;
-#endif
 
-//     KTR_STATE2(KTR_SCHED, "thread", sched_tdname(td), "runq add",
-//         "prio:%d", td->td_priority, KTR_ATTR_LINKED,
-//         sched_tdname(curthread));
-//     KTR_POINT1(KTR_SCHED, "thread", sched_tdname(curthread), "wokeup",
-//         KTR_ATTR_LINKED, sched_tdname(td));
-//     SDT_PROBE4(sched, , , enqueue, td, td->td_proc, NULL,
-//         flags & SRQ_PREEMPTED);
-       THREAD_LOCK_ASSERT(td, MA_OWNED);
-       /*
-        * Recalculate the priority before we select the target cpu or
-        * run-queue.
-        */
-//     if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
-//             sched_priority(td);
-#ifdef SMP
-       /*
-        * Pick the destination cpu and if it isn't ours transfer to the
-        * target cpu.
-        */
-       cpu = sched_pickcpu(td, flags);
-       tdq = sched_setcpu(td, cpu, flags);
-       tdq_add(tdq, td, flags);
-       if (cpu != PCPU_GET(cpuid)) {
-               tdq_notify(tdq, td);
-               return;
-       }
-#else
        tdq = TDQ_SELF();
        TDQ_LOCK(tdq);
-       /*
-        * Now that the thread is moving to the run-queue, set the lock
-        * to the scheduler's lock.
-        */
 
        thread_lock_set(td, TDQ_LOCKPTR(tdq));
-       tdq_add(tdq, td, flags);
 
-#endif
+       td->td_sched->fw_task.real_task = td;
+       fw_notify(FW_ADMIT, (void*)&td->td_sched->fw_task);
+
+       tdq->tdq_load++;
+
        if (!(flags & SRQ_YIELDING))
                sched_setpreempt(td);
 }
@@ -2502,19 +2348,9 @@ sched_rem(struct thread *td)
 {
        struct tdq *tdq;
 
-       KTR_STATE1(KTR_SCHED, "thread", sched_tdname(td), "runq rem",
-           "prio:%d", td->td_priority);
-       SDT_PROBE3(sched, , , dequeue, td, td->td_proc, NULL);
        tdq = TDQ_CPU(td->td_sched->ts_cpu);
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED);
-       MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
-       KASSERT(TD_ON_RUNQ(td),
-           ("sched_rem: thread not on run queue"));
-       tdq_runq_rem(tdq, td);
-       tdq_load_rem(tdq, td);
+       tdq->tdq_load--;
        TD_SET_CAN_RUN(td);
-//     if (td->td_priority == tdq->tdq_lowpri)
-//             tdq_setlowpri(tdq, NULL);
 }
 
 /*
@@ -2523,15 +2359,15 @@ sched_rem(struct thread *td)
 fixpt_t
 sched_pctcpu(struct thread *td)
 {
-       fixpt_t pctcpu;
-       struct td_sched *ts;
-
-       pctcpu = 0;
-       ts = td->td_sched;
-       if (ts == NULL)
-               return (0);
-
-       THREAD_LOCK_ASSERT(td, MA_OWNED);
+//     fixpt_t pctcpu;
+//     struct td_sched *ts;
+//
+//     pctcpu = 0;
+//     ts = td->td_sched;
+//     if (ts == NULL)
+//             return (0);
+//
+//     THREAD_LOCK_ASSERT(td, MA_OWNED);
 //     sched_pctcpu_update(ts, TD_IS_RUNNING(td));
 //     if (ts->ts_ticks) {
 //             int rtick;
@@ -2540,8 +2376,8 @@ sched_pctcpu(struct thread *td)
 //             rtick = min(SCHED_TICK_HZ(ts) / SCHED_TICK_SECS, hz);
 //             pctcpu = (FSCALE * ((FSCALE * rtick)/hz)) >> FSHIFT;
 //     }
-
-       return (pctcpu);
+//
+//     return (pctcpu);
 }
 
 /*
@@ -2642,17 +2478,17 @@ sched_relinquish(struct thread *td)
 int
 sched_load(void)
 {
-#ifdef SMP
-       int total;
-       int i;
-
-       total = 0;
-       CPU_FOREACH(i)
-               total += TDQ_CPU(i)->tdq_sysload;
-       return (total);
-#else
-       return (TDQ_SELF()->tdq_sysload);
-#endif
+//#ifdef SMP
+//     int total;
+//     int i;
+//
+//     total = 0;
+//     CPU_FOREACH(i)
+//             total += TDQ_CPU(i)->tdq_sysload;
+//     return (total);
+//#else
+//     return (TDQ_SELF()->tdq_sysload);
+//#endif
 }
 
 int
@@ -2696,7 +2532,6 @@ sched_idletd(void *dummy)
                }
 #endif
                switchcnt = tdq->tdq_switchcnt + tdq->tdq_oldswitchcnt;
-//             tdq_print(TDQ_ID(tdq));
                /*
                 * If we're switching very frequently, spin while checking
                 * for load rather than entering a low power state that
@@ -2736,9 +2571,9 @@ sched_throw(struct thread *td)
 {
        struct thread *newtd;
        struct tdq *tdq;
-       struct fw_queues *q = cpu_queue(0);
 
        tdq = TDQ_SELF();
+
        if (td == NULL) {
                /* Correct spinlock nesting and acquire the correct lock. */
                TDQ_LOCK(tdq);
@@ -2747,14 +2582,12 @@ sched_throw(struct thread *td)
                PCPU_SET(switchticks, ticks);
        } else {
                MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
-               tdq_load_rem(tdq, td);
+               tdq->tdq_load--;
                lock_profile_release_lock(&TDQ_LOCKPTR(tdq)->lock_object);
        }
-       KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
-       fw_notify(FW_DISPATCH, NULL);
-//     newtd = choosethread();
-       newtd = (struct thread*)q->fw_current->real_task;
-       TD_SET_RUNNING(newtd);
+
+       newtd = sched_choose();
+
        TDQ_LOCKPTR(tdq)->mtx_lock = (uintptr_t)newtd;
        cpu_throw(td, newtd);           /* doesn't return */
 }
@@ -2766,24 +2599,12 @@ sched_throw(struct thread *td)
 void
 sched_fork_exit(struct thread *td)
 {
-       struct td_sched *ts;
        struct tdq *tdq;
-       int cpuid;
 
-       /*
-        * Finish setting up thread glue so that it begins execution in a
-        * non-nested critical section with the scheduler lock held.
-        */
-       cpuid = PCPU_GET(cpuid);
        tdq = TDQ_CPU(cpuid);
-       ts = td->td_sched;
+
        if (TD_IS_IDLETHREAD(td))
                td->td_lock = TDQ_LOCKPTR(tdq);
-       MPASS(td->td_lock == TDQ_LOCKPTR(tdq));
-       td->td_oncpu = cpuid;
-       TDQ_LOCK_ASSERT(tdq, MA_OWNED | MA_NOTRECURSED);
-       lock_profile_obtain_lock_success(
-           &TDQ_LOCKPTR(tdq)->lock_object, 0, 0, __FILE__, __LINE__);
 }
 
 /*
@@ -2935,3 +2756,4 @@ SYSCTL_PROC(_kern_sched, OID_AUTO, topology_spec, CTLTYPE_STRING |
 /* ps compat.  All cpu percentages from ULE are weighted. */
 static int ccpu = 0;
 SYSCTL_INT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, "");
+