sched.h

/*
 * Copyright (C) 2008 Philippe Gerum <[email protected]>.
 *
 * Xenomai is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published
 * by the Free Software Foundation; either version 2 of the License,
 * or (at your option) any later version.
 *
 * Xenomai is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Xenomai; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
 * 02111-1307, USA.
 */
#ifndef _COBALT_KERNEL_SCHED_H
#define _COBALT_KERNEL_SCHED_H
 
#include <linux/percpu.h>
#include <cobalt/kernel/lock.h>
#include <cobalt/kernel/thread.h>
#include <cobalt/kernel/schedqueue.h>
#include <cobalt/kernel/sched-tp.h>
#include <cobalt/kernel/sched-weak.h>
#include <cobalt/kernel/sched-sporadic.h>
#include <cobalt/kernel/sched-quota.h>
#include <cobalt/kernel/vfile.h>
#include <cobalt/kernel/assert.h>
#include <asm/xenomai/machine.h>
 
/* Sched status flags */
#define XNRESCHED       0x10000000      /* Needs rescheduling */
#define XNINSW          0x20000000      /* In context switch */
#define XNINTCK         0x40000000      /* In master tick handler context */
 
/* Sched local flags */
#define XNIDLE          0x00010000      /* Idle (no outstanding timer) */
#define XNHTICK         0x00008000      /* Host tick pending  */
#define XNINIRQ         0x00004000      /* In IRQ handling context */
#define XNHDEFER        0x00002000      /* Host tick deferred */
 
struct xnsched_rt {
        xnsched_queue_t runnable;       
};
 
struct xnsched {
        unsigned long status;
        unsigned long lflags;
        struct xnthread *curr;
#ifdef CONFIG_SMP
        int cpu;
        cpumask_t resched;
#endif
        struct xnsched_rt rt;
#ifdef CONFIG_XENO_OPT_SCHED_WEAK
        struct xnsched_weak weak;
#endif
#ifdef CONFIG_XENO_OPT_SCHED_TP
        struct xnsched_tp tp;
#endif
#ifdef CONFIG_XENO_OPT_SCHED_SPORADIC
        struct xnsched_sporadic pss;
#endif
#ifdef CONFIG_XENO_OPT_SCHED_QUOTA
        struct xnsched_quota quota;
#endif
        volatile unsigned inesting;
        struct xntimer htimer;
        struct xntimer rrbtimer;
        struct xnthread rootcb;
#ifdef CONFIG_XENO_ARCH_UNLOCKED_SWITCH
        struct xnthread *last;
#endif
#ifdef CONFIG_XENO_ARCH_FPU
        struct xnthread *fpuholder;
#endif
#ifdef CONFIG_XENO_OPT_WATCHDOG
        struct xntimer wdtimer;
        int wdcount;
#endif
#ifdef CONFIG_XENO_OPT_STATS
        xnticks_t last_account_switch;
        xnstat_exectime_t *current_account;
#endif
};
 
DECLARE_PER_CPU(struct xnsched, nksched);
 
extern cpumask_t cobalt_cpu_affinity;
 
extern struct list_head nkthreadq;
 
extern int cobalt_nrthreads;
 
#ifdef CONFIG_XENO_OPT_VFILE
extern struct xnvfile_rev_tag nkthreadlist_tag;
#endif
 
union xnsched_policy_param;
 
struct xnsched_class {
        void (*sched_init)(struct xnsched *sched);
        void (*sched_enqueue)(struct xnthread *thread);
        void (*sched_dequeue)(struct xnthread *thread);
        void (*sched_requeue)(struct xnthread *thread);
        struct xnthread *(*sched_pick)(struct xnsched *sched);
        void (*sched_tick)(struct xnsched *sched);
        void (*sched_rotate)(struct xnsched *sched,
                             const union xnsched_policy_param *p);
        void (*sched_migrate)(struct xnthread *thread,
                              struct xnsched *sched);
        int (*sched_chkparam)(struct xnthread *thread,
                              const union xnsched_policy_param *p);
        void (*sched_setparam)(struct xnthread *thread,
                               const union xnsched_policy_param *p);
        void (*sched_getparam)(struct xnthread *thread,
                               union xnsched_policy_param *p);
        void (*sched_trackprio)(struct xnthread *thread,
                                const union xnsched_policy_param *p);
        int (*sched_declare)(struct xnthread *thread,
                             const union xnsched_policy_param *p);
        void (*sched_forget)(struct xnthread *thread);
        void (*sched_kick)(struct xnthread *thread);
#ifdef CONFIG_XENO_OPT_VFILE
        int (*sched_init_vfile)(struct xnsched_class *schedclass,
                                struct xnvfile_directory *vfroot);
        void (*sched_cleanup_vfile)(struct xnsched_class *schedclass);
#endif
        int nthreads;
        struct xnsched_class *next;
        int weight;
        int policy;
        const char *name;
};
 
#define XNSCHED_CLASS_WEIGHT(n)         (n * XNSCHED_CLASS_WEIGHT_FACTOR)
 
/* Placeholder for current thread priority */
#define XNSCHED_RUNPRIO   0x80000000
 
#define xnsched_for_each_thread(__thread)       \
        list_for_each_entry(__thread, &nkthreadq, glink)
 
#ifdef CONFIG_SMP
static inline int xnsched_cpu(struct xnsched *sched)
{
        return sched->cpu;
}
#else /* !CONFIG_SMP */
static inline int xnsched_cpu(struct xnsched *sched)
{
        return 0;
}
#endif /* CONFIG_SMP */
 
static inline struct xnsched *xnsched_struct(int cpu)
{
        return &per_cpu(nksched, cpu);
}
 
static inline struct xnsched *xnsched_current(void)
{
        /* IRQs off */
        return raw_cpu_ptr(&nksched);
}
 
static inline struct xnthread *xnsched_current_thread(void)
{
        return xnsched_current()->curr;
}
 
/* Test resched flag of given sched. */
static inline int xnsched_resched_p(struct xnsched *sched)
{
        return sched->status & XNRESCHED;
}
 
/* Set self resched flag for the current scheduler. */
static inline void xnsched_set_self_resched(struct xnsched *sched)
{
        sched->status |= XNRESCHED;
}
 
#define xnsched_realtime_domain  cobalt_pipeline.domain
 
/* Set resched flag for the given scheduler. */
#ifdef CONFIG_SMP
 
static inline void xnsched_set_resched(struct xnsched *sched)
{
        struct xnsched *current_sched = xnsched_current();
 
        if (current_sched == sched)
                current_sched->status |= XNRESCHED;
        else if (!xnsched_resched_p(sched)) {
                cpumask_set_cpu(xnsched_cpu(sched), &current_sched->resched);
                sched->status |= XNRESCHED;
                current_sched->status |= XNRESCHED;
        }
}
 
#define xnsched_realtime_cpus    cobalt_pipeline.supported_cpus
 
static inline int xnsched_supported_cpu(int cpu)
{
        return cpumask_test_cpu(cpu, &xnsched_realtime_cpus);
}
 
static inline int xnsched_threading_cpu(int cpu)
{
        return cpumask_test_cpu(cpu, &cobalt_cpu_affinity);
}
 
#else /* !CONFIG_SMP */
 
static inline void xnsched_set_resched(struct xnsched *sched)
{
        xnsched_set_self_resched(sched);
}
 
#define xnsched_realtime_cpus CPU_MASK_ALL
 
static inline int xnsched_supported_cpu(int cpu)
{
        return 1;
}
 
static inline int xnsched_threading_cpu(int cpu)
{
        return 1;
}
 
#endif /* !CONFIG_SMP */
 
#define for_each_realtime_cpu(cpu)              \
        for_each_online_cpu(cpu)                \
                if (xnsched_supported_cpu(cpu)) \
 
int ___xnsched_run(struct xnsched *sched);
 
void __xnsched_run_handler(void);
 
static inline int __xnsched_run(struct xnsched *sched)
{
        /*
         * Reschedule if XNSCHED is pending, but never over an IRQ
         * handler or in the middle of unlocked context switch.
         */
        if (((sched->status|sched->lflags) &
             (XNINIRQ|XNINSW|XNRESCHED)) != XNRESCHED)
                return 0;
 
        return ___xnsched_run(sched);
}
 
static inline int xnsched_run(void)
{
        struct xnsched *sched = xnsched_current();
        /*
         * sched->curr is shared locklessly with ___xnsched_run().
         * READ_ONCE() makes sure the compiler never uses load tearing
         * for reading this pointer piecemeal, so that multiple stores
         * occurring concurrently on remote CPUs never yield a
         * spurious merged value on the local one.
         */
        struct xnthread *curr = READ_ONCE(sched->curr);
 
        /*
         * If running over the root thread, hard irqs must be off
         * (asserted out of line in ___xnsched_run()).
         */
        return curr->lock_count > 0 ? 0 : __xnsched_run(sched);
}
 
void xnsched_lock(void);
 
void xnsched_unlock(void);
 
static inline int xnsched_interrupt_p(void)
{
        return xnsched_current()->lflags & XNINIRQ;
}
 
static inline int xnsched_root_p(void)
{
        return xnthread_test_state(xnsched_current_thread(), XNROOT);
}
 
static inline int xnsched_unblockable_p(void)
{
        return xnsched_interrupt_p() || xnsched_root_p();
}
 
static inline int xnsched_primary_p(void)
{
        return !xnsched_unblockable_p();
}
 
#ifdef CONFIG_XENO_ARCH_UNLOCKED_SWITCH
 
struct xnsched *xnsched_finish_unlocked_switch(struct xnsched *sched);
 
#define xnsched_resched_after_unlocked_switch() xnsched_run()
 
static inline
int xnsched_maybe_resched_after_unlocked_switch(struct xnsched *sched)
{
        return sched->status & XNRESCHED;
}
 
#else /* !CONFIG_XENO_ARCH_UNLOCKED_SWITCH */
 
static inline struct xnsched *
xnsched_finish_unlocked_switch(struct xnsched *sched)
{
        XENO_BUG_ON(COBALT, !hard_irqs_disabled());
        return xnsched_current();
}
 
static inline void xnsched_resched_after_unlocked_switch(void) { }
 
static inline int
xnsched_maybe_resched_after_unlocked_switch(struct xnsched *sched)
{
        return 0;
}
 
#endif /* !CONFIG_XENO_ARCH_UNLOCKED_SWITCH */
 
#ifdef CONFIG_XENO_OPT_WATCHDOG
static inline void xnsched_reset_watchdog(struct xnsched *sched)
{
        sched->wdcount = 0;
}
#else /* !CONFIG_XENO_OPT_WATCHDOG */
static inline void xnsched_reset_watchdog(struct xnsched *sched)
{
}
#endif /* CONFIG_XENO_OPT_WATCHDOG */
 
#include <cobalt/kernel/sched-idle.h>
#include <cobalt/kernel/sched-rt.h>
 
int xnsched_init_proc(void);
 
void xnsched_cleanup_proc(void);
 
void xnsched_register_classes(void);
 
void xnsched_init(struct xnsched *sched, int cpu);
 
void xnsched_destroy(struct xnsched *sched);
 
struct xnthread *xnsched_pick_next(struct xnsched *sched);
 
void xnsched_putback(struct xnthread *thread);
 
int xnsched_set_policy(struct xnthread *thread,
                       struct xnsched_class *sched_class,
                       const union xnsched_policy_param *p);
 
void xnsched_track_policy(struct xnthread *thread,
                          struct xnthread *target);
 
void xnsched_migrate(struct xnthread *thread,
                     struct xnsched *sched);
 
void xnsched_migrate_passive(struct xnthread *thread,
                             struct xnsched *sched);
 
static inline void xnsched_rotate(struct xnsched *sched,
                                  struct xnsched_class *sched_class,
                                  const union xnsched_policy_param *sched_param)
{
        sched_class->sched_rotate(sched, sched_param);
}
 
static inline int xnsched_init_thread(struct xnthread *thread)
{
        int ret = 0;
 
        xnsched_idle_init_thread(thread);
        xnsched_rt_init_thread(thread);
 
#ifdef CONFIG_XENO_OPT_SCHED_TP
        ret = xnsched_tp_init_thread(thread);
        if (ret)
                return ret;
#endif /* CONFIG_XENO_OPT_SCHED_TP */
#ifdef CONFIG_XENO_OPT_SCHED_SPORADIC
        ret = xnsched_sporadic_init_thread(thread);
        if (ret)
                return ret;
#endif /* CONFIG_XENO_OPT_SCHED_SPORADIC */
#ifdef CONFIG_XENO_OPT_SCHED_QUOTA
        ret = xnsched_quota_init_thread(thread);
        if (ret)
                return ret;
#endif /* CONFIG_XENO_OPT_SCHED_QUOTA */
 
        return ret;
}
 
static inline int xnsched_root_priority(struct xnsched *sched)
{
        return sched->rootcb.cprio;
}
 
static inline struct xnsched_class *xnsched_root_class(struct xnsched *sched)
{
        return sched->rootcb.sched_class;
}
 
static inline void xnsched_tick(struct xnsched *sched)
{
        struct xnthread *curr = sched->curr;
        struct xnsched_class *sched_class = curr->sched_class;
        /*
         * A thread that undergoes round-robin scheduling only
         * consumes its time slice when it runs within its own
         * scheduling class, which excludes temporary PIP boosts, and
         * does not hold the scheduler lock.
         */
        if (sched_class == curr->base_class &&
            sched_class->sched_tick &&
            xnthread_test_state(curr, XNTHREAD_BLOCK_BITS|XNRRB) == XNRRB &&
                curr->lock_count == 0)
                sched_class->sched_tick(sched);
}
 
static inline int xnsched_chkparam(struct xnsched_class *sched_class,
                                   struct xnthread *thread,
                                   const union xnsched_policy_param *p)
{
        if (sched_class->sched_chkparam)
                return sched_class->sched_chkparam(thread, p);
 
        return 0;
}
 
static inline int xnsched_declare(struct xnsched_class *sched_class,
                                  struct xnthread *thread,
                                  const union xnsched_policy_param *p)
{
        int ret;
 
        if (sched_class->sched_declare) {
                ret = sched_class->sched_declare(thread, p);
                if (ret)
                        return ret;
        }
        if (sched_class != thread->base_class)
                sched_class->nthreads++;
 
        return 0;
}
 
#ifdef CONFIG_XENO_OPT_SCHED_CLASSES
 
static inline void xnsched_enqueue(struct xnthread *thread)
{
        struct xnsched_class *sched_class = thread->sched_class;
 
        if (sched_class != &xnsched_class_idle)
                sched_class->sched_enqueue(thread);
}
 
static inline void xnsched_dequeue(struct xnthread *thread)
{
        struct xnsched_class *sched_class = thread->sched_class;
 
        if (sched_class != &xnsched_class_idle)
                sched_class->sched_dequeue(thread);
}
 
static inline void xnsched_requeue(struct xnthread *thread)
{
        struct xnsched_class *sched_class = thread->sched_class;
 
        if (sched_class != &xnsched_class_idle)
                sched_class->sched_requeue(thread);
}
 
static inline void xnsched_setparam(struct xnthread *thread,
                                    const union xnsched_policy_param *p)
{
        thread->sched_class->sched_setparam(thread, p);
        thread->wprio = thread->cprio + thread->sched_class->weight;
}
 
static inline void xnsched_getparam(struct xnthread *thread,
                                    union xnsched_policy_param *p)
{
        thread->sched_class->sched_getparam(thread, p);
}
 
static inline void xnsched_trackprio(struct xnthread *thread,
                                     const union xnsched_policy_param *p)
{
        thread->sched_class->sched_trackprio(thread, p);
        thread->wprio = thread->cprio + thread->sched_class->weight;
}
 
static inline void xnsched_forget(struct xnthread *thread)
{
        struct xnsched_class *sched_class = thread->base_class;
 
        --sched_class->nthreads;
 
        if (sched_class->sched_forget)
                sched_class->sched_forget(thread);
}
 
static inline void xnsched_kick(struct xnthread *thread)
{
        struct xnsched_class *sched_class = thread->base_class;
 
        xnthread_set_info(thread, XNKICKED);
 
        if (sched_class->sched_kick)
                sched_class->sched_kick(thread);
 
        xnsched_set_resched(thread->sched);
}
 
#else /* !CONFIG_XENO_OPT_SCHED_CLASSES */
 
/*
 * If only the RT and IDLE scheduling classes are compiled in, we can
 * fully inline common helpers for dealing with those.
 */
 
static inline void xnsched_enqueue(struct xnthread *thread)
{
        struct xnsched_class *sched_class = thread->sched_class;
 
        if (sched_class != &xnsched_class_idle)
                __xnsched_rt_enqueue(thread);
}
 
static inline void xnsched_dequeue(struct xnthread *thread)
{
        struct xnsched_class *sched_class = thread->sched_class;
 
        if (sched_class != &xnsched_class_idle)
                __xnsched_rt_dequeue(thread);
}
 
static inline void xnsched_requeue(struct xnthread *thread)
{
        struct xnsched_class *sched_class = thread->sched_class;
 
        if (sched_class != &xnsched_class_idle)
                __xnsched_rt_requeue(thread);
}
 
static inline void xnsched_setparam(struct xnthread *thread,
                                    const union xnsched_policy_param *p)
{
        struct xnsched_class *sched_class = thread->sched_class;
 
        if (sched_class != &xnsched_class_idle)
                __xnsched_rt_setparam(thread, p);
        else
                __xnsched_idle_setparam(thread, p);
 
        thread->wprio = thread->cprio + sched_class->weight;
}
 
static inline void xnsched_getparam(struct xnthread *thread,
                                    union xnsched_policy_param *p)
{
        struct xnsched_class *sched_class = thread->sched_class;
 
        if (sched_class != &xnsched_class_idle)
                __xnsched_rt_getparam(thread, p);
        else
                __xnsched_idle_getparam(thread, p);
}
 
static inline void xnsched_trackprio(struct xnthread *thread,
                                     const union xnsched_policy_param *p)
{
        struct xnsched_class *sched_class = thread->sched_class;
 
        if (sched_class != &xnsched_class_idle)
                __xnsched_rt_trackprio(thread, p);
        else
                __xnsched_idle_trackprio(thread, p);
 
        thread->wprio = thread->cprio + sched_class->weight;
}
 
static inline void xnsched_forget(struct xnthread *thread)
{
        --thread->base_class->nthreads;
        __xnsched_rt_forget(thread);
}
 
static inline void xnsched_kick(struct xnthread *thread)
{
        xnthread_set_info(thread, XNKICKED);
        xnsched_set_resched(thread->sched);
}
 
#endif /* !CONFIG_XENO_OPT_SCHED_CLASSES */
 
#endif /* !_COBALT_KERNEL_SCHED_H */