/*
 * Return the number of cycles by which our itc differs from the itc on the master
 * (time-keeper) CPU.  A positive number indicates our itc is ahead of the master,
 * negative that it is behind.
 */
static inline long
get_delta (long *rt, long *master)
{
	unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
	unsigned long tcenter, t0, t1, tm;
	long i;

	for (i = 0; i < NUM_ITERS; ++i) {
		t0 = ia64_get_itc();
		go[MASTER] = 1;
		while (!(tm = go[SLAVE]))
			cpu_relax();
		go[SLAVE] = 0;
		t1 = ia64_get_itc();

		if (t1 - t0 < best_t1 - best_t0)
			best_t0 = t0, best_t1 = t1, best_tm = tm;
	}

	*rt = best_t1 - best_t0;
	*master = best_tm - best_t0;

	/* average best_t0 and best_t1 without overflow: */
	tcenter = (best_t0/2 + best_t1/2);
	if (best_t0 % 2 + best_t1 % 2 == 2)
		++tcenter;
	return tcenter - best_tm;
}

static irqreturn_t
timer_interrupt (int irq, void *dev_id, struct pt_regs *regs)
{
    unsigned long new_itm;

    if (unlikely(cpu_is_offline(smp_processor_id()))) {
        return IRQ_HANDLED;
    }

    platform_timer_interrupt(irq, dev_id, regs);

    new_itm = local_cpu_data->itm_next;

    if (!time_after(ia64_get_itc(), new_itm))
        printk(KERN_ERR "Oops: timer tick before it's due (itc=%lx,itm=%lx)\n",
               ia64_get_itc(), new_itm);

    profile_tick(CPU_PROFILING, regs);

    while (1) {
        update_process_times(user_mode(regs));

        new_itm += local_cpu_data->itm_delta;

        if (smp_processor_id() == TIME_KEEPER_ID) {
            /*
             * Here we are in the timer irq handler. We have irqs locally
             * disabled, but we don't know if the timer_bh is running on
             * another CPU. We need to avoid to SMP race by acquiring the
             * xtime_lock.
             */
            write_seqlock(&xtime_lock);
            do_timer(regs);
            local_cpu_data->itm_next = new_itm;
            write_sequnlock(&xtime_lock);
        } else
            local_cpu_data->itm_next = new_itm;

        if (time_after(new_itm, ia64_get_itc()))
            break;
    }

    do {
        /*
         * If we're too close to the next clock tick for
         * comfort, we increase the safety margin by
         * intentionally dropping the next tick(s).  We do NOT
         * update itm.next because that would force us to call
         * do_timer() which in turn would let our clock run
         * too fast (with the potentially devastating effect
         * of losing monotony of time).
         */
        while (!time_after(new_itm, ia64_get_itc() + local_cpu_data->itm_delta/2))
            new_itm += local_cpu_data->itm_delta;
        ia64_set_itm(new_itm);
        /* double check, in case we got hit by a (slow) PMI: */
    } while (time_after_eq(ia64_get_itc(), new_itm));
    return IRQ_HANDLED;
}

static void
timer_interrupt(int irq, void *dev_id, struct pt_regs *regs)
{
	unsigned long new_itm;

	new_itm = local_cpu_data->itm_next;

	if (!time_after(ia64_get_itc(), new_itm))
		printk(KERN_ERR "Oops: timer tick before it's due (itc=%lx,itm=%lx)\n",
		       ia64_get_itc(), new_itm);

	while (1) {
		/*
		 * Do kernel PC profiling here.  We multiply the instruction number by
		 * four so that we can use a prof_shift of 2 to get instruction-level
		 * instead of just bundle-level accuracy.
		 */
		if (!user_mode(regs))
			do_profile(regs->cr_iip + 4*ia64_psr(regs)->ri);

#ifdef CONFIG_SMP
		smp_do_timer(regs);
#endif
		new_itm += local_cpu_data->itm_delta;

		if (smp_processor_id() == 0) {
			/*
			 * Here we are in the timer irq handler. We have irqs locally
			 * disabled, but we don't know if the timer_bh is running on
			 * another CPU. We need to avoid to SMP race by acquiring the
			 * xtime_lock.
			 */
			write_lock(&xtime_lock);
			do_timer(regs);
			local_cpu_data->itm_next = new_itm;
			write_unlock(&xtime_lock);
		} else
			local_cpu_data->itm_next = new_itm;

		if (time_after(new_itm, ia64_get_itc()))
			break;
	}

	do {
	    /*
	     * If we're too close to the next clock tick for comfort, we increase the
	     * saftey margin by intentionally dropping the next tick(s).  We do NOT update
	     * itm.next because that would force us to call do_timer() which in turn would
	     * let our clock run too fast (with the potentially devastating effect of
	     * losing monotony of time).
	     */
	    while (!time_after(new_itm, ia64_get_itc() + local_cpu_data->itm_delta/2))
	      new_itm += local_cpu_data->itm_delta;
	    ia64_set_itm(new_itm);
	    /* double check, in case we got hit by a (slow) PMI: */
	} while (time_after_eq(ia64_get_itc(), new_itm));
}

/*
 * Generic udelay assumes that if preemption is allowed and the thread
 * migrates to another CPU, that the ITC values are synchronized across
 * all CPUs.
 */
static void
ia64_itc_udelay (unsigned long usecs)
{
    unsigned long start = ia64_get_itc();
    unsigned long end = start + usecs*local_cpu_data->cyc_per_usec;

    while (time_before(ia64_get_itc(), end))
        cpu_relax();
}

static irqreturn_t
timer_interrupt (int irq, void *dev_id)
{
	unsigned long new_itm;

	if (cpu_is_offline(smp_processor_id())) {
		return IRQ_HANDLED;
	}

	platform_timer_interrupt(irq, dev_id);

	new_itm = local_cpu_data->itm_next;

	if (!time_after(ia64_get_itc(), new_itm))
		printk(KERN_ERR "Oops: timer tick before it's due (itc=%lx,itm=%lx)\n",
		       ia64_get_itc(), new_itm);

	profile_tick(CPU_PROFILING);

	while (1) {
		update_process_times(user_mode(get_irq_regs()));

		new_itm += local_cpu_data->itm_delta;

		if (smp_processor_id() == time_keeper_id)
			xtime_update(1);

		local_cpu_data->itm_next = new_itm;

		if (time_after(new_itm, ia64_get_itc()))
			break;

		/*
		 * Allow IPIs to interrupt the timer loop.
		 */
		local_irq_enable();
		local_irq_disable();
	}

	do {
		/*
		 * If we're too close to the next clock tick for
		 * comfort, we increase the safety margin by
		 * intentionally dropping the next tick(s).  We do NOT
		 * update itm.next because that would force us to call
		 * xtime_update() which in turn would let our clock run
		 * too fast (with the potentially devastating effect
		 * of losing monotony of time).
		 */
		while (!time_after(new_itm, ia64_get_itc() + local_cpu_data->itm_delta/2))
			new_itm += local_cpu_data->itm_delta;
		ia64_set_itm(new_itm);
		/* double check, in case we got hit by a (slow) PMI: */
	} while (time_after_eq(ia64_get_itc(), new_itm));
	return IRQ_HANDLED;
}

static void
sn2_ipi_flush_all_tlb(struct mm_struct *mm)
{
	unsigned long itc;

	itc = ia64_get_itc();
	smp_flush_tlb_cpumask(*mm_cpumask(mm));
	itc = ia64_get_itc() - itc;
	__this_cpu_add(ptcstats.shub_ipi_flushes_itc_clocks, itc);
	__this_cpu_inc(ptcstats.shub_ipi_flushes);
}

static void
sn2_ipi_flush_all_tlb(struct mm_struct *mm)
{
	unsigned long itc;

	itc = ia64_get_itc();
	smp_flush_tlb_cpumask(*mm_cpumask(mm));
	itc = ia64_get_itc() - itc;
	__get_cpu_var(ptcstats).shub_ipi_flushes_itc_clocks += itc;
	__get_cpu_var(ptcstats).shub_ipi_flushes++;
}

/*
 * Transfer the bte_test_buffer from our node to the specified
 * destination and print out timing results.
 */
static void
brt_time_xfer(int dest_node, int iterations, int xfer_lines)
{
	int iteration;
	char *src, *dst;
	u64 xfer_len, src_phys, dst_phys;
	u64 itc_before, itc_after, mem_intvl, bte_intvl;


	xfer_len = xfer_lines * L1_CACHE_BYTES;

	src = nodepda->bte_if[0].bte_test_buf;
	src_phys = __pa(src);
	dst = NODEPDA(dest_node)->bte_if[1].bte_test_buf;
	dst_phys = __pa(dst);
	mem_intvl = 0;

	for (iteration = 0; iteration < iterations; iteration++) {
		if (tm_memcpy) {
			itc_before = ia64_get_itc();
			memcpy(dst, src, xfer_len);
			itc_after = ia64_get_itc();
			mem_intvl = itc_after - itc_before;
		}

		itc_before = ia64_get_itc();
		bte_copy(src_phys, dst_phys, xfer_len, BTE_NOTIFY, NULL);
		itc_after = ia64_get_itc();
		bte_intvl = itc_after - itc_before;

		if (tm_memcpy) {
			printk("%3d,%3d,%3d,%5d,%4ld,%7ld,%3ld,"
			       "%7ld,%7ld,%7ld\n",
			       smp_processor_id(), NASID_GET(src),
			       NASID_GET(dst), xfer_lines,
			       NSEC(bte_setup_time),
			       NSEC(bte_transfer_time),
			       NSEC(bte_tear_down_time),
			       NSEC(bte_execute_time), NSEC(bte_intvl),
			       NSEC(mem_intvl));
		} else {
			printk("%3d,%3d,%3d,%5d,%4ld,%7ld,%3ld,"
			       "%7ld,%7ld\n",
			       smp_processor_id(), NASID_GET(src),
			       NASID_GET(dst), xfer_lines,
			       NSEC(bte_setup_time),
			       NSEC(bte_transfer_time),
			       NSEC(bte_tear_down_time),
			       NSEC(bte_execute_time), NSEC(bte_intvl));
		}
	}

}

/*
 * Synchronize ar.itc of the current (slave) CPU with the ar.itc of the MASTER CPU
 * (normally the time-keeper CPU).  We use a closed loop to eliminate the possibility of
 * unaccounted-for errors (such as getting a machine check in the middle of a calibration
 * step).  The basic idea is for the slave to ask the master what itc value it has and to
 * read its own itc before and after the master responds.  Each iteration gives us three
 * timestamps:
 *
 *	slave		master
 *
 *	t0 ---\
 *             ---\
 *		   --->
 *			tm
 *		   /---
 *	       /---
 *	t1 <---
 *
 *
 * The goal is to adjust the slave's ar.itc such that tm falls exactly half-way between t0
 * and t1.  If we achieve this, the clocks are synchronized provided the interconnect
 * between the slave and the master is symmetric.  Even if the interconnect were
 * asymmetric, we would still know that the synchronization error is smaller than the
 * roundtrip latency (t0 - t1).
 *
 * When the interconnect is quiet and symmetric, this lets us synchronize the itc to
 * within one or two cycles.  However, we can only *guarantee* that the synchronization is
 * accurate to within a round-trip time, which is typically in the range of several
 * hundred cycles (e.g., ~500 cycles).  In practice, this means that the itc's are usually
 * almost perfectly synchronized, but we shouldn't assume that the accuracy is much better
 * than half a micro second or so.
 */
void
ia64_sync_itc (unsigned int master)
{
	long i, delta, adj, adjust_latency = 0, done = 0;
	unsigned long flags, rt, master_time_stamp, bound;
#if DEBUG_ITC_SYNC
	struct {
		long rt;	/* roundtrip time */
		long master;	/* master's timestamp */
		long diff;	/* difference between midpoint and master's timestamp */
		long lat;	/* estimate of itc adjustment latency */
	} t[NUM_ROUNDS];
#endif

	go[MASTER] = 1;

	if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) {
		printk("sync_itc: failed to get attention of CPU %u!\n", master);
		return;
	}

	while (go[MASTER]);	/* wait for master to be ready */

	spin_lock_irqsave(&itc_sync_lock, flags);
	{
		for (i = 0; i < NUM_ROUNDS; ++i) {
			delta = get_delta(&rt, &master_time_stamp);
			if (delta == 0) {
				done = 1;	/* let's lock on to this... */
				bound = rt;
			}

			if (!done) {
				if (i > 0) {
					adjust_latency += -delta;
					adj = -delta + adjust_latency/4;
				} else
					adj = -delta;

				ia64_set_itc(ia64_get_itc() + adj);
			}
#if DEBUG_ITC_SYNC
			t[i].rt = rt;
			t[i].master = master_time_stamp;
			t[i].diff = delta;
			t[i].lat = adjust_latency/4;
#endif
		}
	}
	spin_unlock_irqrestore(&itc_sync_lock, flags);

#if DEBUG_ITC_SYNC
	for (i = 0; i < NUM_ROUNDS; ++i)
		printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
		       t[i].rt, t[i].master, t[i].diff, t[i].lat);
#endif

	printk("CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, maxerr %lu cycles)\n",
	       smp_processor_id(), master, delta, rt);
}

static int xen_do_steal_accounting(unsigned long *new_itm)
{
	unsigned long delta_itm;
	delta_itm = consider_steal_time(*new_itm);
	*new_itm += delta_itm;
	if (time_after(*new_itm, ia64_get_itc()) && delta_itm)
		return 1;

	return 0;
}

void
pcpu_initclock(void)
{

	PCPU_SET(clockadj, 0);
	PCPU_SET(clock, ia64_get_itc());
	ia64_set_itm(PCPU_GET(clock) + ia64_clock_reload);
	ia64_set_itv(CLOCK_VECTOR);	/* highest priority class */
	ia64_srlz_d();
}

/*
 * Account time for a transition between system, hard irq or soft irq state.
 * Note that this function is called with interrupts enabled.
 */
static __u64 vtime_delta(struct task_struct *tsk)
{
	struct thread_info *ti = task_thread_info(tsk);
	__u64 now, delta_stime;

	WARN_ON_ONCE(!irqs_disabled());

	now = ia64_get_itc();
	delta_stime = now - ti->ac_stamp;
	ti->ac_stamp = now;

	return delta_stime;
}

/*
 * Account time for a transition between system, hard irq or soft irq state.
 * Note that this function is called with interrupts enabled.
 */
static cputime_t vtime_delta(struct task_struct *tsk)
{
	struct thread_info *ti = task_thread_info(tsk);
	cputime_t delta_stime;
	__u64 now;

	WARN_ON_ONCE(!irqs_disabled());

	now = ia64_get_itc();

	delta_stime = cycle_to_cputime(ti->ac_stime + (now - ti->ac_stamp));
	ti->ac_stime = 0;
	ti->ac_stamp = now;

	return delta_stime;
}

void
sync_master (void *arg)
{
	unsigned long flags, i;

	go[MASTER] = 0;

	local_irq_save(flags);
	{
		for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) {
			while (!go[MASTER]);
			go[MASTER] = 0;
			go[SLAVE] = ia64_get_itc();
		}
	}
	local_irq_restore(flags);
}

/*
 * Return the number of micro-seconds that elapsed since the last update to jiffy.  The
 * xtime_lock must be at least read-locked when calling this routine.
 */
static inline unsigned long
gettimeoffset (void)
{
	unsigned long elapsed_cycles, lost = jiffies - wall_jiffies;
	unsigned long now, last_tick;
#	define time_keeper_id	0	/* smp_processor_id() of time-keeper */

	last_tick = (cpu_data(time_keeper_id)->itm_next
		     - (lost + 1)*cpu_data(time_keeper_id)->itm_delta);

	now = ia64_get_itc();
	if ((long) (now - last_tick) < 0) {
		printk(KERN_ERR "CPU %d: now < last_tick (now=0x%lx,last_tick=0x%lx)!\n",
		       smp_processor_id(), now, last_tick);
		return last_time_offset;
	}
	elapsed_cycles = now - last_tick;
	return (elapsed_cycles*local_cpu_data->usec_per_cyc) >> IA64_USEC_PER_CYC_SHIFT;
}

/*
 * Encapsulate access to the itm structure for SMP.
 */
void
ia64_cpu_local_tick (void)
{
    int cpu = smp_processor_id();
    unsigned long shift = 0, delta;

    /* arrange for the cycle counter to generate a timer interrupt: */
    ia64_set_itv(IA64_TIMER_VECTOR);

    delta = local_cpu_data->itm_delta;
    /*
     * Stagger the timer tick for each CPU so they don't occur all at (almost) the
     * same time:
     */
    if (cpu) {
        unsigned long hi = 1UL << ia64_fls(cpu);
        shift = (2*(cpu - hi) + 1) * delta/hi/2;
    }
    local_cpu_data->itm_next = ia64_get_itc() + delta + shift;
    ia64_set_itm(local_cpu_data->itm_next);
}

/*
 * Account time for a transition between system, hard irq or soft irq state.
 * Note that this function is called with interrupts enabled.
 */
void account_system_vtime(struct task_struct *tsk)
{
    struct thread_info *ti = task_thread_info(tsk);
    unsigned long flags;
    cputime_t delta_stime;
    __u64 now;

    local_irq_save(flags);

    now = ia64_get_itc();

    delta_stime = cycle_to_cputime(ti->ac_stime + (now - ti->ac_stamp));
    if (irq_count() || idle_task(smp_processor_id()) != tsk)
        account_system_time(tsk, 0, delta_stime, delta_stime);
    else
        account_idle_time(delta_stime);
    ti->ac_stime = 0;

    ti->ac_stamp = now;

    local_irq_restore(flags);
}

/*
 * Called from the context switch with interrupts disabled, to charge all
 * accumulated times to the current process, and to prepare accounting on
 * the next process.
 */
void ia64_account_on_switch(struct task_struct *prev, struct task_struct *next)
{
    struct thread_info *pi = task_thread_info(prev);
    struct thread_info *ni = task_thread_info(next);
    cputime_t delta_stime, delta_utime;
    __u64 now;

    now = ia64_get_itc();

    delta_stime = cycle_to_cputime(pi->ac_stime + (now - pi->ac_stamp));
    if (idle_task(smp_processor_id()) != prev)
        account_system_time(prev, 0, delta_stime, delta_stime);
    else
        account_idle_time(delta_stime);

    if (pi->ac_utime) {
        delta_utime = cycle_to_cputime(pi->ac_utime);
        account_user_time(prev, delta_utime, delta_utime);
    }

    pi->ac_stamp = ni->ac_stamp = now;
    ni->ac_stime = ni->ac_utime = 0;
}

static void __init
check_sal_cache_flush (void)
{
	unsigned long flags;
	int cpu;
	u64 vector;

	cpu = get_cpu();
	local_irq_save(flags);

	/*
	 * Schedule a timer interrupt, wait until it's reported, and see if
	 * SAL_CACHE_FLUSH drops it.
	 */
	ia64_set_itv(IA64_TIMER_VECTOR);
	ia64_set_itm(ia64_get_itc() + 1000);

	while (!ia64_get_irr(IA64_TIMER_VECTOR))
		cpu_relax();

	ia64_sal_cache_flush(3);

	if (ia64_get_irr(IA64_TIMER_VECTOR)) {
		vector = ia64_get_ivr();
		ia64_eoi();
		WARN_ON(vector != IA64_TIMER_VECTOR);
	} else {
		sal_cache_flush_drops_interrupts = 1;
		printk(KERN_ERR "SAL: SAL_CACHE_FLUSH drops interrupts; "
			"PAL_CACHE_FLUSH will be used instead\n");
		ia64_eoi();
	}

	local_irq_restore(flags);
	put_cpu();
}

/*
 * Synchronize ar.itc of the current (slave) CPU with the ar.itc of the MASTER CPU
 * (normally the time-keeper CPU).  We use a closed loop to eliminate the possibility of
 * unaccounted-for errors (such as getting a machine check in the middle of a calibration
 * step).  The basic idea is for the slave to ask the master what itc value it has and to
 * read its own itc before and after the master responds.  Each iteration gives us three
 * timestamps:
 *
 *	slave		master
 *
 *	t0 ---\
 *             ---\
 *		   --->
 *			tm
 *		   /---
 *	       /---
 *	t1 <---
 *
 *
 * The goal is to adjust the slave's ar.itc such that tm falls exactly half-way between t0
 * and t1.  If we achieve this, the clocks are synchronized provided the interconnect
 * between the slave and the master is symmetric.  Even if the interconnect were
 * asymmetric, we would still know that the synchronization error is smaller than the
 * roundtrip latency (t0 - t1).
 *
 * When the interconnect is quiet and symmetric, this lets us synchronize the itc to
 * within one or two cycles.  However, we can only *guarantee* that the synchronization is
 * accurate to within a round-trip time, which is typically in the range of several
 * hundred cycles (e.g., ~500 cycles).  In practice, this means that the itc's are usually
 * almost perfectly synchronized, but we shouldn't assume that the accuracy is much better
 * than half a micro second or so.
 */
void
ia64_sync_itc (unsigned int master)
{
	long i, delta, adj, adjust_latency = 0, done = 0;
	unsigned long flags, rt, master_time_stamp, bound;
#if DEBUG_ITC_SYNC
	struct {
		long rt;	/* roundtrip time */
		long master;	/* master's timestamp */
		long diff;	/* difference between midpoint and master's timestamp */
		long lat;	/* estimate of itc adjustment latency */
	} t[NUM_ROUNDS];
#endif

	/*
	 * Make sure local timer ticks are disabled while we sync.  If
	 * they were enabled, we'd have to worry about nasty issues
	 * like setting the ITC ahead of (or a long time before) the
	 * next scheduled tick.
	 */
	BUG_ON((ia64_get_itv() & (1 << 16)) == 0);

	go[MASTER] = 1;

	if (smp_call_function_single(master, sync_master, NULL, 1, 0) < 0) {
		printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master);
		return;
	}

	while (go[MASTER])
		cpu_relax();	/* wait for master to be ready */

	spin_lock_irqsave(&itc_sync_lock, flags);
	{
		for (i = 0; i < NUM_ROUNDS; ++i) {
			delta = get_delta(&rt, &master_time_stamp);
			if (delta == 0) {
				done = 1;	/* let's lock on to this... */
				bound = rt;
			}

			if (!done) {
				if (i > 0) {
					adjust_latency += -delta;
					adj = -delta + adjust_latency/4;
				} else
					adj = -delta;

				ia64_set_itc(ia64_get_itc() + adj);
			}
#if DEBUG_ITC_SYNC
			t[i].rt = rt;
			t[i].master = master_time_stamp;
			t[i].diff = delta;
			t[i].lat = adjust_latency/4;
#endif
		}
	}
	spin_unlock_irqrestore(&itc_sync_lock, flags);

#if DEBUG_ITC_SYNC
	for (i = 0; i < NUM_ROUNDS; ++i)
		printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
		       t[i].rt, t[i].master, t[i].diff, t[i].lat);
#endif

	printk(KERN_INFO "CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, "
	       "maxerr %lu cycles)\n", smp_processor_id(), master, delta, rt);
//.........这里部分代码省略.........