linux/arch/x86/include/asm/uv/uv_bau.h

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * SGI UV Broadcast Assist Unit definitions
 *
 * Copyright (C) 2008-2011 Silicon Graphics, Inc. All rights reserved.
 */

#ifndef _ASM_X86_UV_UV_BAU_H
#define _ASM_X86_UV_UV_BAU_H

#include <linux/bitmap.h>
#define BITSPERBYTE 8

/*
 * Broadcast Assist Unit messaging structures
 *
 * Selective Broadcast activations are induced by software action
 * specifying a particular 8-descriptor "set" via a 6-bit index written
 * to an MMR.
 * Thus there are 64 unique 512-byte sets of SB descriptors - one set for
 * each 6-bit index value. These descriptor sets are mapped in sequence
 * starting with set 0 located at the address specified in the
 * BAU_SB_DESCRIPTOR_BASE register, set 1 is located at BASE + 512,
 * set 2 is at BASE + 2*512, set 3 at BASE + 3*512, and so on.
 *
 * We will use one set for sending BAU messages from each of the
 * cpu's on the uvhub.
 *
 * TLB shootdown will use the first of the 8 descriptors of each set.
 * Each of the descriptors is 64 bytes in size (8*64 = 512 bytes in a set).
 */

#define MAX_CPUS_PER_UVHUB		128
#define MAX_CPUS_PER_SOCKET		64
#define ADP_SZ				64 /* hardware-provided max. */
#define UV_CPUS_PER_AS			32 /* hardware-provided max. */
#define ITEMS_PER_DESC			8
/* the 'throttle' to prevent the hardware stay-busy bug */
#define MAX_BAU_CONCURRENT		3
#define UV_ACT_STATUS_MASK		0x3
#define UV_ACT_STATUS_SIZE		2
#define UV_DISTRIBUTION_SIZE		256
#define UV_SW_ACK_NPENDING		8
#define UV1_NET_ENDPOINT_INTD		0x38
#define UV2_NET_ENDPOINT_INTD		0x28
#define UV_NET_ENDPOINT_INTD		(is_uv1_hub() ?			\
			UV1_NET_ENDPOINT_INTD : UV2_NET_ENDPOINT_INTD)
#define UV_DESC_PSHIFT			49
#define UV_PAYLOADQ_GNODE_SHIFT		49
#define UV_PTC_BASENAME			"sgi_uv/ptc_statistics"
#define UV_BAU_BASENAME			"sgi_uv/bau_tunables"
#define UV_BAU_TUNABLES_DIR		"sgi_uv"
#define UV_BAU_TUNABLES_FILE		"bau_tunables"
#define WHITESPACE			" \t\n"
#define cpubit_isset(cpu, bau_local_cpumask) \
	test_bit((cpu), (bau_local_cpumask).bits)

/* [19:16] SOFT_ACK timeout period  19: 1 is urgency 7  17:16 1 is multiplier */
/*
 * UV2: Bit 19 selects between
 *  (0): 10 microsecond timebase and
 *  (1): 80 microseconds
 *  we're using 560us, similar to UV1: 65 units of 10us
 */
#define UV1_INTD_SOFT_ACK_TIMEOUT_PERIOD (9UL)
#define UV2_INTD_SOFT_ACK_TIMEOUT_PERIOD (15UL)

#define UV_INTD_SOFT_ACK_TIMEOUT_PERIOD	(is_uv1_hub() ?			\
		UV1_INTD_SOFT_ACK_TIMEOUT_PERIOD :			\
		UV2_INTD_SOFT_ACK_TIMEOUT_PERIOD)
/* assuming UV3 is the same */

#define BAU_MISC_CONTROL_MULT_MASK	3

#define UVH_AGING_PRESCALE_SEL		0x000000b000UL
/* [30:28] URGENCY_7  an index into a table of times */
#define BAU_URGENCY_7_SHIFT		28
#define BAU_URGENCY_7_MASK		7

#define UVH_TRANSACTION_TIMEOUT		0x000000b200UL
/* [45:40] BAU - BAU transaction timeout select - a multiplier */
#define BAU_TRANS_SHIFT			40
#define BAU_TRANS_MASK			0x3f

/*
 * shorten some awkward names
 */
#define AS_PUSH_SHIFT UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT
#define SOFTACK_MSHIFT UVH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT
#define SOFTACK_PSHIFT UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT
#define SOFTACK_TIMEOUT_PERIOD UV_INTD_SOFT_ACK_TIMEOUT_PERIOD
#define PREFETCH_HINT_SHFT UV3H_LB_BAU_MISC_CONTROL_ENABLE_INTD_PREFETCH_HINT_SHFT
#define SB_STATUS_SHFT UV3H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT
#define write_gmmr	uv_write_global_mmr64
#define write_lmmr	uv_write_local_mmr
#define read_lmmr	uv_read_local_mmr
#define read_gmmr	uv_read_global_mmr64

/*
 * bits in UVH_LB_BAU_SB_ACTIVATION_STATUS_0/1
 */
#define DS_IDLE				0
#define DS_ACTIVE			1
#define DS_DESTINATION_TIMEOUT		2
#define DS_SOURCE_TIMEOUT		3
/*
 * bits put together from HRP_LB_BAU_SB_ACTIVATION_STATUS_0/1/2
 * values 1 and 3 will not occur
 *        Decoded meaning              ERROR  BUSY    AUX ERR
 * -------------------------------     ----   -----   -------
 * IDLE                                 0       0        0
 * BUSY (active)                        0       1        0
 * SW Ack Timeout (destination)         1       0        0
 * SW Ack INTD rejected (strong NACK)   1       0        1
 * Source Side Time Out Detected        1       1        0
 * Destination Side PUT Failed          1       1        1
 */
#define UV2H_DESC_IDLE			0
#define UV2H_DESC_BUSY			2
#define UV2H_DESC_DEST_TIMEOUT		4
#define UV2H_DESC_DEST_STRONG_NACK	5
#define UV2H_DESC_SOURCE_TIMEOUT	6
#define UV2H_DESC_DEST_PUT_ERR		7

/*
 * delay for 'plugged' timeout retries, in microseconds
 */
#define PLUGGED_DELAY			10

/*
 * threshholds at which to use IPI to free resources
 */
/* after this # consecutive 'plugged' timeouts, use IPI to release resources */
#define PLUGSB4RESET			100
/* after this many consecutive timeouts, use IPI to release resources */
#define TIMEOUTSB4RESET			1
/* at this number uses of IPI to release resources, giveup the request */
#define IPI_RESET_LIMIT			1
/* after this # consecutive successes, bump up the throttle if it was lowered */
#define COMPLETE_THRESHOLD		5
/* after this # of giveups (fall back to kernel IPI's) disable the use of
   the BAU for a period of time */
#define GIVEUP_LIMIT			100

#define UV_LB_SUBNODEID			0x10

/* these two are the same for UV1 and UV2: */
#define UV_SA_SHFT UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT
#define UV_SA_MASK UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK
/* 4 bits of software ack period */
#define UV2_ACK_MASK			0x7UL
#define UV2_ACK_UNITS_SHFT		3
#define UV2_EXT_SHFT UV2H_LB_BAU_MISC_CONTROL_ENABLE_EXTENDED_SB_STATUS_SHFT

/*
 * number of entries in the destination side payload queue
 */
#define DEST_Q_SIZE			20
/*
 * number of destination side software ack resources
 */
#define DEST_NUM_RESOURCES		8
/*
 * completion statuses for sending a TLB flush message
 */
#define FLUSH_RETRY_PLUGGED		1
#define FLUSH_RETRY_TIMEOUT		2
#define FLUSH_GIVEUP			3
#define FLUSH_COMPLETE			4

/*
 * tuning the action when the numalink network is extremely delayed
 */
#define CONGESTED_RESPONSE_US		1000	/* 'long' response time, in
						   microseconds */
#define CONGESTED_REPS			10	/* long delays averaged over
						   this many broadcasts */
#define DISABLED_PERIOD			10	/* time for the bau to be
						   disabled, in seconds */
/* see msg_type: */
#define MSG_NOOP			0
#define MSG_REGULAR			1
#define MSG_RETRY			2

#define BAU_DESC_QUALIFIER		0x534749

enum uv_bau_version {
	UV_BAU_V1 = 1,
	UV_BAU_V2,
	UV_BAU_V3,
	UV_BAU_V4,
};

/*
 * Distribution: 32 bytes (256 bits) (bytes 0-0x1f of descriptor)
 * If the 'multilevel' flag in the header portion of the descriptor
 * has been set to 0, then endpoint multi-unicast mode is selected.
 * The distribution specification (32 bytes) is interpreted as a 256-bit
 * distribution vector. Adjacent bits correspond to consecutive even numbered
 * nodeIDs. The result of adding the index of a given bit to the 15-bit
 * 'base_dest_nasid' field of the header corresponds to the
 * destination nodeID associated with that specified bit.
 */
struct pnmask {
	unsigned long		bits[BITS_TO_LONGS(UV_DISTRIBUTION_SIZE)];
};

/*
 * mask of cpu's on a uvhub
 * (during initialization we need to check that unsigned long has
 *  enough bits for max. cpu's per uvhub)
 */
struct bau_local_cpumask {
	unsigned long		bits;
};

/*
 * Payload: 16 bytes (128 bits) (bytes 0x20-0x2f of descriptor)
 * only 12 bytes (96 bits) of the payload area are usable.
 * An additional 3 bytes (bits 27:4) of the header address are carried
 * to the next bytes of the destination payload queue.
 * And an additional 2 bytes of the header Suppl_A field are also
 * carried to the destination payload queue.
 * But the first byte of the Suppl_A becomes bits 127:120 (the 16th byte)
 * of the destination payload queue, which is written by the hardware
 * with the s/w ack resource bit vector.
 * [ effective message contents (16 bytes (128 bits) maximum), not counting
 *   the s/w ack bit vector  ]
 */

/**
 * struct uv1_2_3_bau_msg_payload - defines payload for INTD transactions
 * @address:		Signifies a page or all TLB's of the cpu
 * @sending_cpu:	CPU from which the message originates
 * @acknowledge_count:	CPUs on the destination Hub that received the interrupt
 */
struct uv1_2_3_bau_msg_payload {
	u64 address;
	u16 sending_cpu;
	u16 acknowledge_count;
};

/**
 * struct uv4_bau_msg_payload - defines payload for INTD transactions
 * @address:		Signifies a page or all TLB's of the cpu
 * @sending_cpu:	CPU from which the message originates
 * @acknowledge_count:	CPUs on the destination Hub that received the interrupt
 * @qualifier:		Set by source to verify origin of INTD broadcast
 */
struct uv4_bau_msg_payload {
	u64 address;
	u16 sending_cpu;
	u16 acknowledge_count;
	u32 reserved:8;
	u32 qualifier:24;
};

/*
 * UV1 Message header:  16 bytes (128 bits) (bytes 0x30-0x3f of descriptor)
 * see table 4.2.3.0.1 in broacast_assist spec.
 */
struct uv1_bau_msg_header {
	unsigned int	dest_subnodeid:6;	/* must be 0x10, for the LB */
	/* bits 5:0 */
	unsigned int	base_dest_nasid:15;	/* nasid of the first bit */
	/* bits 20:6 */				/* in uvhub map */
	unsigned int	command:8;		/* message type */
	/* bits 28:21 */
	/* 0x38: SN3net EndPoint Message */
	unsigned int	rsvd_1:3;		/* must be zero */
	/* bits 31:29 */
	/* int will align on 32 bits */
	unsigned int	rsvd_2:9;		/* must be zero */
	/* bits 40:32 */
	/* Suppl_A is 56-41 */
	unsigned int	sequence:16;		/* message sequence number */
	/* bits 56:41 */			/* becomes bytes 16-17 of msg */
						/* Address field (96:57) is
						   never used as an address
						   (these are address bits
						   42:3) */

	unsigned int	rsvd_3:1;		/* must be zero */
	/* bit 57 */
	/* address bits 27:4 are payload */
	/* these next 24  (58-81) bits become bytes 12-14 of msg */
	/* bits 65:58 land in byte 12 */
	unsigned int	replied_to:1;		/* sent as 0 by the source to
						   byte 12 */
	/* bit 58 */
	unsigned int	msg_type:3;		/* software type of the
						   message */
	/* bits 61:59 */
	unsigned int	canceled:1;		/* message canceled, resource
						   is to be freed*/
	/* bit 62 */
	unsigned int	payload_1a:1;		/* not currently used */
	/* bit 63 */
	unsigned int	payload_1b:2;		/* not currently used */
	/* bits 65:64 */

	/* bits 73:66 land in byte 13 */
	unsigned int	payload_1ca:6;		/* not currently used */
	/* bits 71:66 */
	unsigned int	payload_1c:2;		/* not currently used */
	/* bits 73:72 */

	/* bits 81:74 land in byte 14 */
	unsigned int	payload_1d:6;		/* not currently used */
	/* bits 79:74 */
	unsigned int	payload_1e:2;		/* not currently used */
	/* bits 81:80 */

	unsigned int	rsvd_4:7;		/* must be zero */
	/* bits 88:82 */
	unsigned int	swack_flag:1;		/* software acknowledge flag */
	/* bit 89 */
						/* INTD trasactions at
						   destination are to wait for
						   software acknowledge */
	unsigned int	rsvd_5:6;		/* must be zero */
	/* bits 95:90 */
	unsigned int	rsvd_6:5;		/* must be zero */
	/* bits 100:96 */
	unsigned int	int_both:1;		/* if 1, interrupt both sockets
						   on the uvhub */
	/* bit 101*/
	unsigned int	fairness:3;		/* usually zero */
	/* bits 104:102 */
	unsigned int	multilevel:1;		/* multi-level multicast
						   format */
	/* bit 105 */
	/* 0 for TLB: endpoint multi-unicast messages */
	unsigned int	chaining:1;		/* next descriptor is part of
						   this activation*/
	/* bit 106 */
	unsigned int	rsvd_7:21;		/* must be zero */
	/* bits 127:107 */
};

/*
 * UV2 Message header:  16 bytes (128 bits) (bytes 0x30-0x3f of descriptor)
 * see figure 9-2 of harp_sys.pdf
 * assuming UV3 is the same
 */
struct uv2_3_bau_msg_header {
	unsigned int	base_dest_nasid:15;	/* nasid of the first bit */
	/* bits 14:0 */				/* in uvhub map */
	unsigned int	dest_subnodeid:5;	/* must be 0x10, for the LB */
	/* bits 19:15 */
	unsigned int	rsvd_1:1;		/* must be zero */
	/* bit 20 */
	/* Address bits 59:21 */
	/* bits 25:2 of address (44:21) are payload */
	/* these next 24 bits become bytes 12-14 of msg */
	/* bits 28:21 land in byte 12 */
	unsigned int	replied_to:1;		/* sent as 0 by the source to
						   byte 12 */
	/* bit 21 */
	unsigned int	msg_type:3;		/* software type of the
						   message */
	/* bits 24:22 */
	unsigned int	canceled:1;		/* message canceled, resource
						   is to be freed*/
	/* bit 25 */
	unsigned int	payload_1:3;		/* not currently used */
	/* bits 28:26 */

	/* bits 36:29 land in byte 13 */
	unsigned int	payload_2a:3;		/* not currently used */
	unsigned int	payload_2b:5;		/* not currently used */
	/* bits 36:29 */

	/* bits 44:37 land in byte 14 */
	unsigned int	payload_3:8;		/* not currently used */
	/* bits 44:37 */

	unsigned int	rsvd_2:7;		/* reserved */
	/* bits 51:45 */
	unsigned int	swack_flag:1;		/* software acknowledge flag */
	/* bit 52 */
	unsigned int	rsvd_3a:3;		/* must be zero */
	unsigned int	rsvd_3b:8;		/* must be zero */
	unsigned int	rsvd_3c:8;		/* must be zero */
	unsigned int	rsvd_3d:3;		/* must be zero */
	/* bits 74:53 */
	unsigned int	fairness:3;		/* usually zero */
	/* bits 77:75 */

	unsigned int	sequence:16;		/* message sequence number */
	/* bits 93:78  Suppl_A  */
	unsigned int	chaining:1;		/* next descriptor is part of
						   this activation*/
	/* bit 94 */
	unsigned int	multilevel:1;		/* multi-level multicast
						   format */
	/* bit 95 */
	unsigned int	rsvd_4:24;		/* ordered / source node /
						   source subnode / aging
						   must be zero */
	/* bits 119:96 */
	unsigned int	command:8;		/* message type */
	/* bits 127:120 */
};

/*
 * The activation descriptor:
 * The format of the message to send, plus all accompanying control
 * Should be 64 bytes
 */
struct bau_desc {
	struct pnmask				distribution;
	/*
	 * message template, consisting of header and payload:
	 */
	union bau_msg_header {
		struct uv1_bau_msg_header	uv1_hdr;
		struct uv2_3_bau_msg_header	uv2_3_hdr;
	} header;

	union bau_payload_header {
		struct uv1_2_3_bau_msg_payload	uv1_2_3;
		struct uv4_bau_msg_payload	uv4;
	} payload;
};
/* UV1:
 *   -payload--    ---------header------
 *   bytes 0-11    bits 41-56  bits 58-81
 *       A           B  (2)      C (3)
 *
 *            A/B/C are moved to:
 *       A            C          B
 *   bytes 0-11  bytes 12-14  bytes 16-17  (byte 15 filled in by hw as vector)
 *   ------------payload queue-----------
 */
/* UV2:
 *   -payload--    ---------header------
 *   bytes 0-11    bits 70-78  bits 21-44
 *       A           B  (2)      C (3)
 *
 *            A/B/C are moved to:
 *       A            C          B
 *   bytes 0-11  bytes 12-14  bytes 16-17  (byte 15 filled in by hw as vector)
 *   ------------payload queue-----------
 */

/*
 * The payload queue on the destination side is an array of these.
 * With BAU_MISC_CONTROL set for software acknowledge mode, the messages
 * are 32 bytes (2 micropackets) (256 bits) in length, but contain only 17
 * bytes of usable data, including the sw ack vector in byte 15 (bits 127:120)
 * (12 bytes come from bau_msg_payload, 3 from payload_1, 2 from
 *  swack_vec and payload_2)
 * "Enabling Software Acknowledgment mode (see Section 4.3.3 Software
 *  Acknowledge Processing) also selects 32 byte (17 bytes usable) payload
 *  operation."
 */
struct bau_pq_entry {
	unsigned long	address;	/* signifies a page or all TLB's
					   of the cpu */
	/* 64 bits, bytes 0-7 */
	unsigned short	sending_cpu;	/* cpu that sent the message */
	/* 16 bits, bytes 8-9 */
	unsigned short	acknowledge_count; /* filled in by destination */
	/* 16 bits, bytes 10-11 */
	/* these next 3 bytes come from bits 58-81 of the message header */
	unsigned short	replied_to:1;	/* sent as 0 by the source */
	unsigned short	msg_type:3;	/* software message type */
	unsigned short	canceled:1;	/* sent as 0 by the source */
	unsigned short	unused1:3;	/* not currently using */
	/* byte 12 */
	unsigned char	unused2a;	/* not currently using */
	/* byte 13 */
	unsigned char	unused2;	/* not currently using */
	/* byte 14 */
	unsigned char	swack_vec;	/* filled in by the hardware */
	/* byte 15 (bits 127:120) */
	unsigned short	sequence;	/* message sequence number */
	/* bytes 16-17 */
	unsigned char	unused4[2];	/* not currently using bytes 18-19 */
	/* bytes 18-19 */
	int		number_of_cpus;	/* filled in at destination */
	/* 32 bits, bytes 20-23 (aligned) */
	unsigned char	unused5[8];	/* not using */
	/* bytes 24-31 */
};

struct msg_desc {
	struct bau_pq_entry	*msg;
	int			msg_slot;
	struct bau_pq_entry	*queue_first;
	struct bau_pq_entry	*queue_last;
};

struct reset_args {
	int			sender;
};

/*
 * This structure is allocated per_cpu for UV TLB shootdown statistics.
 */
struct ptc_stats {
	/* sender statistics */
	unsigned long	s_giveup;		/* number of fall backs to
						   IPI-style flushes */
	unsigned long	s_requestor;		/* number of shootdown
						   requests */
	unsigned long	s_stimeout;		/* source side timeouts */
	unsigned long	s_dtimeout;		/* destination side timeouts */
	unsigned long	s_strongnacks;		/* number of strong nack's */
	unsigned long	s_time;			/* time spent in sending side */
	unsigned long	s_retriesok;		/* successful retries */
	unsigned long	s_ntargcpu;		/* total number of cpu's
						   targeted */
	unsigned long	s_ntargself;		/* times the sending cpu was
						   targeted */
	unsigned long	s_ntarglocals;		/* targets of cpus on the local
						   blade */
	unsigned long	s_ntargremotes;		/* targets of cpus on remote
						   blades */
	unsigned long	s_ntarglocaluvhub;	/* targets of the local hub */
	unsigned long	s_ntargremoteuvhub;	/* remotes hubs targeted */
	unsigned long	s_ntarguvhub;		/* total number of uvhubs
						   targeted */
	unsigned long	s_ntarguvhub16;		/* number of times target
						   hubs >= 16*/
	unsigned long	s_ntarguvhub8;		/* number of times target
						   hubs >= 8 */
	unsigned long	s_ntarguvhub4;		/* number of times target
						   hubs >= 4 */
	unsigned long	s_ntarguvhub2;		/* number of times target
						   hubs >= 2 */
	unsigned long	s_ntarguvhub1;		/* number of times target
						   hubs == 1 */
	unsigned long	s_resets_plug;		/* ipi-style resets from plug
						   state */
	unsigned long	s_resets_timeout;	/* ipi-style resets from
						   timeouts */
	unsigned long	s_busy;			/* status stayed busy past
						   s/w timer */
	unsigned long	s_throttles;		/* waits in throttle */
	unsigned long	s_retry_messages;	/* retry broadcasts */
	unsigned long	s_bau_reenabled;	/* for bau enable/disable */
	unsigned long	s_bau_disabled;		/* for bau enable/disable */
	unsigned long	s_uv2_wars;		/* uv2 workaround, perm. busy */
	unsigned long	s_uv2_wars_hw;		/* uv2 workaround, hiwater */
	unsigned long	s_uv2_war_waits;	/* uv2 workaround, long waits */
	unsigned long	s_overipilimit;		/* over the ipi reset limit */
	unsigned long	s_giveuplimit;		/* disables, over giveup limit*/
	unsigned long	s_enters;		/* entries to the driver */
	unsigned long	s_ipifordisabled;	/* fall back to IPI; disabled */
	unsigned long	s_plugged;		/* plugged by h/w bug*/
	unsigned long	s_congested;		/* giveup on long wait */
	/* destination statistics */
	unsigned long	d_alltlb;		/* times all tlb's on this
						   cpu were flushed */
	unsigned long	d_onetlb;		/* times just one tlb on this
						   cpu was flushed */
	unsigned long	d_multmsg;		/* interrupts with multiple
						   messages */
	unsigned long	d_nomsg;		/* interrupts with no message */
	unsigned long	d_time;			/* time spent on destination
						   side */
	unsigned long	d_requestee;		/* number of messages
						   processed */
	unsigned long	d_retries;		/* number of retry messages
						   processed */
	unsigned long	d_canceled;		/* number of messages canceled
						   by retries */
	unsigned long	d_nocanceled;		/* retries that found nothing
						   to cancel */
	unsigned long	d_resets;		/* number of ipi-style requests
						   processed */
	unsigned long	d_rcanceled;		/* number of messages canceled
						   by resets */
};

struct tunables {
	int			*tunp;
	int			deflt;
};

struct hub_and_pnode {
	short			uvhub;
	short			pnode;
};

struct socket_desc {
	short			num_cpus;
	short			cpu_number[MAX_CPUS_PER_SOCKET];
};

struct uvhub_desc {
	unsigned short		socket_mask;
	short			num_cpus;
	short			uvhub;
	short			pnode;
	struct socket_desc	socket[2];
};

/**
 * struct bau_control
 * @status_mmr: location of status mmr, determined by uvhub_cpu
 * @status_index: index of ERR|BUSY bits in status mmr, determined by uvhub_cpu
 *
 * Per-cpu control struct containing CPU topology information and BAU tuneables.
 */
struct bau_control {
	struct bau_desc		*descriptor_base;
	struct bau_pq_entry	*queue_first;
	struct bau_pq_entry	*queue_last;
	struct bau_pq_entry	*bau_msg_head;
	struct bau_control	*uvhub_master;
	struct bau_control	*socket_master;
	struct ptc_stats	*statp;
	cpumask_t		*cpumask;
	unsigned long		timeout_interval;
	unsigned long		set_bau_on_time;
	atomic_t		active_descriptor_count;
	int			plugged_tries;
	int			timeout_tries;
	int			ipi_attempts;
	int			conseccompletes;
	u64			status_mmr;
	int			status_index;
	bool			nobau;
	short			baudisabled;
	short			cpu;
	short			osnode;
	short			uvhub_cpu;
	short			uvhub;
	short			uvhub_version;
	short			cpus_in_socket;
	short			cpus_in_uvhub;
	short			partition_base_pnode;
	short			busy;       /* all were busy (war) */
	unsigned short		message_number;
	unsigned short		uvhub_quiesce;
	short			socket_acknowledge_count[DEST_Q_SIZE];
	cycles_t		send_message;
	cycles_t		period_end;
	cycles_t		period_time;
	spinlock_t		uvhub_lock;
	spinlock_t		queue_lock;
	spinlock_t		disable_lock;
	/* tunables */
	int			max_concurr;
	int			max_concurr_const;
	int			plugged_delay;
	int			plugsb4reset;
	int			timeoutsb4reset;
	int			ipi_reset_limit;
	int			complete_threshold;
	int			cong_response_us;
	int			cong_reps;
	cycles_t		disabled_period;
	int			period_giveups;
	int			giveup_limit;
	long			period_requests;
	struct hub_and_pnode	*thp;
};

/* Abstracted BAU functions */
struct bau_operations {
	unsigned long	(*read_l_sw_ack)(void);
	unsigned long	(*read_g_sw_ack)(int pnode);
	unsigned long	(*bau_gpa_to_offset)(unsigned long vaddr);
	void		(*write_l_sw_ack)(unsigned long mmr);
	void		(*write_g_sw_ack)(int pnode, unsigned long mmr);
	void		(*write_payload_first)(int pnode, unsigned long mmr);
	void		(*write_payload_last)(int pnode, unsigned long mmr);
	int		(*wait_completion)(struct bau_desc*,
				struct bau_control*, long try);
};

static inline void write_mmr_data_broadcast(int pnode, unsigned long mmr_image)
{
	write_gmmr(pnode, UVH_BAU_DATA_BROADCAST, mmr_image);
}

static inline void write_mmr_descriptor_base(int pnode, unsigned long mmr_image)
{
	write_gmmr(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE, mmr_image);
}

static inline void write_mmr_activation(unsigned long index)
{
	write_lmmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
}

static inline void write_gmmr_activation(int pnode, unsigned long mmr_image)
{
	write_gmmr(pnode, UVH_LB_BAU_SB_ACTIVATION_CONTROL, mmr_image);
}

static inline void write_mmr_proc_payload_first(int pnode, unsigned long mmr_image)
{
	write_gmmr(pnode, UV4H_LB_PROC_INTD_QUEUE_FIRST, mmr_image);
}

static inline void write_mmr_proc_payload_last(int pnode, unsigned long mmr_image)
{
	write_gmmr(pnode, UV4H_LB_PROC_INTD_QUEUE_LAST, mmr_image);
}

static inline void write_mmr_payload_first(int pnode, unsigned long mmr_image)
{
	write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, mmr_image);
}

static inline void write_mmr_payload_tail(int pnode, unsigned long mmr_image)
{
	write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, mmr_image);
}

static inline void write_mmr_payload_last(int pnode, unsigned long mmr_image)
{
	write_gmmr(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, mmr_image);
}

static inline void write_mmr_misc_control(int pnode, unsigned long mmr_image)
{
	write_gmmr(pnode, UVH_LB_BAU_MISC_CONTROL, mmr_image);
}

static inline unsigned long read_mmr_misc_control(int pnode)
{
	return read_gmmr(pnode, UVH_LB_BAU_MISC_CONTROL);
}

static inline void write_mmr_sw_ack(unsigned long mr)
{
	uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, mr);
}

static inline void write_gmmr_sw_ack(int pnode, unsigned long mr)
{
	write_gmmr(pnode, UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, mr);
}

static inline unsigned long read_mmr_sw_ack(void)
{
	return read_lmmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE);
}

static inline unsigned long read_gmmr_sw_ack(int pnode)
{
	return read_gmmr(pnode, UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE);
}

static inline void write_mmr_proc_sw_ack(unsigned long mr)
{
	uv_write_local_mmr(UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR, mr);
}

static inline void write_gmmr_proc_sw_ack(int pnode, unsigned long mr)
{
	write_gmmr(pnode, UV4H_LB_PROC_INTD_SOFT_ACK_CLEAR, mr);
}

static inline unsigned long read_mmr_proc_sw_ack(void)
{
	return read_lmmr(UV4H_LB_PROC_INTD_SOFT_ACK_PENDING);
}

static inline unsigned long read_gmmr_proc_sw_ack(int pnode)
{
	return read_gmmr(pnode, UV4H_LB_PROC_INTD_SOFT_ACK_PENDING);
}

static inline void write_mmr_data_config(int pnode, unsigned long mr)
{
	uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, mr);
}

static inline int bau_uvhub_isset(int uvhub, struct pnmask *dstp)
{
	return constant_test_bit(uvhub, &dstp->bits[0]);
}
static inline void bau_uvhub_set(int pnode, struct pnmask *dstp)
{
	__set_bit(pnode, &dstp->bits[0]);
}
static inline void bau_uvhubs_clear(struct pnmask *dstp,
				    int nbits)
{
	bitmap_zero(&dstp->bits[0], nbits);
}
static inline int bau_uvhub_weight(struct pnmask *dstp)
{
	return bitmap_weight((unsigned long *)&dstp->bits[0],
				UV_DISTRIBUTION_SIZE);
}

static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits)
{
	bitmap_zero(&dstp->bits, nbits);
}

extern void uv_bau_message_intr1(void);
#ifdef CONFIG_TRACING
#define trace_uv_bau_message_intr1 uv_bau_message_intr1
#endif
extern void uv_bau_timeout_intr1(void);

struct atomic_short {
	short counter;
};

/*
 * atomic_read_short - read a short atomic variable
 * @v: pointer of type atomic_short
 *
 * Atomically reads the value of @v.
 */
static inline int atomic_read_short(const struct atomic_short *v)
{
	return v->counter;
}

/*
 * atom_asr - add and return a short int
 * @i: short value to add
 * @v: pointer of type atomic_short
 *
 * Atomically adds @i to @v and returns @i + @v
 */
static inline int atom_asr(short i, struct atomic_short *v)
{
	short __i = i;
	asm volatile(LOCK_PREFIX "xaddw %0, %1"
			: "+r" (i), "+m" (v->counter)
			: : "memory");
	return i + __i;
}

/*
 * conditionally add 1 to *v, unless *v is >= u
 * return 0 if we cannot add 1 to *v because it is >= u
 * return 1 if we can add 1 to *v because it is < u
 * the add is atomic
 *
 * This is close to atomic_add_unless(), but this allows the 'u' value
 * to be lowered below the current 'v'.  atomic_add_unless can only stop
 * on equal.
 */
static inline int atomic_inc_unless_ge(spinlock_t *lock, atomic_t *v, int u)
{
	spin_lock(lock);
	if (atomic_read(v) >= u) {
		spin_unlock(lock);
		return 0;
	}
	atomic_inc(v);
	spin_unlock(lock);
	return 1;
}

#endif /* _ASM_X86_UV_UV_BAU_H */