HardenedBSD src tree https://hardenedbsd.org/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

10995 lines
286 KiB

  1. /*-
  2. * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  3. *
  4. * Copyright (c) 2011 Chelsio Communications, Inc.
  5. * All rights reserved.
  6. * Written by: Navdeep Parhar <np@FreeBSD.org>
  7. *
  8. * Redistribution and use in source and binary forms, with or without
  9. * modification, are permitted provided that the following conditions
  10. * are met:
  11. * 1. Redistributions of source code must retain the above copyright
  12. * notice, this list of conditions and the following disclaimer.
  13. * 2. Redistributions in binary form must reproduce the above copyright
  14. * notice, this list of conditions and the following disclaimer in the
  15. * documentation and/or other materials provided with the distribution.
  16. *
  17. * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18. * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20. * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21. * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22. * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23. * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24. * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25. * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26. * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27. * SUCH DAMAGE.
  28. */
  29. #include <sys/cdefs.h>
  30. __FBSDID("$FreeBSD$");
  31. #include "opt_ddb.h"
  32. #include "opt_inet.h"
  33. #include "opt_inet6.h"
  34. #include "opt_ratelimit.h"
  35. #include "opt_rss.h"
  36. #include <sys/param.h>
  37. #include <sys/conf.h>
  38. #include <sys/priv.h>
  39. #include <sys/kernel.h>
  40. #include <sys/bus.h>
  41. #include <sys/module.h>
  42. #include <sys/malloc.h>
  43. #include <sys/queue.h>
  44. #include <sys/taskqueue.h>
  45. #include <sys/pciio.h>
  46. #include <dev/pci/pcireg.h>
  47. #include <dev/pci/pcivar.h>
  48. #include <dev/pci/pci_private.h>
  49. #include <sys/firmware.h>
  50. #include <sys/sbuf.h>
  51. #include <sys/smp.h>
  52. #include <sys/socket.h>
  53. #include <sys/sockio.h>
  54. #include <sys/sysctl.h>
  55. #include <net/ethernet.h>
  56. #include <net/if.h>
  57. #include <net/if_types.h>
  58. #include <net/if_dl.h>
  59. #include <net/if_vlan_var.h>
  60. #ifdef RSS
  61. #include <net/rss_config.h>
  62. #endif
  63. #include <netinet/in.h>
  64. #include <netinet/ip.h>
  65. #if defined(__i386__) || defined(__amd64__)
  66. #include <machine/md_var.h>
  67. #include <machine/cputypes.h>
  68. #include <vm/vm.h>
  69. #include <vm/pmap.h>
  70. #endif
  71. #ifdef DDB
  72. #include <ddb/ddb.h>
  73. #include <ddb/db_lex.h>
  74. #endif
  75. #include "common/common.h"
  76. #include "common/t4_msg.h"
  77. #include "common/t4_regs.h"
  78. #include "common/t4_regs_values.h"
  79. #include "cudbg/cudbg.h"
  80. #include "t4_clip.h"
  81. #include "t4_ioctl.h"
  82. #include "t4_l2t.h"
  83. #include "t4_mp_ring.h"
  84. #include "t4_if.h"
  85. #include "t4_smt.h"
  86. /* T4 bus driver interface */
  87. static int t4_probe(device_t);
  88. static int t4_attach(device_t);
  89. static int t4_detach(device_t);
  90. static int t4_child_location_str(device_t, device_t, char *, size_t);
  91. static int t4_ready(device_t);
  92. static int t4_read_port_device(device_t, int, device_t *);
  93. static device_method_t t4_methods[] = {
  94. DEVMETHOD(device_probe, t4_probe),
  95. DEVMETHOD(device_attach, t4_attach),
  96. DEVMETHOD(device_detach, t4_detach),
  97. DEVMETHOD(bus_child_location_str, t4_child_location_str),
  98. DEVMETHOD(t4_is_main_ready, t4_ready),
  99. DEVMETHOD(t4_read_port_device, t4_read_port_device),
  100. DEVMETHOD_END
  101. };
  102. static driver_t t4_driver = {
  103. "t4nex",
  104. t4_methods,
  105. sizeof(struct adapter)
  106. };
  107. /* T4 port (cxgbe) interface */
  108. static int cxgbe_probe(device_t);
  109. static int cxgbe_attach(device_t);
  110. static int cxgbe_detach(device_t);
  111. device_method_t cxgbe_methods[] = {
  112. DEVMETHOD(device_probe, cxgbe_probe),
  113. DEVMETHOD(device_attach, cxgbe_attach),
  114. DEVMETHOD(device_detach, cxgbe_detach),
  115. { 0, 0 }
  116. };
  117. static driver_t cxgbe_driver = {
  118. "cxgbe",
  119. cxgbe_methods,
  120. sizeof(struct port_info)
  121. };
  122. /* T4 VI (vcxgbe) interface */
  123. static int vcxgbe_probe(device_t);
  124. static int vcxgbe_attach(device_t);
  125. static int vcxgbe_detach(device_t);
  126. static device_method_t vcxgbe_methods[] = {
  127. DEVMETHOD(device_probe, vcxgbe_probe),
  128. DEVMETHOD(device_attach, vcxgbe_attach),
  129. DEVMETHOD(device_detach, vcxgbe_detach),
  130. { 0, 0 }
  131. };
  132. static driver_t vcxgbe_driver = {
  133. "vcxgbe",
  134. vcxgbe_methods,
  135. sizeof(struct vi_info)
  136. };
  137. static d_ioctl_t t4_ioctl;
  138. static struct cdevsw t4_cdevsw = {
  139. .d_version = D_VERSION,
  140. .d_ioctl = t4_ioctl,
  141. .d_name = "t4nex",
  142. };
  143. /* T5 bus driver interface */
  144. static int t5_probe(device_t);
  145. static device_method_t t5_methods[] = {
  146. DEVMETHOD(device_probe, t5_probe),
  147. DEVMETHOD(device_attach, t4_attach),
  148. DEVMETHOD(device_detach, t4_detach),
  149. DEVMETHOD(bus_child_location_str, t4_child_location_str),
  150. DEVMETHOD(t4_is_main_ready, t4_ready),
  151. DEVMETHOD(t4_read_port_device, t4_read_port_device),
  152. DEVMETHOD_END
  153. };
  154. static driver_t t5_driver = {
  155. "t5nex",
  156. t5_methods,
  157. sizeof(struct adapter)
  158. };
  159. /* T5 port (cxl) interface */
  160. static driver_t cxl_driver = {
  161. "cxl",
  162. cxgbe_methods,
  163. sizeof(struct port_info)
  164. };
  165. /* T5 VI (vcxl) interface */
  166. static driver_t vcxl_driver = {
  167. "vcxl",
  168. vcxgbe_methods,
  169. sizeof(struct vi_info)
  170. };
  171. /* T6 bus driver interface */
  172. static int t6_probe(device_t);
  173. static device_method_t t6_methods[] = {
  174. DEVMETHOD(device_probe, t6_probe),
  175. DEVMETHOD(device_attach, t4_attach),
  176. DEVMETHOD(device_detach, t4_detach),
  177. DEVMETHOD(bus_child_location_str, t4_child_location_str),
  178. DEVMETHOD(t4_is_main_ready, t4_ready),
  179. DEVMETHOD(t4_read_port_device, t4_read_port_device),
  180. DEVMETHOD_END
  181. };
  182. static driver_t t6_driver = {
  183. "t6nex",
  184. t6_methods,
  185. sizeof(struct adapter)
  186. };
  187. /* T6 port (cc) interface */
  188. static driver_t cc_driver = {
  189. "cc",
  190. cxgbe_methods,
  191. sizeof(struct port_info)
  192. };
  193. /* T6 VI (vcc) interface */
  194. static driver_t vcc_driver = {
  195. "vcc",
  196. vcxgbe_methods,
  197. sizeof(struct vi_info)
  198. };
  199. /* ifnet interface */
  200. static void cxgbe_init(void *);
  201. static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
  202. static int cxgbe_transmit(struct ifnet *, struct mbuf *);
  203. static void cxgbe_qflush(struct ifnet *);
  204. MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
  205. /*
  206. * Correct lock order when you need to acquire multiple locks is t4_list_lock,
  207. * then ADAPTER_LOCK, then t4_uld_list_lock.
  208. */
  209. static struct sx t4_list_lock;
  210. SLIST_HEAD(, adapter) t4_list;
  211. #ifdef TCP_OFFLOAD
  212. static struct sx t4_uld_list_lock;
  213. SLIST_HEAD(, uld_info) t4_uld_list;
  214. #endif
  215. /*
  216. * Tunables. See tweak_tunables() too.
  217. *
  218. * Each tunable is set to a default value here if it's known at compile-time.
  219. * Otherwise it is set to -n as an indication to tweak_tunables() that it should
  220. * provide a reasonable default (upto n) when the driver is loaded.
  221. *
  222. * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to
  223. * T5 are under hw.cxl.
  224. */
  225. SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD, 0, "cxgbe(4) parameters");
  226. SYSCTL_NODE(_hw, OID_AUTO, cxl, CTLFLAG_RD, 0, "cxgbe(4) T5+ parameters");
  227. SYSCTL_NODE(_hw_cxgbe, OID_AUTO, toe, CTLFLAG_RD, 0, "cxgbe(4) TOE parameters");
  228. /*
  229. * Number of queues for tx and rx, NIC and offload.
  230. */
  231. #define NTXQ 16
  232. int t4_ntxq = -NTXQ;
  233. SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq, CTLFLAG_RDTUN, &t4_ntxq, 0,
  234. "Number of TX queues per port");
  235. TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq); /* Old name, undocumented */
  236. #define NRXQ 8
  237. int t4_nrxq = -NRXQ;
  238. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq, CTLFLAG_RDTUN, &t4_nrxq, 0,
  239. "Number of RX queues per port");
  240. TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq); /* Old name, undocumented */
  241. #define NTXQ_VI 1
  242. static int t4_ntxq_vi = -NTXQ_VI;
  243. SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq_vi, CTLFLAG_RDTUN, &t4_ntxq_vi, 0,
  244. "Number of TX queues per VI");
  245. #define NRXQ_VI 1
  246. static int t4_nrxq_vi = -NRXQ_VI;
  247. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq_vi, CTLFLAG_RDTUN, &t4_nrxq_vi, 0,
  248. "Number of RX queues per VI");
  249. static int t4_rsrv_noflowq = 0;
  250. SYSCTL_INT(_hw_cxgbe, OID_AUTO, rsrv_noflowq, CTLFLAG_RDTUN, &t4_rsrv_noflowq,
  251. 0, "Reserve TX queue 0 of each VI for non-flowid packets");
  252. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  253. #define NOFLDTXQ 8
  254. static int t4_nofldtxq = -NOFLDTXQ;
  255. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq, CTLFLAG_RDTUN, &t4_nofldtxq, 0,
  256. "Number of offload TX queues per port");
  257. #define NOFLDRXQ 2
  258. static int t4_nofldrxq = -NOFLDRXQ;
  259. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq, CTLFLAG_RDTUN, &t4_nofldrxq, 0,
  260. "Number of offload RX queues per port");
  261. #define NOFLDTXQ_VI 1
  262. static int t4_nofldtxq_vi = -NOFLDTXQ_VI;
  263. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq_vi, CTLFLAG_RDTUN, &t4_nofldtxq_vi, 0,
  264. "Number of offload TX queues per VI");
  265. #define NOFLDRXQ_VI 1
  266. static int t4_nofldrxq_vi = -NOFLDRXQ_VI;
  267. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq_vi, CTLFLAG_RDTUN, &t4_nofldrxq_vi, 0,
  268. "Number of offload RX queues per VI");
  269. #define TMR_IDX_OFLD 1
  270. int t4_tmr_idx_ofld = TMR_IDX_OFLD;
  271. SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx_ofld, CTLFLAG_RDTUN,
  272. &t4_tmr_idx_ofld, 0, "Holdoff timer index for offload queues");
  273. #define PKTC_IDX_OFLD (-1)
  274. int t4_pktc_idx_ofld = PKTC_IDX_OFLD;
  275. SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx_ofld, CTLFLAG_RDTUN,
  276. &t4_pktc_idx_ofld, 0, "holdoff packet counter index for offload queues");
  277. /* 0 means chip/fw default, non-zero number is value in microseconds */
  278. static u_long t4_toe_keepalive_idle = 0;
  279. SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_idle, CTLFLAG_RDTUN,
  280. &t4_toe_keepalive_idle, 0, "TOE keepalive idle timer (us)");
  281. /* 0 means chip/fw default, non-zero number is value in microseconds */
  282. static u_long t4_toe_keepalive_interval = 0;
  283. SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_interval, CTLFLAG_RDTUN,
  284. &t4_toe_keepalive_interval, 0, "TOE keepalive interval timer (us)");
  285. /* 0 means chip/fw default, non-zero number is # of keepalives before abort */
  286. static int t4_toe_keepalive_count = 0;
  287. SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, keepalive_count, CTLFLAG_RDTUN,
  288. &t4_toe_keepalive_count, 0, "Number of TOE keepalive probes before abort");
  289. /* 0 means chip/fw default, non-zero number is value in microseconds */
  290. static u_long t4_toe_rexmt_min = 0;
  291. SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_min, CTLFLAG_RDTUN,
  292. &t4_toe_rexmt_min, 0, "Minimum TOE retransmit interval (us)");
  293. /* 0 means chip/fw default, non-zero number is value in microseconds */
  294. static u_long t4_toe_rexmt_max = 0;
  295. SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_max, CTLFLAG_RDTUN,
  296. &t4_toe_rexmt_max, 0, "Maximum TOE retransmit interval (us)");
  297. /* 0 means chip/fw default, non-zero number is # of rexmt before abort */
  298. static int t4_toe_rexmt_count = 0;
  299. SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, rexmt_count, CTLFLAG_RDTUN,
  300. &t4_toe_rexmt_count, 0, "Number of TOE retransmissions before abort");
  301. /* -1 means chip/fw default, other values are raw backoff values to use */
  302. static int t4_toe_rexmt_backoff[16] = {
  303. -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
  304. };
  305. SYSCTL_NODE(_hw_cxgbe_toe, OID_AUTO, rexmt_backoff, CTLFLAG_RD, 0,
  306. "cxgbe(4) TOE retransmit backoff values");
  307. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 0, CTLFLAG_RDTUN,
  308. &t4_toe_rexmt_backoff[0], 0, "");
  309. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 1, CTLFLAG_RDTUN,
  310. &t4_toe_rexmt_backoff[1], 0, "");
  311. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 2, CTLFLAG_RDTUN,
  312. &t4_toe_rexmt_backoff[2], 0, "");
  313. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 3, CTLFLAG_RDTUN,
  314. &t4_toe_rexmt_backoff[3], 0, "");
  315. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 4, CTLFLAG_RDTUN,
  316. &t4_toe_rexmt_backoff[4], 0, "");
  317. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 5, CTLFLAG_RDTUN,
  318. &t4_toe_rexmt_backoff[5], 0, "");
  319. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 6, CTLFLAG_RDTUN,
  320. &t4_toe_rexmt_backoff[6], 0, "");
  321. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 7, CTLFLAG_RDTUN,
  322. &t4_toe_rexmt_backoff[7], 0, "");
  323. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 8, CTLFLAG_RDTUN,
  324. &t4_toe_rexmt_backoff[8], 0, "");
  325. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 9, CTLFLAG_RDTUN,
  326. &t4_toe_rexmt_backoff[9], 0, "");
  327. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 10, CTLFLAG_RDTUN,
  328. &t4_toe_rexmt_backoff[10], 0, "");
  329. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 11, CTLFLAG_RDTUN,
  330. &t4_toe_rexmt_backoff[11], 0, "");
  331. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 12, CTLFLAG_RDTUN,
  332. &t4_toe_rexmt_backoff[12], 0, "");
  333. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 13, CTLFLAG_RDTUN,
  334. &t4_toe_rexmt_backoff[13], 0, "");
  335. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 14, CTLFLAG_RDTUN,
  336. &t4_toe_rexmt_backoff[14], 0, "");
  337. SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 15, CTLFLAG_RDTUN,
  338. &t4_toe_rexmt_backoff[15], 0, "");
  339. #endif
  340. #ifdef DEV_NETMAP
  341. #define NN_MAIN_VI (1 << 0) /* Native netmap on the main VI */
  342. #define NN_EXTRA_VI (1 << 1) /* Native netmap on the extra VI(s) */
  343. static int t4_native_netmap = NN_EXTRA_VI;
  344. SYSCTL_INT(_hw_cxgbe, OID_AUTO, native_netmap, CTLFLAG_RDTUN, &t4_native_netmap,
  345. 0, "Native netmap support. bit 0 = main VI, bit 1 = extra VIs");
  346. #define NNMTXQ 8
  347. static int t4_nnmtxq = -NNMTXQ;
  348. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq, CTLFLAG_RDTUN, &t4_nnmtxq, 0,
  349. "Number of netmap TX queues");
  350. #define NNMRXQ 8
  351. static int t4_nnmrxq = -NNMRXQ;
  352. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq, CTLFLAG_RDTUN, &t4_nnmrxq, 0,
  353. "Number of netmap RX queues");
  354. #define NNMTXQ_VI 2
  355. static int t4_nnmtxq_vi = -NNMTXQ_VI;
  356. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq_vi, CTLFLAG_RDTUN, &t4_nnmtxq_vi, 0,
  357. "Number of netmap TX queues per VI");
  358. #define NNMRXQ_VI 2
  359. static int t4_nnmrxq_vi = -NNMRXQ_VI;
  360. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq_vi, CTLFLAG_RDTUN, &t4_nnmrxq_vi, 0,
  361. "Number of netmap RX queues per VI");
  362. #endif
  363. /*
  364. * Holdoff parameters for ports.
  365. */
  366. #define TMR_IDX 1
  367. int t4_tmr_idx = TMR_IDX;
  368. SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx, CTLFLAG_RDTUN, &t4_tmr_idx,
  369. 0, "Holdoff timer index");
  370. TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx); /* Old name */
  371. #define PKTC_IDX (-1)
  372. int t4_pktc_idx = PKTC_IDX;
  373. SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx, CTLFLAG_RDTUN, &t4_pktc_idx,
  374. 0, "Holdoff packet counter index");
  375. TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx); /* Old name */
  376. /*
  377. * Size (# of entries) of each tx and rx queue.
  378. */
  379. unsigned int t4_qsize_txq = TX_EQ_QSIZE;
  380. SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_txq, CTLFLAG_RDTUN, &t4_qsize_txq, 0,
  381. "Number of descriptors in each TX queue");
  382. unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
  383. SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_rxq, CTLFLAG_RDTUN, &t4_qsize_rxq, 0,
  384. "Number of descriptors in each RX queue");
  385. /*
  386. * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
  387. */
  388. int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
  389. SYSCTL_INT(_hw_cxgbe, OID_AUTO, interrupt_types, CTLFLAG_RDTUN, &t4_intr_types,
  390. 0, "Interrupt types allowed (bit 0 = INTx, 1 = MSI, 2 = MSI-X)");
  391. /*
  392. * Configuration file. All the _CF names here are special.
  393. */
  394. #define DEFAULT_CF "default"
  395. #define BUILTIN_CF "built-in"
  396. #define FLASH_CF "flash"
  397. #define UWIRE_CF "uwire"
  398. #define FPGA_CF "fpga"
  399. static char t4_cfg_file[32] = DEFAULT_CF;
  400. SYSCTL_STRING(_hw_cxgbe, OID_AUTO, config_file, CTLFLAG_RDTUN, t4_cfg_file,
  401. sizeof(t4_cfg_file), "Firmware configuration file");
  402. /*
  403. * PAUSE settings (bit 0, 1, 2 = rx_pause, tx_pause, pause_autoneg respectively).
  404. * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
  405. * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
  406. * mark or when signalled to do so, 0 to never emit PAUSE.
  407. * pause_autoneg = 1 means PAUSE will be negotiated if possible and the
  408. * negotiated settings will override rx_pause/tx_pause.
  409. * Otherwise rx_pause/tx_pause are applied forcibly.
  410. */
  411. static int t4_pause_settings = PAUSE_RX | PAUSE_TX | PAUSE_AUTONEG;
  412. SYSCTL_INT(_hw_cxgbe, OID_AUTO, pause_settings, CTLFLAG_RDTUN,
  413. &t4_pause_settings, 0,
  414. "PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)");
  415. /*
  416. * Forward Error Correction settings (bit 0, 1 = RS, BASER respectively).
  417. * -1 to run with the firmware default. Same as FEC_AUTO (bit 5)
  418. * 0 to disable FEC.
  419. */
  420. static int t4_fec = -1;
  421. SYSCTL_INT(_hw_cxgbe, OID_AUTO, fec, CTLFLAG_RDTUN, &t4_fec, 0,
  422. "Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)");
  423. /*
  424. * Link autonegotiation.
  425. * -1 to run with the firmware default.
  426. * 0 to disable.
  427. * 1 to enable.
  428. */
  429. static int t4_autoneg = -1;
  430. SYSCTL_INT(_hw_cxgbe, OID_AUTO, autoneg, CTLFLAG_RDTUN, &t4_autoneg, 0,
  431. "Link autonegotiation");
  432. /*
  433. * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
  434. * encouraged respectively). '-n' is the same as 'n' except the firmware
  435. * version used in the checks is read from the firmware bundled with the driver.
  436. */
  437. static int t4_fw_install = 1;
  438. SYSCTL_INT(_hw_cxgbe, OID_AUTO, fw_install, CTLFLAG_RDTUN, &t4_fw_install, 0,
  439. "Firmware auto-install (0 = prohibited, 1 = allowed, 2 = encouraged)");
  440. /*
  441. * ASIC features that will be used. Disable the ones you don't want so that the
  442. * chip resources aren't wasted on features that will not be used.
  443. */
  444. static int t4_nbmcaps_allowed = 0;
  445. SYSCTL_INT(_hw_cxgbe, OID_AUTO, nbmcaps_allowed, CTLFLAG_RDTUN,
  446. &t4_nbmcaps_allowed, 0, "Default NBM capabilities");
  447. static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */
  448. SYSCTL_INT(_hw_cxgbe, OID_AUTO, linkcaps_allowed, CTLFLAG_RDTUN,
  449. &t4_linkcaps_allowed, 0, "Default link capabilities");
  450. static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS |
  451. FW_CAPS_CONFIG_SWITCH_EGRESS;
  452. SYSCTL_INT(_hw_cxgbe, OID_AUTO, switchcaps_allowed, CTLFLAG_RDTUN,
  453. &t4_switchcaps_allowed, 0, "Default switch capabilities");
  454. #ifdef RATELIMIT
  455. static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC |
  456. FW_CAPS_CONFIG_NIC_HASHFILTER | FW_CAPS_CONFIG_NIC_ETHOFLD;
  457. #else
  458. static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC |
  459. FW_CAPS_CONFIG_NIC_HASHFILTER;
  460. #endif
  461. SYSCTL_INT(_hw_cxgbe, OID_AUTO, niccaps_allowed, CTLFLAG_RDTUN,
  462. &t4_niccaps_allowed, 0, "Default NIC capabilities");
  463. static int t4_toecaps_allowed = -1;
  464. SYSCTL_INT(_hw_cxgbe, OID_AUTO, toecaps_allowed, CTLFLAG_RDTUN,
  465. &t4_toecaps_allowed, 0, "Default TCP offload capabilities");
  466. static int t4_rdmacaps_allowed = -1;
  467. SYSCTL_INT(_hw_cxgbe, OID_AUTO, rdmacaps_allowed, CTLFLAG_RDTUN,
  468. &t4_rdmacaps_allowed, 0, "Default RDMA capabilities");
  469. static int t4_cryptocaps_allowed = -1;
  470. SYSCTL_INT(_hw_cxgbe, OID_AUTO, cryptocaps_allowed, CTLFLAG_RDTUN,
  471. &t4_cryptocaps_allowed, 0, "Default crypto capabilities");
  472. static int t4_iscsicaps_allowed = -1;
  473. SYSCTL_INT(_hw_cxgbe, OID_AUTO, iscsicaps_allowed, CTLFLAG_RDTUN,
  474. &t4_iscsicaps_allowed, 0, "Default iSCSI capabilities");
  475. static int t4_fcoecaps_allowed = 0;
  476. SYSCTL_INT(_hw_cxgbe, OID_AUTO, fcoecaps_allowed, CTLFLAG_RDTUN,
  477. &t4_fcoecaps_allowed, 0, "Default FCoE capabilities");
  478. static int t5_write_combine = 0;
  479. SYSCTL_INT(_hw_cxl, OID_AUTO, write_combine, CTLFLAG_RDTUN, &t5_write_combine,
  480. 0, "Use WC instead of UC for BAR2");
  481. static int t4_num_vis = 1;
  482. SYSCTL_INT(_hw_cxgbe, OID_AUTO, num_vis, CTLFLAG_RDTUN, &t4_num_vis, 0,
  483. "Number of VIs per port");
  484. /*
  485. * PCIe Relaxed Ordering.
  486. * -1: driver should figure out a good value.
  487. * 0: disable RO.
  488. * 1: enable RO.
  489. * 2: leave RO alone.
  490. */
  491. static int pcie_relaxed_ordering = -1;
  492. SYSCTL_INT(_hw_cxgbe, OID_AUTO, pcie_relaxed_ordering, CTLFLAG_RDTUN,
  493. &pcie_relaxed_ordering, 0,
  494. "PCIe Relaxed Ordering: 0 = disable, 1 = enable, 2 = leave alone");
  495. static int t4_panic_on_fatal_err = 0;
  496. SYSCTL_INT(_hw_cxgbe, OID_AUTO, panic_on_fatal_err, CTLFLAG_RDTUN,
  497. &t4_panic_on_fatal_err, 0, "panic on fatal errors");
  498. #ifdef TCP_OFFLOAD
  499. /*
  500. * TOE tunables.
  501. */
  502. static int t4_cop_managed_offloading = 0;
  503. SYSCTL_INT(_hw_cxgbe, OID_AUTO, cop_managed_offloading, CTLFLAG_RDTUN,
  504. &t4_cop_managed_offloading, 0,
  505. "COP (Connection Offload Policy) controls all TOE offload");
  506. #endif
  507. /* Functions used by VIs to obtain unique MAC addresses for each VI. */
  508. static int vi_mac_funcs[] = {
  509. FW_VI_FUNC_ETH,
  510. FW_VI_FUNC_OFLD,
  511. FW_VI_FUNC_IWARP,
  512. FW_VI_FUNC_OPENISCSI,
  513. FW_VI_FUNC_OPENFCOE,
  514. FW_VI_FUNC_FOISCSI,
  515. FW_VI_FUNC_FOFCOE,
  516. };
  517. struct intrs_and_queues {
  518. uint16_t intr_type; /* INTx, MSI, or MSI-X */
  519. uint16_t num_vis; /* number of VIs for each port */
  520. uint16_t nirq; /* Total # of vectors */
  521. uint16_t ntxq; /* # of NIC txq's for each port */
  522. uint16_t nrxq; /* # of NIC rxq's for each port */
  523. uint16_t nofldtxq; /* # of TOE/ETHOFLD txq's for each port */
  524. uint16_t nofldrxq; /* # of TOE rxq's for each port */
  525. uint16_t nnmtxq; /* # of netmap txq's */
  526. uint16_t nnmrxq; /* # of netmap rxq's */
  527. /* The vcxgbe/vcxl interfaces use these and not the ones above. */
  528. uint16_t ntxq_vi; /* # of NIC txq's */
  529. uint16_t nrxq_vi; /* # of NIC rxq's */
  530. uint16_t nofldtxq_vi; /* # of TOE txq's */
  531. uint16_t nofldrxq_vi; /* # of TOE rxq's */
  532. uint16_t nnmtxq_vi; /* # of netmap txq's */
  533. uint16_t nnmrxq_vi; /* # of netmap rxq's */
  534. };
  535. static void setup_memwin(struct adapter *);
  536. static void position_memwin(struct adapter *, int, uint32_t);
  537. static int validate_mem_range(struct adapter *, uint32_t, uint32_t);
  538. static int fwmtype_to_hwmtype(int);
  539. static int validate_mt_off_len(struct adapter *, int, uint32_t, uint32_t,
  540. uint32_t *);
  541. static int fixup_devlog_params(struct adapter *);
  542. static int cfg_itype_and_nqueues(struct adapter *, struct intrs_and_queues *);
  543. static int contact_firmware(struct adapter *);
  544. static int partition_resources(struct adapter *);
  545. static int get_params__pre_init(struct adapter *);
  546. static int set_params__pre_init(struct adapter *);
  547. static int get_params__post_init(struct adapter *);
  548. static int set_params__post_init(struct adapter *);
  549. static void t4_set_desc(struct adapter *);
  550. static bool fixed_ifmedia(struct port_info *);
  551. static void build_medialist(struct port_info *);
  552. static void init_link_config(struct port_info *);
  553. static int fixup_link_config(struct port_info *);
  554. static int apply_link_config(struct port_info *);
  555. static int cxgbe_init_synchronized(struct vi_info *);
  556. static int cxgbe_uninit_synchronized(struct vi_info *);
  557. static void quiesce_txq(struct adapter *, struct sge_txq *);
  558. static void quiesce_wrq(struct adapter *, struct sge_wrq *);
  559. static void quiesce_iq(struct adapter *, struct sge_iq *);
  560. static void quiesce_fl(struct adapter *, struct sge_fl *);
  561. static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
  562. driver_intr_t *, void *, char *);
  563. static int t4_free_irq(struct adapter *, struct irq *);
  564. static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
  565. static void vi_refresh_stats(struct adapter *, struct vi_info *);
  566. static void cxgbe_refresh_stats(struct adapter *, struct port_info *);
  567. static void cxgbe_tick(void *);
  568. static void cxgbe_sysctls(struct port_info *);
  569. static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
  570. static int sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS);
  571. static int sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS);
  572. static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
  573. static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
  574. static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
  575. static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
  576. static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
  577. static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
  578. static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
  579. static int sysctl_fec(SYSCTL_HANDLER_ARGS);
  580. static int sysctl_module_fec(SYSCTL_HANDLER_ARGS);
  581. static int sysctl_autoneg(SYSCTL_HANDLER_ARGS);
  582. static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
  583. static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
  584. static int sysctl_vdd(SYSCTL_HANDLER_ARGS);
  585. static int sysctl_reset_sensor(SYSCTL_HANDLER_ARGS);
  586. static int sysctl_loadavg(SYSCTL_HANDLER_ARGS);
  587. static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
  588. static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
  589. static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
  590. static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
  591. static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
  592. static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
  593. static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
  594. static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
  595. static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
  596. static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
  597. static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
  598. static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
  599. static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
  600. static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
  601. static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
  602. static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS);
  603. static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
  604. static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
  605. static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
  606. static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
  607. static int sysctl_tids(SYSCTL_HANDLER_ARGS);
  608. static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
  609. static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS);
  610. static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
  611. static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
  612. static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
  613. static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
  614. static int sysctl_cpus(SYSCTL_HANDLER_ARGS);
  615. #ifdef TCP_OFFLOAD
  616. static int sysctl_tls_rx_ports(SYSCTL_HANDLER_ARGS);
  617. static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS);
  618. static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS);
  619. static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS);
  620. static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS);
  621. static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS);
  622. static int sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS);
  623. static int sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS);
  624. #endif
  625. static int get_sge_context(struct adapter *, struct t4_sge_context *);
  626. static int load_fw(struct adapter *, struct t4_data *);
  627. static int load_cfg(struct adapter *, struct t4_data *);
  628. static int load_boot(struct adapter *, struct t4_bootrom *);
  629. static int load_bootcfg(struct adapter *, struct t4_data *);
  630. static int cudbg_dump(struct adapter *, struct t4_cudbg_dump *);
  631. static void free_offload_policy(struct t4_offload_policy *);
  632. static int set_offload_policy(struct adapter *, struct t4_offload_policy *);
  633. static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
  634. static int read_i2c(struct adapter *, struct t4_i2c_data *);
  635. static int clear_stats(struct adapter *, u_int);
  636. #ifdef TCP_OFFLOAD
  637. static int toe_capability(struct vi_info *, int);
  638. #endif
  639. static int mod_event(module_t, int, void *);
  640. static int notify_siblings(device_t, int);
  641. struct {
  642. uint16_t device;
  643. char *desc;
  644. } t4_pciids[] = {
  645. {0xa000, "Chelsio Terminator 4 FPGA"},
  646. {0x4400, "Chelsio T440-dbg"},
  647. {0x4401, "Chelsio T420-CR"},
  648. {0x4402, "Chelsio T422-CR"},
  649. {0x4403, "Chelsio T440-CR"},
  650. {0x4404, "Chelsio T420-BCH"},
  651. {0x4405, "Chelsio T440-BCH"},
  652. {0x4406, "Chelsio T440-CH"},
  653. {0x4407, "Chelsio T420-SO"},
  654. {0x4408, "Chelsio T420-CX"},
  655. {0x4409, "Chelsio T420-BT"},
  656. {0x440a, "Chelsio T404-BT"},
  657. {0x440e, "Chelsio T440-LP-CR"},
  658. }, t5_pciids[] = {
  659. {0xb000, "Chelsio Terminator 5 FPGA"},
  660. {0x5400, "Chelsio T580-dbg"},
  661. {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */
  662. {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */
  663. {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */
  664. {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */
  665. {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */
  666. {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */
  667. {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */
  668. {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */
  669. {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */
  670. {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */
  671. {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */
  672. {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */
  673. {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */
  674. {0x5418, "Chelsio T540-BT"}, /* 4 x 10GBaseT */
  675. {0x5419, "Chelsio T540-LP-BT"}, /* 4 x 10GBaseT */
  676. {0x541a, "Chelsio T540-SO-BT"}, /* 4 x 10GBaseT, nomem */
  677. {0x541b, "Chelsio T540-SO-CR"}, /* 4 x 10G, nomem */
  678. /* Custom */
  679. {0x5483, "Custom T540-CR"},
  680. {0x5484, "Custom T540-BT"},
  681. }, t6_pciids[] = {
  682. {0xc006, "Chelsio Terminator 6 FPGA"}, /* T6 PE10K6 FPGA (PF0) */
  683. {0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */
  684. {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */
  685. {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */
  686. {0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */
  687. {0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */
  688. {0x6405, "Chelsio T6225-OCP-SO"}, /* 2 x 10/25G, nomem */
  689. {0x6406, "Chelsio T62100-OCP-SO"}, /* 2 x 40/50/100G, nomem */
  690. {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */
  691. {0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */
  692. {0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */
  693. {0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */
  694. {0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */
  695. {0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */
  696. {0x6414, "Chelsio T61100-OCP-SO"}, /* 1 x 40/50/100G, nomem */
  697. {0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */
  698. /* Custom */
  699. {0x6480, "Custom T6225-CR"},
  700. {0x6481, "Custom T62100-CR"},
  701. {0x6482, "Custom T6225-CR"},
  702. {0x6483, "Custom T62100-CR"},
  703. {0x6484, "Custom T64100-CR"},
  704. {0x6485, "Custom T6240-SO"},
  705. {0x6486, "Custom T6225-SO-CR"},
  706. {0x6487, "Custom T6225-CR"},
  707. };
  708. #ifdef TCP_OFFLOAD
  709. /*
  710. * service_iq_fl() has an iq and needs the fl. Offset of fl from the iq should
  711. * be exactly the same for both rxq and ofld_rxq.
  712. */
  713. CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
  714. CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
  715. #endif
  716. CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
  717. static int
  718. t4_probe(device_t dev)
  719. {
  720. int i;
  721. uint16_t v = pci_get_vendor(dev);
  722. uint16_t d = pci_get_device(dev);
  723. uint8_t f = pci_get_function(dev);
  724. if (v != PCI_VENDOR_ID_CHELSIO)
  725. return (ENXIO);
  726. /* Attach only to PF0 of the FPGA */
  727. if (d == 0xa000 && f != 0)
  728. return (ENXIO);
  729. for (i = 0; i < nitems(t4_pciids); i++) {
  730. if (d == t4_pciids[i].device) {
  731. device_set_desc(dev, t4_pciids[i].desc);
  732. return (BUS_PROBE_DEFAULT);
  733. }
  734. }
  735. return (ENXIO);
  736. }
  737. static int
  738. t5_probe(device_t dev)
  739. {
  740. int i;
  741. uint16_t v = pci_get_vendor(dev);
  742. uint16_t d = pci_get_device(dev);
  743. uint8_t f = pci_get_function(dev);
  744. if (v != PCI_VENDOR_ID_CHELSIO)
  745. return (ENXIO);
  746. /* Attach only to PF0 of the FPGA */
  747. if (d == 0xb000 && f != 0)
  748. return (ENXIO);
  749. for (i = 0; i < nitems(t5_pciids); i++) {
  750. if (d == t5_pciids[i].device) {
  751. device_set_desc(dev, t5_pciids[i].desc);
  752. return (BUS_PROBE_DEFAULT);
  753. }
  754. }
  755. return (ENXIO);
  756. }
  757. static int
  758. t6_probe(device_t dev)
  759. {
  760. int i;
  761. uint16_t v = pci_get_vendor(dev);
  762. uint16_t d = pci_get_device(dev);
  763. if (v != PCI_VENDOR_ID_CHELSIO)
  764. return (ENXIO);
  765. for (i = 0; i < nitems(t6_pciids); i++) {
  766. if (d == t6_pciids[i].device) {
  767. device_set_desc(dev, t6_pciids[i].desc);
  768. return (BUS_PROBE_DEFAULT);
  769. }
  770. }
  771. return (ENXIO);
  772. }
  773. static void
  774. t5_attribute_workaround(device_t dev)
  775. {
  776. device_t root_port;
  777. uint32_t v;
  778. /*
  779. * The T5 chips do not properly echo the No Snoop and Relaxed
  780. * Ordering attributes when replying to a TLP from a Root
  781. * Port. As a workaround, find the parent Root Port and
  782. * disable No Snoop and Relaxed Ordering. Note that this
  783. * affects all devices under this root port.
  784. */
  785. root_port = pci_find_pcie_root_port(dev);
  786. if (root_port == NULL) {
  787. device_printf(dev, "Unable to find parent root port\n");
  788. return;
  789. }
  790. v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
  791. PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
  792. if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
  793. 0)
  794. device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
  795. device_get_nameunit(root_port));
  796. }
  797. static const struct devnames devnames[] = {
  798. {
  799. .nexus_name = "t4nex",
  800. .ifnet_name = "cxgbe",
  801. .vi_ifnet_name = "vcxgbe",
  802. .pf03_drv_name = "t4iov",
  803. .vf_nexus_name = "t4vf",
  804. .vf_ifnet_name = "cxgbev"
  805. }, {
  806. .nexus_name = "t5nex",
  807. .ifnet_name = "cxl",
  808. .vi_ifnet_name = "vcxl",
  809. .pf03_drv_name = "t5iov",
  810. .vf_nexus_name = "t5vf",
  811. .vf_ifnet_name = "cxlv"
  812. }, {
  813. .nexus_name = "t6nex",
  814. .ifnet_name = "cc",
  815. .vi_ifnet_name = "vcc",
  816. .pf03_drv_name = "t6iov",
  817. .vf_nexus_name = "t6vf",
  818. .vf_ifnet_name = "ccv"
  819. }
  820. };
  821. void
  822. t4_init_devnames(struct adapter *sc)
  823. {
  824. int id;
  825. id = chip_id(sc);
  826. if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames))
  827. sc->names = &devnames[id - CHELSIO_T4];
  828. else {
  829. device_printf(sc->dev, "chip id %d is not supported.\n", id);
  830. sc->names = NULL;
  831. }
  832. }
  833. static int
  834. t4_ifnet_unit(struct adapter *sc, struct port_info *pi)
  835. {
  836. const char *parent, *name;
  837. long value;
  838. int line, unit;
  839. line = 0;
  840. parent = device_get_nameunit(sc->dev);
  841. name = sc->names->ifnet_name;
  842. while (resource_find_dev(&line, name, &unit, "at", parent) == 0) {
  843. if (resource_long_value(name, unit, "port", &value) == 0 &&
  844. value == pi->port_id)
  845. return (unit);
  846. }
  847. return (-1);
  848. }
  849. static int
  850. t4_attach(device_t dev)
  851. {
  852. struct adapter *sc;
  853. int rc = 0, i, j, rqidx, tqidx, nports;
  854. struct make_dev_args mda;
  855. struct intrs_and_queues iaq;
  856. struct sge *s;
  857. uint32_t *buf;
  858. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  859. int ofld_tqidx;
  860. #endif
  861. #ifdef TCP_OFFLOAD
  862. int ofld_rqidx;
  863. #endif
  864. #ifdef DEV_NETMAP
  865. int nm_rqidx, nm_tqidx;
  866. #endif
  867. int num_vis;
  868. sc = device_get_softc(dev);
  869. sc->dev = dev;
  870. TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);
  871. if ((pci_get_device(dev) & 0xff00) == 0x5400)
  872. t5_attribute_workaround(dev);
  873. pci_enable_busmaster(dev);
  874. if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
  875. uint32_t v;
  876. pci_set_max_read_req(dev, 4096);
  877. v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
  878. sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
  879. if (pcie_relaxed_ordering == 0 &&
  880. (v & PCIEM_CTL_RELAXED_ORD_ENABLE) != 0) {
  881. v &= ~PCIEM_CTL_RELAXED_ORD_ENABLE;
  882. pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
  883. } else if (pcie_relaxed_ordering == 1 &&
  884. (v & PCIEM_CTL_RELAXED_ORD_ENABLE) == 0) {
  885. v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
  886. pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
  887. }
  888. }
  889. sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
  890. sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL);
  891. sc->traceq = -1;
  892. mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
  893. snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
  894. device_get_nameunit(dev));
  895. snprintf(sc->lockname, sizeof(sc->lockname), "%s",
  896. device_get_nameunit(dev));
  897. mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
  898. t4_add_adapter(sc);
  899. mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
  900. TAILQ_INIT(&sc->sfl);
  901. callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
  902. mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF);
  903. sc->policy = NULL;
  904. rw_init(&sc->policy_lock, "connection offload policy");
  905. rc = t4_map_bars_0_and_4(sc);
  906. if (rc != 0)
  907. goto done; /* error message displayed already */
  908. memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
  909. /* Prepare the adapter for operation. */
  910. buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK);
  911. rc = -t4_prep_adapter(sc, buf);
  912. free(buf, M_CXGBE);
  913. if (rc != 0) {
  914. device_printf(dev, "failed to prepare adapter: %d.\n", rc);
  915. goto done;
  916. }
  917. /*
  918. * This is the real PF# to which we're attaching. Works from within PCI
  919. * passthrough environments too, where pci_get_function() could return a
  920. * different PF# depending on the passthrough configuration. We need to
  921. * use the real PF# in all our communication with the firmware.
  922. */
  923. j = t4_read_reg(sc, A_PL_WHOAMI);
  924. sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j);
  925. sc->mbox = sc->pf;
  926. t4_init_devnames(sc);
  927. if (sc->names == NULL) {
  928. rc = ENOTSUP;
  929. goto done; /* error message displayed already */
  930. }
  931. /*
  932. * Do this really early, with the memory windows set up even before the
  933. * character device. The userland tool's register i/o and mem read
  934. * will work even in "recovery mode".
  935. */
  936. setup_memwin(sc);
  937. if (t4_init_devlog_params(sc, 0) == 0)
  938. fixup_devlog_params(sc);
  939. make_dev_args_init(&mda);
  940. mda.mda_devsw = &t4_cdevsw;
  941. mda.mda_uid = UID_ROOT;
  942. mda.mda_gid = GID_WHEEL;
  943. mda.mda_mode = 0600;
  944. mda.mda_si_drv1 = sc;
  945. rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev));
  946. if (rc != 0)
  947. device_printf(dev, "failed to create nexus char device: %d.\n",
  948. rc);
  949. /* Go no further if recovery mode has been requested. */
  950. if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
  951. device_printf(dev, "recovery mode.\n");
  952. goto done;
  953. }
  954. #if defined(__i386__)
  955. if ((cpu_feature & CPUID_CX8) == 0) {
  956. device_printf(dev, "64 bit atomics not available.\n");
  957. rc = ENOTSUP;
  958. goto done;
  959. }
  960. #endif
  961. /* Contact the firmware and try to become the master driver. */
  962. rc = contact_firmware(sc);
  963. if (rc != 0)
  964. goto done; /* error message displayed already */
  965. MPASS(sc->flags & FW_OK);
  966. rc = get_params__pre_init(sc);
  967. if (rc != 0)
  968. goto done; /* error message displayed already */
  969. if (sc->flags & MASTER_PF) {
  970. rc = partition_resources(sc);
  971. if (rc != 0)
  972. goto done; /* error message displayed already */
  973. t4_intr_clear(sc);
  974. }
  975. rc = get_params__post_init(sc);
  976. if (rc != 0)
  977. goto done; /* error message displayed already */
  978. rc = set_params__post_init(sc);
  979. if (rc != 0)
  980. goto done; /* error message displayed already */
  981. rc = t4_map_bar_2(sc);
  982. if (rc != 0)
  983. goto done; /* error message displayed already */
  984. rc = t4_create_dma_tag(sc);
  985. if (rc != 0)
  986. goto done; /* error message displayed already */
  987. /*
  988. * First pass over all the ports - allocate VIs and initialize some
  989. * basic parameters like mac address, port type, etc.
  990. */
  991. for_each_port(sc, i) {
  992. struct port_info *pi;
  993. pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
  994. sc->port[i] = pi;
  995. /* These must be set before t4_port_init */
  996. pi->adapter = sc;
  997. pi->port_id = i;
  998. /*
  999. * XXX: vi[0] is special so we can't delay this allocation until
  1000. * pi->nvi's final value is known.
  1001. */
  1002. pi->vi = malloc(sizeof(struct vi_info) * t4_num_vis, M_CXGBE,
  1003. M_ZERO | M_WAITOK);
  1004. /*
  1005. * Allocate the "main" VI and initialize parameters
  1006. * like mac addr.
  1007. */
  1008. rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
  1009. if (rc != 0) {
  1010. device_printf(dev, "unable to initialize port %d: %d\n",
  1011. i, rc);
  1012. free(pi->vi, M_CXGBE);
  1013. free(pi, M_CXGBE);
  1014. sc->port[i] = NULL;
  1015. goto done;
  1016. }
  1017. snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
  1018. device_get_nameunit(dev), i);
  1019. mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
  1020. sc->chan_map[pi->tx_chan] = i;
  1021. /* All VIs on this port share this media. */
  1022. ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
  1023. cxgbe_media_status);
  1024. PORT_LOCK(pi);
  1025. init_link_config(pi);
  1026. fixup_link_config(pi);
  1027. build_medialist(pi);
  1028. if (fixed_ifmedia(pi))
  1029. pi->flags |= FIXED_IFMEDIA;
  1030. PORT_UNLOCK(pi);
  1031. pi->dev = device_add_child(dev, sc->names->ifnet_name,
  1032. t4_ifnet_unit(sc, pi));
  1033. if (pi->dev == NULL) {
  1034. device_printf(dev,
  1035. "failed to add device for port %d.\n", i);
  1036. rc = ENXIO;
  1037. goto done;
  1038. }
  1039. pi->vi[0].dev = pi->dev;
  1040. device_set_softc(pi->dev, pi);
  1041. }
  1042. /*
  1043. * Interrupt type, # of interrupts, # of rx/tx queues, etc.
  1044. */
  1045. nports = sc->params.nports;
  1046. rc = cfg_itype_and_nqueues(sc, &iaq);
  1047. if (rc != 0)
  1048. goto done; /* error message displayed already */
  1049. num_vis = iaq.num_vis;
  1050. sc->intr_type = iaq.intr_type;
  1051. sc->intr_count = iaq.nirq;
  1052. s = &sc->sge;
  1053. s->nrxq = nports * iaq.nrxq;
  1054. s->ntxq = nports * iaq.ntxq;
  1055. if (num_vis > 1) {
  1056. s->nrxq += nports * (num_vis - 1) * iaq.nrxq_vi;
  1057. s->ntxq += nports * (num_vis - 1) * iaq.ntxq_vi;
  1058. }
  1059. s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */
  1060. s->neq += nports; /* ctrl queues: 1 per port */
  1061. s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
  1062. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  1063. if (is_offload(sc) || is_ethoffload(sc)) {
  1064. s->nofldtxq = nports * iaq.nofldtxq;
  1065. if (num_vis > 1)
  1066. s->nofldtxq += nports * (num_vis - 1) * iaq.nofldtxq_vi;
  1067. s->neq += s->nofldtxq;
  1068. s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
  1069. M_CXGBE, M_ZERO | M_WAITOK);
  1070. }
  1071. #endif
  1072. #ifdef TCP_OFFLOAD
  1073. if (is_offload(sc)) {
  1074. s->nofldrxq = nports * iaq.nofldrxq;
  1075. if (num_vis > 1)
  1076. s->nofldrxq += nports * (num_vis - 1) * iaq.nofldrxq_vi;
  1077. s->neq += s->nofldrxq; /* free list */
  1078. s->niq += s->nofldrxq;
  1079. s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
  1080. M_CXGBE, M_ZERO | M_WAITOK);
  1081. }
  1082. #endif
  1083. #ifdef DEV_NETMAP
  1084. s->nnmrxq = 0;
  1085. s->nnmtxq = 0;
  1086. if (t4_native_netmap & NN_MAIN_VI) {
  1087. s->nnmrxq += nports * iaq.nnmrxq;
  1088. s->nnmtxq += nports * iaq.nnmtxq;
  1089. }
  1090. if (num_vis > 1 && t4_native_netmap & NN_EXTRA_VI) {
  1091. s->nnmrxq += nports * (num_vis - 1) * iaq.nnmrxq_vi;
  1092. s->nnmtxq += nports * (num_vis - 1) * iaq.nnmtxq_vi;
  1093. }
  1094. s->neq += s->nnmtxq + s->nnmrxq;
  1095. s->niq += s->nnmrxq;
  1096. s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
  1097. M_CXGBE, M_ZERO | M_WAITOK);
  1098. s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
  1099. M_CXGBE, M_ZERO | M_WAITOK);
  1100. #endif
  1101. s->ctrlq = malloc(nports * sizeof(struct sge_wrq), M_CXGBE,
  1102. M_ZERO | M_WAITOK);
  1103. s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
  1104. M_ZERO | M_WAITOK);
  1105. s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
  1106. M_ZERO | M_WAITOK);
  1107. s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
  1108. M_ZERO | M_WAITOK);
  1109. s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
  1110. M_ZERO | M_WAITOK);
  1111. sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
  1112. M_ZERO | M_WAITOK);
  1113. t4_init_l2t(sc, M_WAITOK);
  1114. t4_init_smt(sc, M_WAITOK);
  1115. t4_init_tx_sched(sc);
  1116. #ifdef RATELIMIT
  1117. t4_init_etid_table(sc);
  1118. #endif
  1119. #ifdef INET6
  1120. t4_init_clip_table(sc);
  1121. #endif
  1122. if (sc->vres.key.size != 0)
  1123. sc->key_map = vmem_create("T4TLS key map", sc->vres.key.start,
  1124. sc->vres.key.size, 32, 0, M_FIRSTFIT | M_WAITOK);
  1125. /*
  1126. * Second pass over the ports. This time we know the number of rx and
  1127. * tx queues that each port should get.
  1128. */
  1129. rqidx = tqidx = 0;
  1130. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  1131. ofld_tqidx = 0;
  1132. #endif
  1133. #ifdef TCP_OFFLOAD
  1134. ofld_rqidx = 0;
  1135. #endif
  1136. #ifdef DEV_NETMAP
  1137. nm_rqidx = nm_tqidx = 0;
  1138. #endif
  1139. for_each_port(sc, i) {
  1140. struct port_info *pi = sc->port[i];
  1141. struct vi_info *vi;
  1142. if (pi == NULL)
  1143. continue;
  1144. pi->nvi = num_vis;
  1145. for_each_vi(pi, j, vi) {
  1146. vi->pi = pi;
  1147. vi->qsize_rxq = t4_qsize_rxq;
  1148. vi->qsize_txq = t4_qsize_txq;
  1149. vi->first_rxq = rqidx;
  1150. vi->first_txq = tqidx;
  1151. vi->tmr_idx = t4_tmr_idx;
  1152. vi->pktc_idx = t4_pktc_idx;
  1153. vi->nrxq = j == 0 ? iaq.nrxq : iaq.nrxq_vi;
  1154. vi->ntxq = j == 0 ? iaq.ntxq : iaq.ntxq_vi;
  1155. rqidx += vi->nrxq;
  1156. tqidx += vi->ntxq;
  1157. if (j == 0 && vi->ntxq > 1)
  1158. vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0;
  1159. else
  1160. vi->rsrv_noflowq = 0;
  1161. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  1162. vi->first_ofld_txq = ofld_tqidx;
  1163. vi->nofldtxq = j == 0 ? iaq.nofldtxq : iaq.nofldtxq_vi;
  1164. ofld_tqidx += vi->nofldtxq;
  1165. #endif
  1166. #ifdef TCP_OFFLOAD
  1167. vi->ofld_tmr_idx = t4_tmr_idx_ofld;
  1168. vi->ofld_pktc_idx = t4_pktc_idx_ofld;
  1169. vi->first_ofld_rxq = ofld_rqidx;
  1170. vi->nofldrxq = j == 0 ? iaq.nofldrxq : iaq.nofldrxq_vi;
  1171. ofld_rqidx += vi->nofldrxq;
  1172. #endif
  1173. #ifdef DEV_NETMAP
  1174. vi->first_nm_rxq = nm_rqidx;
  1175. vi->first_nm_txq = nm_tqidx;
  1176. if (j == 0) {
  1177. vi->nnmrxq = iaq.nnmrxq;
  1178. vi->nnmtxq = iaq.nnmtxq;
  1179. } else {
  1180. vi->nnmrxq = iaq.nnmrxq_vi;
  1181. vi->nnmtxq = iaq.nnmtxq_vi;
  1182. }
  1183. nm_rqidx += vi->nnmrxq;
  1184. nm_tqidx += vi->nnmtxq;
  1185. #endif
  1186. }
  1187. }
  1188. rc = t4_setup_intr_handlers(sc);
  1189. if (rc != 0) {
  1190. device_printf(dev,
  1191. "failed to setup interrupt handlers: %d\n", rc);
  1192. goto done;
  1193. }
  1194. rc = bus_generic_probe(dev);
  1195. if (rc != 0) {
  1196. device_printf(dev, "failed to probe child drivers: %d\n", rc);
  1197. goto done;
  1198. }
  1199. /*
  1200. * Ensure thread-safe mailbox access (in debug builds).
  1201. *
  1202. * So far this was the only thread accessing the mailbox but various
  1203. * ifnets and sysctls are about to be created and their handlers/ioctls
  1204. * will access the mailbox from different threads.
  1205. */
  1206. sc->flags |= CHK_MBOX_ACCESS;
  1207. rc = bus_generic_attach(dev);
  1208. if (rc != 0) {
  1209. device_printf(dev,
  1210. "failed to attach all child ports: %d\n", rc);
  1211. goto done;
  1212. }
  1213. device_printf(dev,
  1214. "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
  1215. sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
  1216. sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
  1217. (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
  1218. sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
  1219. t4_set_desc(sc);
  1220. notify_siblings(dev, 0);
  1221. done:
  1222. if (rc != 0 && sc->cdev) {
  1223. /* cdev was created and so cxgbetool works; recover that way. */
  1224. device_printf(dev,
  1225. "error during attach, adapter is now in recovery mode.\n");
  1226. rc = 0;
  1227. }
  1228. if (rc != 0)
  1229. t4_detach_common(dev);
  1230. else
  1231. t4_sysctls(sc);
  1232. return (rc);
  1233. }
  1234. static int
  1235. t4_child_location_str(device_t bus, device_t dev, char *buf, size_t buflen)
  1236. {
  1237. struct adapter *sc;
  1238. struct port_info *pi;
  1239. int i;
  1240. sc = device_get_softc(bus);
  1241. buf[0] = '\0';
  1242. for_each_port(sc, i) {
  1243. pi = sc->port[i];
  1244. if (pi != NULL && pi->dev == dev) {
  1245. snprintf(buf, buflen, "port=%d", pi->port_id);
  1246. break;
  1247. }
  1248. }
  1249. return (0);
  1250. }
  1251. static int
  1252. t4_ready(device_t dev)
  1253. {
  1254. struct adapter *sc;
  1255. sc = device_get_softc(dev);
  1256. if (sc->flags & FW_OK)
  1257. return (0);
  1258. return (ENXIO);
  1259. }
  1260. static int
  1261. t4_read_port_device(device_t dev, int port, device_t *child)
  1262. {
  1263. struct adapter *sc;
  1264. struct port_info *pi;
  1265. sc = device_get_softc(dev);
  1266. if (port < 0 || port >= MAX_NPORTS)
  1267. return (EINVAL);
  1268. pi = sc->port[port];
  1269. if (pi == NULL || pi->dev == NULL)
  1270. return (ENXIO);
  1271. *child = pi->dev;
  1272. return (0);
  1273. }
  1274. static int
  1275. notify_siblings(device_t dev, int detaching)
  1276. {
  1277. device_t sibling;
  1278. int error, i;
  1279. error = 0;
  1280. for (i = 0; i < PCI_FUNCMAX; i++) {
  1281. if (i == pci_get_function(dev))
  1282. continue;
  1283. sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
  1284. pci_get_slot(dev), i);
  1285. if (sibling == NULL || !device_is_attached(sibling))
  1286. continue;
  1287. if (detaching)
  1288. error = T4_DETACH_CHILD(sibling);
  1289. else
  1290. (void)T4_ATTACH_CHILD(sibling);
  1291. if (error)
  1292. break;
  1293. }
  1294. return (error);
  1295. }
  1296. /*
  1297. * Idempotent
  1298. */
  1299. static int
  1300. t4_detach(device_t dev)
  1301. {
  1302. struct adapter *sc;
  1303. int rc;
  1304. sc = device_get_softc(dev);
  1305. rc = notify_siblings(dev, 1);
  1306. if (rc) {
  1307. device_printf(dev,
  1308. "failed to detach sibling devices: %d\n", rc);
  1309. return (rc);
  1310. }
  1311. return (t4_detach_common(dev));
  1312. }
  1313. int
  1314. t4_detach_common(device_t dev)
  1315. {
  1316. struct adapter *sc;
  1317. struct port_info *pi;
  1318. int i, rc;
  1319. sc = device_get_softc(dev);
  1320. if (sc->cdev) {
  1321. destroy_dev(sc->cdev);
  1322. sc->cdev = NULL;
  1323. }
  1324. sx_xlock(&t4_list_lock);
  1325. SLIST_REMOVE(&t4_list, sc, adapter, link);
  1326. sx_xunlock(&t4_list_lock);
  1327. sc->flags &= ~CHK_MBOX_ACCESS;
  1328. if (sc->flags & FULL_INIT_DONE) {
  1329. if (!(sc->flags & IS_VF))
  1330. t4_intr_disable(sc);
  1331. }
  1332. if (device_is_attached(dev)) {
  1333. rc = bus_generic_detach(dev);
  1334. if (rc) {
  1335. device_printf(dev,
  1336. "failed to detach child devices: %d\n", rc);
  1337. return (rc);
  1338. }
  1339. }
  1340. for (i = 0; i < sc->intr_count; i++)
  1341. t4_free_irq(sc, &sc->irq[i]);
  1342. if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
  1343. t4_free_tx_sched(sc);
  1344. for (i = 0; i < MAX_NPORTS; i++) {
  1345. pi = sc->port[i];
  1346. if (pi) {
  1347. t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
  1348. if (pi->dev)
  1349. device_delete_child(dev, pi->dev);
  1350. mtx_destroy(&pi->pi_lock);
  1351. free(pi->vi, M_CXGBE);
  1352. free(pi, M_CXGBE);
  1353. }
  1354. }
  1355. device_delete_children(dev);
  1356. if (sc->flags & FULL_INIT_DONE)
  1357. adapter_full_uninit(sc);
  1358. if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
  1359. t4_fw_bye(sc, sc->mbox);
  1360. if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
  1361. pci_release_msi(dev);
  1362. if (sc->regs_res)
  1363. bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
  1364. sc->regs_res);
  1365. if (sc->udbs_res)
  1366. bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
  1367. sc->udbs_res);
  1368. if (sc->msix_res)
  1369. bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
  1370. sc->msix_res);
  1371. if (sc->l2t)
  1372. t4_free_l2t(sc->l2t);
  1373. if (sc->smt)
  1374. t4_free_smt(sc->smt);
  1375. #ifdef RATELIMIT
  1376. t4_free_etid_table(sc);
  1377. #endif
  1378. if (sc->key_map)
  1379. vmem_destroy(sc->key_map);
  1380. #ifdef INET6
  1381. t4_destroy_clip_table(sc);
  1382. #endif
  1383. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  1384. free(sc->sge.ofld_txq, M_CXGBE);
  1385. #endif
  1386. #ifdef TCP_OFFLOAD
  1387. free(sc->sge.ofld_rxq, M_CXGBE);
  1388. #endif
  1389. #ifdef DEV_NETMAP
  1390. free(sc->sge.nm_rxq, M_CXGBE);
  1391. free(sc->sge.nm_txq, M_CXGBE);
  1392. #endif
  1393. free(sc->irq, M_CXGBE);
  1394. free(sc->sge.rxq, M_CXGBE);
  1395. free(sc->sge.txq, M_CXGBE);
  1396. free(sc->sge.ctrlq, M_CXGBE);
  1397. free(sc->sge.iqmap, M_CXGBE);
  1398. free(sc->sge.eqmap, M_CXGBE);
  1399. free(sc->tids.ftid_tab, M_CXGBE);
  1400. free(sc->tids.hpftid_tab, M_CXGBE);
  1401. free_hftid_hash(&sc->tids);
  1402. free(sc->tids.atid_tab, M_CXGBE);
  1403. free(sc->tids.tid_tab, M_CXGBE);
  1404. free(sc->tt.tls_rx_ports, M_CXGBE);
  1405. t4_destroy_dma_tag(sc);
  1406. callout_drain(&sc->sfl_callout);
  1407. if (mtx_initialized(&sc->tids.ftid_lock)) {
  1408. mtx_destroy(&sc->tids.ftid_lock);
  1409. cv_destroy(&sc->tids.ftid_cv);
  1410. }
  1411. if (mtx_initialized(&sc->tids.atid_lock))
  1412. mtx_destroy(&sc->tids.atid_lock);
  1413. if (mtx_initialized(&sc->ifp_lock))
  1414. mtx_destroy(&sc->ifp_lock);
  1415. if (rw_initialized(&sc->policy_lock)) {
  1416. rw_destroy(&sc->policy_lock);
  1417. #ifdef TCP_OFFLOAD
  1418. if (sc->policy != NULL)
  1419. free_offload_policy(sc->policy);
  1420. #endif
  1421. }
  1422. for (i = 0; i < NUM_MEMWIN; i++) {
  1423. struct memwin *mw = &sc->memwin[i];
  1424. if (rw_initialized(&mw->mw_lock))
  1425. rw_destroy(&mw->mw_lock);
  1426. }
  1427. mtx_destroy(&sc->sfl_lock);
  1428. mtx_destroy(&sc->reg_lock);
  1429. mtx_destroy(&sc->sc_lock);
  1430. bzero(sc, sizeof(*sc));
  1431. return (0);
  1432. }
  1433. static int
  1434. cxgbe_probe(device_t dev)
  1435. {
  1436. char buf[128];
  1437. struct port_info *pi = device_get_softc(dev);
  1438. snprintf(buf, sizeof(buf), "port %d", pi->port_id);
  1439. device_set_desc_copy(dev, buf);
  1440. return (BUS_PROBE_DEFAULT);
  1441. }
  1442. #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
  1443. IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
  1444. IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS | \
  1445. IFCAP_HWRXTSTMP)
  1446. #define T4_CAP_ENABLE (T4_CAP)
  1447. static int
  1448. cxgbe_vi_attach(device_t dev, struct vi_info *vi)
  1449. {
  1450. struct ifnet *ifp;
  1451. struct sbuf *sb;
  1452. vi->xact_addr_filt = -1;
  1453. callout_init(&vi->tick, 1);
  1454. /* Allocate an ifnet and set it up */
  1455. ifp = if_alloc(IFT_ETHER);
  1456. if (ifp == NULL) {
  1457. device_printf(dev, "Cannot allocate ifnet\n");
  1458. return (ENOMEM);
  1459. }
  1460. vi->ifp = ifp;
  1461. ifp->if_softc = vi;
  1462. if_initname(ifp, device_get_name(dev), device_get_unit(dev));
  1463. ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
  1464. ifp->if_init = cxgbe_init;
  1465. ifp->if_ioctl = cxgbe_ioctl;
  1466. ifp->if_transmit = cxgbe_transmit;
  1467. ifp->if_qflush = cxgbe_qflush;
  1468. ifp->if_get_counter = cxgbe_get_counter;
  1469. #ifdef RATELIMIT
  1470. ifp->if_snd_tag_alloc = cxgbe_snd_tag_alloc;
  1471. ifp->if_snd_tag_modify = cxgbe_snd_tag_modify;
  1472. ifp->if_snd_tag_query = cxgbe_snd_tag_query;
  1473. ifp->if_snd_tag_free = cxgbe_snd_tag_free;
  1474. #endif
  1475. ifp->if_capabilities = T4_CAP;
  1476. ifp->if_capenable = T4_CAP_ENABLE;
  1477. #ifdef TCP_OFFLOAD
  1478. if (vi->nofldrxq != 0)
  1479. ifp->if_capabilities |= IFCAP_TOE;
  1480. #endif
  1481. #ifdef RATELIMIT
  1482. if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0) {
  1483. ifp->if_capabilities |= IFCAP_TXRTLMT;
  1484. ifp->if_capenable |= IFCAP_TXRTLMT;
  1485. }
  1486. #endif
  1487. ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
  1488. CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
  1489. ifp->if_hw_tsomax = IP_MAXPACKET;
  1490. ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_TSO;
  1491. #ifdef RATELIMIT
  1492. if (is_ethoffload(vi->pi->adapter) && vi->nofldtxq != 0)
  1493. ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS_EO_TSO;
  1494. #endif
  1495. ifp->if_hw_tsomaxsegsize = 65536;
  1496. ether_ifattach(ifp, vi->hw_addr);
  1497. #ifdef DEV_NETMAP
  1498. if (vi->nnmrxq != 0)
  1499. cxgbe_nm_attach(vi);
  1500. #endif
  1501. sb = sbuf_new_auto();
  1502. sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
  1503. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  1504. switch (ifp->if_capabilities & (IFCAP_TOE | IFCAP_TXRTLMT)) {
  1505. case IFCAP_TOE:
  1506. sbuf_printf(sb, "; %d txq (TOE)", vi->nofldtxq);
  1507. break;
  1508. case IFCAP_TOE | IFCAP_TXRTLMT:
  1509. sbuf_printf(sb, "; %d txq (TOE/ETHOFLD)", vi->nofldtxq);
  1510. break;
  1511. case IFCAP_TXRTLMT:
  1512. sbuf_printf(sb, "; %d txq (ETHOFLD)", vi->nofldtxq);
  1513. break;
  1514. }
  1515. #endif
  1516. #ifdef TCP_OFFLOAD
  1517. if (ifp->if_capabilities & IFCAP_TOE)
  1518. sbuf_printf(sb, ", %d rxq (TOE)", vi->nofldrxq);
  1519. #endif
  1520. #ifdef DEV_NETMAP
  1521. if (ifp->if_capabilities & IFCAP_NETMAP)
  1522. sbuf_printf(sb, "; %d txq, %d rxq (netmap)",
  1523. vi->nnmtxq, vi->nnmrxq);
  1524. #endif
  1525. sbuf_finish(sb);
  1526. device_printf(dev, "%s\n", sbuf_data(sb));
  1527. sbuf_delete(sb);
  1528. vi_sysctls(vi);
  1529. return (0);
  1530. }
  1531. static int
  1532. cxgbe_attach(device_t dev)
  1533. {
  1534. struct port_info *pi = device_get_softc(dev);
  1535. struct adapter *sc = pi->adapter;
  1536. struct vi_info *vi;
  1537. int i, rc;
  1538. callout_init_mtx(&pi->tick, &pi->pi_lock, 0);
  1539. rc = cxgbe_vi_attach(dev, &pi->vi[0]);
  1540. if (rc)
  1541. return (rc);
  1542. for_each_vi(pi, i, vi) {
  1543. if (i == 0)
  1544. continue;
  1545. vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, -1);
  1546. if (vi->dev == NULL) {
  1547. device_printf(dev, "failed to add VI %d\n", i);
  1548. continue;
  1549. }
  1550. device_set_softc(vi->dev, vi);
  1551. }
  1552. cxgbe_sysctls(pi);
  1553. bus_generic_attach(dev);
  1554. return (0);
  1555. }
  1556. static void
  1557. cxgbe_vi_detach(struct vi_info *vi)
  1558. {
  1559. struct ifnet *ifp = vi->ifp;
  1560. ether_ifdetach(ifp);
  1561. /* Let detach proceed even if these fail. */
  1562. #ifdef DEV_NETMAP
  1563. if (ifp->if_capabilities & IFCAP_NETMAP)
  1564. cxgbe_nm_detach(vi);
  1565. #endif
  1566. cxgbe_uninit_synchronized(vi);
  1567. callout_drain(&vi->tick);
  1568. vi_full_uninit(vi);
  1569. if_free(vi->ifp);
  1570. vi->ifp = NULL;
  1571. }
  1572. static int
  1573. cxgbe_detach(device_t dev)
  1574. {
  1575. struct port_info *pi = device_get_softc(dev);
  1576. struct adapter *sc = pi->adapter;
  1577. int rc;
  1578. /* Detach the extra VIs first. */
  1579. rc = bus_generic_detach(dev);
  1580. if (rc)
  1581. return (rc);
  1582. device_delete_children(dev);
  1583. doom_vi(sc, &pi->vi[0]);
  1584. if (pi->flags & HAS_TRACEQ) {
  1585. sc->traceq = -1; /* cloner should not create ifnet */
  1586. t4_tracer_port_detach(sc);
  1587. }
  1588. cxgbe_vi_detach(&pi->vi[0]);
  1589. callout_drain(&pi->tick);
  1590. ifmedia_removeall(&pi->media);
  1591. end_synchronized_op(sc, 0);
  1592. return (0);
  1593. }
  1594. static void
  1595. cxgbe_init(void *arg)
  1596. {
  1597. struct vi_info *vi = arg;
  1598. struct adapter *sc = vi->pi->adapter;
  1599. if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0)
  1600. return;
  1601. cxgbe_init_synchronized(vi);
  1602. end_synchronized_op(sc, 0);
  1603. }
  1604. static int
  1605. cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
  1606. {
  1607. int rc = 0, mtu, flags;
  1608. struct vi_info *vi = ifp->if_softc;
  1609. struct port_info *pi = vi->pi;
  1610. struct adapter *sc = pi->adapter;
  1611. struct ifreq *ifr = (struct ifreq *)data;
  1612. uint32_t mask;
  1613. switch (cmd) {
  1614. case SIOCSIFMTU:
  1615. mtu = ifr->ifr_mtu;
  1616. if (mtu < ETHERMIN || mtu > MAX_MTU)
  1617. return (EINVAL);
  1618. rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu");
  1619. if (rc)
  1620. return (rc);
  1621. ifp->if_mtu = mtu;
  1622. if (vi->flags & VI_INIT_DONE) {
  1623. t4_update_fl_bufsize(ifp);
  1624. if (ifp->if_drv_flags & IFF_DRV_RUNNING)
  1625. rc = update_mac_settings(ifp, XGMAC_MTU);
  1626. }
  1627. end_synchronized_op(sc, 0);
  1628. break;
  1629. case SIOCSIFFLAGS:
  1630. rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4flg");
  1631. if (rc)
  1632. return (rc);
  1633. if (ifp->if_flags & IFF_UP) {
  1634. if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  1635. flags = vi->if_flags;
  1636. if ((ifp->if_flags ^ flags) &
  1637. (IFF_PROMISC | IFF_ALLMULTI)) {
  1638. rc = update_mac_settings(ifp,
  1639. XGMAC_PROMISC | XGMAC_ALLMULTI);
  1640. }
  1641. } else {
  1642. rc = cxgbe_init_synchronized(vi);
  1643. }
  1644. vi->if_flags = ifp->if_flags;
  1645. } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  1646. rc = cxgbe_uninit_synchronized(vi);
  1647. }
  1648. end_synchronized_op(sc, 0);
  1649. break;
  1650. case SIOCADDMULTI:
  1651. case SIOCDELMULTI:
  1652. rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4multi");
  1653. if (rc)
  1654. return (rc);
  1655. if (ifp->if_drv_flags & IFF_DRV_RUNNING)
  1656. rc = update_mac_settings(ifp, XGMAC_MCADDRS);
  1657. end_synchronized_op(sc, 0);
  1658. break;
  1659. case SIOCSIFCAP:
  1660. rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap");
  1661. if (rc)
  1662. return (rc);
  1663. mask = ifr->ifr_reqcap ^ ifp->if_capenable;
  1664. if (mask & IFCAP_TXCSUM) {
  1665. ifp->if_capenable ^= IFCAP_TXCSUM;
  1666. ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
  1667. if (IFCAP_TSO4 & ifp->if_capenable &&
  1668. !(IFCAP_TXCSUM & ifp->if_capenable)) {
  1669. mask &= ~IFCAP_TSO4;
  1670. ifp->if_capenable &= ~IFCAP_TSO4;
  1671. if_printf(ifp,
  1672. "tso4 disabled due to -txcsum.\n");
  1673. }
  1674. }
  1675. if (mask & IFCAP_TXCSUM_IPV6) {
  1676. ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
  1677. ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
  1678. if (IFCAP_TSO6 & ifp->if_capenable &&
  1679. !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
  1680. mask &= ~IFCAP_TSO6;
  1681. ifp->if_capenable &= ~IFCAP_TSO6;
  1682. if_printf(ifp,
  1683. "tso6 disabled due to -txcsum6.\n");
  1684. }
  1685. }
  1686. if (mask & IFCAP_RXCSUM)
  1687. ifp->if_capenable ^= IFCAP_RXCSUM;
  1688. if (mask & IFCAP_RXCSUM_IPV6)
  1689. ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
  1690. /*
  1691. * Note that we leave CSUM_TSO alone (it is always set). The
  1692. * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
  1693. * sending a TSO request our way, so it's sufficient to toggle
  1694. * IFCAP_TSOx only.
  1695. */
  1696. if (mask & IFCAP_TSO4) {
  1697. if (!(IFCAP_TSO4 & ifp->if_capenable) &&
  1698. !(IFCAP_TXCSUM & ifp->if_capenable)) {
  1699. if_printf(ifp, "enable txcsum first.\n");
  1700. rc = EAGAIN;
  1701. goto fail;
  1702. }
  1703. ifp->if_capenable ^= IFCAP_TSO4;
  1704. }
  1705. if (mask & IFCAP_TSO6) {
  1706. if (!(IFCAP_TSO6 & ifp->if_capenable) &&
  1707. !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
  1708. if_printf(ifp, "enable txcsum6 first.\n");
  1709. rc = EAGAIN;
  1710. goto fail;
  1711. }
  1712. ifp->if_capenable ^= IFCAP_TSO6;
  1713. }
  1714. if (mask & IFCAP_LRO) {
  1715. #if defined(INET) || defined(INET6)
  1716. int i;
  1717. struct sge_rxq *rxq;
  1718. ifp->if_capenable ^= IFCAP_LRO;
  1719. for_each_rxq(vi, i, rxq) {
  1720. if (ifp->if_capenable & IFCAP_LRO)
  1721. rxq->iq.flags |= IQ_LRO_ENABLED;
  1722. else
  1723. rxq->iq.flags &= ~IQ_LRO_ENABLED;
  1724. }
  1725. #endif
  1726. }
  1727. #ifdef TCP_OFFLOAD
  1728. if (mask & IFCAP_TOE) {
  1729. int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
  1730. rc = toe_capability(vi, enable);
  1731. if (rc != 0)
  1732. goto fail;
  1733. ifp->if_capenable ^= mask;
  1734. }
  1735. #endif
  1736. if (mask & IFCAP_VLAN_HWTAGGING) {
  1737. ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
  1738. if (ifp->if_drv_flags & IFF_DRV_RUNNING)
  1739. rc = update_mac_settings(ifp, XGMAC_VLANEX);
  1740. }
  1741. if (mask & IFCAP_VLAN_MTU) {
  1742. ifp->if_capenable ^= IFCAP_VLAN_MTU;
  1743. /* Need to find out how to disable auto-mtu-inflation */
  1744. }
  1745. if (mask & IFCAP_VLAN_HWTSO)
  1746. ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
  1747. if (mask & IFCAP_VLAN_HWCSUM)
  1748. ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
  1749. #ifdef RATELIMIT
  1750. if (mask & IFCAP_TXRTLMT)
  1751. ifp->if_capenable ^= IFCAP_TXRTLMT;
  1752. #endif
  1753. if (mask & IFCAP_HWRXTSTMP) {
  1754. int i;
  1755. struct sge_rxq *rxq;
  1756. ifp->if_capenable ^= IFCAP_HWRXTSTMP;
  1757. for_each_rxq(vi, i, rxq) {
  1758. if (ifp->if_capenable & IFCAP_HWRXTSTMP)
  1759. rxq->iq.flags |= IQ_RX_TIMESTAMP;
  1760. else
  1761. rxq->iq.flags &= ~IQ_RX_TIMESTAMP;
  1762. }
  1763. }
  1764. #ifdef VLAN_CAPABILITIES
  1765. VLAN_CAPABILITIES(ifp);
  1766. #endif
  1767. fail:
  1768. end_synchronized_op(sc, 0);
  1769. break;
  1770. case SIOCSIFMEDIA:
  1771. case SIOCGIFMEDIA:
  1772. case SIOCGIFXMEDIA:
  1773. ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
  1774. break;
  1775. case SIOCGI2C: {
  1776. struct ifi2creq i2c;
  1777. rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
  1778. if (rc != 0)
  1779. break;
  1780. if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
  1781. rc = EPERM;
  1782. break;
  1783. }
  1784. if (i2c.len > sizeof(i2c.data)) {
  1785. rc = EINVAL;
  1786. break;
  1787. }
  1788. rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c");
  1789. if (rc)
  1790. return (rc);
  1791. rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
  1792. i2c.offset, i2c.len, &i2c.data[0]);
  1793. end_synchronized_op(sc, 0);
  1794. if (rc == 0)
  1795. rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
  1796. break;
  1797. }
  1798. default:
  1799. rc = ether_ioctl(ifp, cmd, data);
  1800. }
  1801. return (rc);
  1802. }
  1803. static int
  1804. cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
  1805. {
  1806. struct vi_info *vi = ifp->if_softc;
  1807. struct port_info *pi = vi->pi;
  1808. struct adapter *sc = pi->adapter;
  1809. struct sge_txq *txq;
  1810. void *items[1];
  1811. int rc;
  1812. M_ASSERTPKTHDR(m);
  1813. MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */
  1814. if (__predict_false(pi->link_cfg.link_ok == false)) {
  1815. m_freem(m);
  1816. return (ENETDOWN);
  1817. }
  1818. rc = parse_pkt(sc, &m);
  1819. if (__predict_false(rc != 0)) {
  1820. MPASS(m == NULL); /* was freed already */
  1821. atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
  1822. return (rc);
  1823. }
  1824. #ifdef RATELIMIT
  1825. if (m->m_pkthdr.snd_tag != NULL) {
  1826. /* EAGAIN tells the stack we are not the correct interface. */
  1827. if (__predict_false(ifp != m->m_pkthdr.snd_tag->ifp)) {
  1828. m_freem(m);
  1829. return (EAGAIN);
  1830. }
  1831. return (ethofld_transmit(ifp, m));
  1832. }
  1833. #endif
  1834. /* Select a txq. */
  1835. txq = &sc->sge.txq[vi->first_txq];
  1836. if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
  1837. txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
  1838. vi->rsrv_noflowq);
  1839. items[0] = m;
  1840. rc = mp_ring_enqueue(txq->r, items, 1, 4096);
  1841. if (__predict_false(rc != 0))
  1842. m_freem(m);
  1843. return (rc);
  1844. }
  1845. static void
  1846. cxgbe_qflush(struct ifnet *ifp)
  1847. {
  1848. struct vi_info *vi = ifp->if_softc;
  1849. struct sge_txq *txq;
  1850. int i;
  1851. /* queues do not exist if !VI_INIT_DONE. */
  1852. if (vi->flags & VI_INIT_DONE) {
  1853. for_each_txq(vi, i, txq) {
  1854. TXQ_LOCK(txq);
  1855. txq->eq.flags |= EQ_QFLUSH;
  1856. TXQ_UNLOCK(txq);
  1857. while (!mp_ring_is_idle(txq->r)) {
  1858. mp_ring_check_drainage(txq->r, 0);
  1859. pause("qflush", 1);
  1860. }
  1861. TXQ_LOCK(txq);
  1862. txq->eq.flags &= ~EQ_QFLUSH;
  1863. TXQ_UNLOCK(txq);
  1864. }
  1865. }
  1866. if_qflush(ifp);
  1867. }
  1868. static uint64_t
  1869. vi_get_counter(struct ifnet *ifp, ift_counter c)
  1870. {
  1871. struct vi_info *vi = ifp->if_softc;
  1872. struct fw_vi_stats_vf *s = &vi->stats;
  1873. vi_refresh_stats(vi->pi->adapter, vi);
  1874. switch (c) {
  1875. case IFCOUNTER_IPACKETS:
  1876. return (s->rx_bcast_frames + s->rx_mcast_frames +
  1877. s->rx_ucast_frames);
  1878. case IFCOUNTER_IERRORS:
  1879. return (s->rx_err_frames);
  1880. case IFCOUNTER_OPACKETS:
  1881. return (s->tx_bcast_frames + s->tx_mcast_frames +
  1882. s->tx_ucast_frames + s->tx_offload_frames);
  1883. case IFCOUNTER_OERRORS:
  1884. return (s->tx_drop_frames);
  1885. case IFCOUNTER_IBYTES:
  1886. return (s->rx_bcast_bytes + s->rx_mcast_bytes +
  1887. s->rx_ucast_bytes);
  1888. case IFCOUNTER_OBYTES:
  1889. return (s->tx_bcast_bytes + s->tx_mcast_bytes +
  1890. s->tx_ucast_bytes + s->tx_offload_bytes);
  1891. case IFCOUNTER_IMCASTS:
  1892. return (s->rx_mcast_frames);
  1893. case IFCOUNTER_OMCASTS:
  1894. return (s->tx_mcast_frames);
  1895. case IFCOUNTER_OQDROPS: {
  1896. uint64_t drops;
  1897. drops = 0;
  1898. if (vi->flags & VI_INIT_DONE) {
  1899. int i;
  1900. struct sge_txq *txq;
  1901. for_each_txq(vi, i, txq)
  1902. drops += counter_u64_fetch(txq->r->drops);
  1903. }
  1904. return (drops);
  1905. }
  1906. default:
  1907. return (if_get_counter_default(ifp, c));
  1908. }
  1909. }
  1910. uint64_t
  1911. cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
  1912. {
  1913. struct vi_info *vi = ifp->if_softc;
  1914. struct port_info *pi = vi->pi;
  1915. struct adapter *sc = pi->adapter;
  1916. struct port_stats *s = &pi->stats;
  1917. if (pi->nvi > 1 || sc->flags & IS_VF)
  1918. return (vi_get_counter(ifp, c));
  1919. cxgbe_refresh_stats(sc, pi);
  1920. switch (c) {
  1921. case IFCOUNTER_IPACKETS:
  1922. return (s->rx_frames);
  1923. case IFCOUNTER_IERRORS:
  1924. return (s->rx_jabber + s->rx_runt + s->rx_too_long +
  1925. s->rx_fcs_err + s->rx_len_err);
  1926. case IFCOUNTER_OPACKETS:
  1927. return (s->tx_frames);
  1928. case IFCOUNTER_OERRORS:
  1929. return (s->tx_error_frames);
  1930. case IFCOUNTER_IBYTES:
  1931. return (s->rx_octets);
  1932. case IFCOUNTER_OBYTES:
  1933. return (s->tx_octets);
  1934. case IFCOUNTER_IMCASTS:
  1935. return (s->rx_mcast_frames);
  1936. case IFCOUNTER_OMCASTS:
  1937. return (s->tx_mcast_frames);
  1938. case IFCOUNTER_IQDROPS:
  1939. return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
  1940. s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
  1941. s->rx_trunc3 + pi->tnl_cong_drops);
  1942. case IFCOUNTER_OQDROPS: {
  1943. uint64_t drops;
  1944. drops = s->tx_drop;
  1945. if (vi->flags & VI_INIT_DONE) {
  1946. int i;
  1947. struct sge_txq *txq;
  1948. for_each_txq(vi, i, txq)
  1949. drops += counter_u64_fetch(txq->r->drops);
  1950. }
  1951. return (drops);
  1952. }
  1953. default:
  1954. return (if_get_counter_default(ifp, c));
  1955. }
  1956. }
  1957. /*
  1958. * The kernel picks a media from the list we had provided but we still validate
  1959. * the requeste.
  1960. */
  1961. int
  1962. cxgbe_media_change(struct ifnet *ifp)
  1963. {
  1964. struct vi_info *vi = ifp->if_softc;
  1965. struct port_info *pi = vi->pi;
  1966. struct ifmedia *ifm = &pi->media;
  1967. struct link_config *lc = &pi->link_cfg;
  1968. struct adapter *sc = pi->adapter;
  1969. int rc;
  1970. rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mec");
  1971. if (rc != 0)
  1972. return (rc);
  1973. PORT_LOCK(pi);
  1974. if (IFM_SUBTYPE(ifm->ifm_media) == IFM_AUTO) {
  1975. /* ifconfig .. media autoselect */
  1976. if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
  1977. rc = ENOTSUP; /* AN not supported by transceiver */
  1978. goto done;
  1979. }
  1980. lc->requested_aneg = AUTONEG_ENABLE;
  1981. lc->requested_speed = 0;
  1982. lc->requested_fc |= PAUSE_AUTONEG;
  1983. } else {
  1984. lc->requested_aneg = AUTONEG_DISABLE;
  1985. lc->requested_speed =
  1986. ifmedia_baudrate(ifm->ifm_media) / 1000000;
  1987. lc->requested_fc = 0;
  1988. if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE)
  1989. lc->requested_fc |= PAUSE_RX;
  1990. if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE)
  1991. lc->requested_fc |= PAUSE_TX;
  1992. }
  1993. if (pi->up_vis > 0) {
  1994. fixup_link_config(pi);
  1995. rc = apply_link_config(pi);
  1996. }
  1997. done:
  1998. PORT_UNLOCK(pi);
  1999. end_synchronized_op(sc, 0);
  2000. return (rc);
  2001. }
  2002. /*
  2003. * Base media word (without ETHER, pause, link active, etc.) for the port at the
  2004. * given speed.
  2005. */
  2006. static int
  2007. port_mword(struct port_info *pi, uint32_t speed)
  2008. {
  2009. MPASS(speed & M_FW_PORT_CAP32_SPEED);
  2010. MPASS(powerof2(speed));
  2011. switch(pi->port_type) {
  2012. case FW_PORT_TYPE_BT_SGMII:
  2013. case FW_PORT_TYPE_BT_XFI:
  2014. case FW_PORT_TYPE_BT_XAUI:
  2015. /* BaseT */
  2016. switch (speed) {
  2017. case FW_PORT_CAP32_SPEED_100M:
  2018. return (IFM_100_T);
  2019. case FW_PORT_CAP32_SPEED_1G:
  2020. return (IFM_1000_T);
  2021. case FW_PORT_CAP32_SPEED_10G:
  2022. return (IFM_10G_T);
  2023. }
  2024. break;
  2025. case FW_PORT_TYPE_KX4:
  2026. if (speed == FW_PORT_CAP32_SPEED_10G)
  2027. return (IFM_10G_KX4);
  2028. break;
  2029. case FW_PORT_TYPE_CX4:
  2030. if (speed == FW_PORT_CAP32_SPEED_10G)
  2031. return (IFM_10G_CX4);
  2032. break;
  2033. case FW_PORT_TYPE_KX:
  2034. if (speed == FW_PORT_CAP32_SPEED_1G)
  2035. return (IFM_1000_KX);
  2036. break;
  2037. case FW_PORT_TYPE_KR:
  2038. case FW_PORT_TYPE_BP_AP:
  2039. case FW_PORT_TYPE_BP4_AP:
  2040. case FW_PORT_TYPE_BP40_BA:
  2041. case FW_PORT_TYPE_KR4_100G:
  2042. case FW_PORT_TYPE_KR_SFP28:
  2043. case FW_PORT_TYPE_KR_XLAUI:
  2044. switch (speed) {
  2045. case FW_PORT_CAP32_SPEED_1G:
  2046. return (IFM_1000_KX);
  2047. case FW_PORT_CAP32_SPEED_10G:
  2048. return (IFM_10G_KR);
  2049. case FW_PORT_CAP32_SPEED_25G:
  2050. return (IFM_25G_KR);
  2051. case FW_PORT_CAP32_SPEED_40G:
  2052. return (IFM_40G_KR4);
  2053. case FW_PORT_CAP32_SPEED_50G:
  2054. return (IFM_50G_KR2);
  2055. case FW_PORT_CAP32_SPEED_100G:
  2056. return (IFM_100G_KR4);
  2057. }
  2058. break;
  2059. case FW_PORT_TYPE_FIBER_XFI:
  2060. case FW_PORT_TYPE_FIBER_XAUI:
  2061. case FW_PORT_TYPE_SFP:
  2062. case FW_PORT_TYPE_QSFP_10G:
  2063. case FW_PORT_TYPE_QSA:
  2064. case FW_PORT_TYPE_QSFP:
  2065. case FW_PORT_TYPE_CR4_QSFP:
  2066. case FW_PORT_TYPE_CR_QSFP:
  2067. case FW_PORT_TYPE_CR2_QSFP:
  2068. case FW_PORT_TYPE_SFP28:
  2069. /* Pluggable transceiver */
  2070. switch (pi->mod_type) {
  2071. case FW_PORT_MOD_TYPE_LR:
  2072. switch (speed) {
  2073. case FW_PORT_CAP32_SPEED_1G:
  2074. return (IFM_1000_LX);
  2075. case FW_PORT_CAP32_SPEED_10G:
  2076. return (IFM_10G_LR);
  2077. case FW_PORT_CAP32_SPEED_25G:
  2078. return (IFM_25G_LR);
  2079. case FW_PORT_CAP32_SPEED_40G:
  2080. return (IFM_40G_LR4);
  2081. case FW_PORT_CAP32_SPEED_50G:
  2082. return (IFM_50G_LR2);
  2083. case FW_PORT_CAP32_SPEED_100G:
  2084. return (IFM_100G_LR4);
  2085. }
  2086. break;
  2087. case FW_PORT_MOD_TYPE_SR:
  2088. switch (speed) {
  2089. case FW_PORT_CAP32_SPEED_1G:
  2090. return (IFM_1000_SX);
  2091. case FW_PORT_CAP32_SPEED_10G:
  2092. return (IFM_10G_SR);
  2093. case FW_PORT_CAP32_SPEED_25G:
  2094. return (IFM_25G_SR);
  2095. case FW_PORT_CAP32_SPEED_40G:
  2096. return (IFM_40G_SR4);
  2097. case FW_PORT_CAP32_SPEED_50G:
  2098. return (IFM_50G_SR2);
  2099. case FW_PORT_CAP32_SPEED_100G:
  2100. return (IFM_100G_SR4);
  2101. }
  2102. break;
  2103. case FW_PORT_MOD_TYPE_ER:
  2104. if (speed == FW_PORT_CAP32_SPEED_10G)
  2105. return (IFM_10G_ER);
  2106. break;
  2107. case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
  2108. case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
  2109. switch (speed) {
  2110. case FW_PORT_CAP32_SPEED_1G:
  2111. return (IFM_1000_CX);
  2112. case FW_PORT_CAP32_SPEED_10G:
  2113. return (IFM_10G_TWINAX);
  2114. case FW_PORT_CAP32_SPEED_25G:
  2115. return (IFM_25G_CR);
  2116. case FW_PORT_CAP32_SPEED_40G:
  2117. return (IFM_40G_CR4);
  2118. case FW_PORT_CAP32_SPEED_50G:
  2119. return (IFM_50G_CR2);
  2120. case FW_PORT_CAP32_SPEED_100G:
  2121. return (IFM_100G_CR4);
  2122. }
  2123. break;
  2124. case FW_PORT_MOD_TYPE_LRM:
  2125. if (speed == FW_PORT_CAP32_SPEED_10G)
  2126. return (IFM_10G_LRM);
  2127. break;
  2128. case FW_PORT_MOD_TYPE_NA:
  2129. MPASS(0); /* Not pluggable? */
  2130. /* fall throough */
  2131. case FW_PORT_MOD_TYPE_ERROR:
  2132. case FW_PORT_MOD_TYPE_UNKNOWN:
  2133. case FW_PORT_MOD_TYPE_NOTSUPPORTED:
  2134. break;
  2135. case FW_PORT_MOD_TYPE_NONE:
  2136. return (IFM_NONE);
  2137. }
  2138. break;
  2139. case FW_PORT_TYPE_NONE:
  2140. return (IFM_NONE);
  2141. }
  2142. return (IFM_UNKNOWN);
  2143. }
  2144. void
  2145. cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
  2146. {
  2147. struct vi_info *vi = ifp->if_softc;
  2148. struct port_info *pi = vi->pi;
  2149. struct adapter *sc = pi->adapter;
  2150. struct link_config *lc = &pi->link_cfg;
  2151. if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4med") != 0)
  2152. return;
  2153. PORT_LOCK(pi);
  2154. if (pi->up_vis == 0) {
  2155. /*
  2156. * If all the interfaces are administratively down the firmware
  2157. * does not report transceiver changes. Refresh port info here
  2158. * so that ifconfig displays accurate ifmedia at all times.
  2159. * This is the only reason we have a synchronized op in this
  2160. * function. Just PORT_LOCK would have been enough otherwise.
  2161. */
  2162. t4_update_port_info(pi);
  2163. build_medialist(pi);
  2164. }
  2165. /* ifm_status */
  2166. ifmr->ifm_status = IFM_AVALID;
  2167. if (lc->link_ok == false)
  2168. goto done;
  2169. ifmr->ifm_status |= IFM_ACTIVE;
  2170. /* ifm_active */
  2171. ifmr->ifm_active = IFM_ETHER | IFM_FDX;
  2172. ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE);
  2173. if (lc->fc & PAUSE_RX)
  2174. ifmr->ifm_active |= IFM_ETH_RXPAUSE;
  2175. if (lc->fc & PAUSE_TX)
  2176. ifmr->ifm_active |= IFM_ETH_TXPAUSE;
  2177. ifmr->ifm_active |= port_mword(pi, speed_to_fwcap(lc->speed));
  2178. done:
  2179. PORT_UNLOCK(pi);
  2180. end_synchronized_op(sc, 0);
  2181. }
  2182. static int
  2183. vcxgbe_probe(device_t dev)
  2184. {
  2185. char buf[128];
  2186. struct vi_info *vi = device_get_softc(dev);
  2187. snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id,
  2188. vi - vi->pi->vi);
  2189. device_set_desc_copy(dev, buf);
  2190. return (BUS_PROBE_DEFAULT);
  2191. }
  2192. static int
  2193. alloc_extra_vi(struct adapter *sc, struct port_info *pi, struct vi_info *vi)
  2194. {
  2195. int func, index, rc;
  2196. uint32_t param, val;
  2197. ASSERT_SYNCHRONIZED_OP(sc);
  2198. index = vi - pi->vi;
  2199. MPASS(index > 0); /* This function deals with _extra_ VIs only */
  2200. KASSERT(index < nitems(vi_mac_funcs),
  2201. ("%s: VI %s doesn't have a MAC func", __func__,
  2202. device_get_nameunit(vi->dev)));
  2203. func = vi_mac_funcs[index];
  2204. rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
  2205. vi->hw_addr, &vi->rss_size, &vi->vfvld, &vi->vin, func, 0);
  2206. if (rc < 0) {
  2207. device_printf(vi->dev, "failed to allocate virtual interface %d"
  2208. "for port %d: %d\n", index, pi->port_id, -rc);
  2209. return (-rc);
  2210. }
  2211. vi->viid = rc;
  2212. if (vi->rss_size == 1) {
  2213. /*
  2214. * This VI didn't get a slice of the RSS table. Reduce the
  2215. * number of VIs being created (hw.cxgbe.num_vis) or modify the
  2216. * configuration file (nvi, rssnvi for this PF) if this is a
  2217. * problem.
  2218. */
  2219. device_printf(vi->dev, "RSS table not available.\n");
  2220. vi->rss_base = 0xffff;
  2221. return (0);
  2222. }
  2223. param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
  2224. V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) |
  2225. V_FW_PARAMS_PARAM_YZ(vi->viid);
  2226. rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
  2227. if (rc)
  2228. vi->rss_base = 0xffff;
  2229. else {
  2230. MPASS((val >> 16) == vi->rss_size);
  2231. vi->rss_base = val & 0xffff;
  2232. }
  2233. return (0);
  2234. }
  2235. static int
  2236. vcxgbe_attach(device_t dev)
  2237. {
  2238. struct vi_info *vi;
  2239. struct port_info *pi;
  2240. struct adapter *sc;
  2241. int rc;
  2242. vi = device_get_softc(dev);
  2243. pi = vi->pi;
  2244. sc = pi->adapter;
  2245. rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4via");
  2246. if (rc)
  2247. return (rc);
  2248. rc = alloc_extra_vi(sc, pi, vi);
  2249. end_synchronized_op(sc, 0);
  2250. if (rc)
  2251. return (rc);
  2252. rc = cxgbe_vi_attach(dev, vi);
  2253. if (rc) {
  2254. t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
  2255. return (rc);
  2256. }
  2257. return (0);
  2258. }
  2259. static int
  2260. vcxgbe_detach(device_t dev)
  2261. {
  2262. struct vi_info *vi;
  2263. struct adapter *sc;
  2264. vi = device_get_softc(dev);
  2265. sc = vi->pi->adapter;
  2266. doom_vi(sc, vi);
  2267. cxgbe_vi_detach(vi);
  2268. t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
  2269. end_synchronized_op(sc, 0);
  2270. return (0);
  2271. }
  2272. static struct callout fatal_callout;
  2273. static void
  2274. delayed_panic(void *arg)
  2275. {
  2276. struct adapter *sc = arg;
  2277. panic("%s: panic on fatal error", device_get_nameunit(sc->dev));
  2278. }
  2279. void
  2280. t4_fatal_err(struct adapter *sc, bool fw_error)
  2281. {
  2282. t4_shutdown_adapter(sc);
  2283. log(LOG_ALERT, "%s: encountered fatal error, adapter stopped.\n",
  2284. device_get_nameunit(sc->dev));
  2285. if (fw_error) {
  2286. ASSERT_SYNCHRONIZED_OP(sc);
  2287. sc->flags |= ADAP_ERR;
  2288. } else {
  2289. ADAPTER_LOCK(sc);
  2290. sc->flags |= ADAP_ERR;
  2291. ADAPTER_UNLOCK(sc);
  2292. }
  2293. if (t4_panic_on_fatal_err) {
  2294. log(LOG_ALERT, "%s: panic on fatal error after 30s",
  2295. device_get_nameunit(sc->dev));
  2296. callout_reset(&fatal_callout, hz * 30, delayed_panic, sc);
  2297. }
  2298. }
  2299. void
  2300. t4_add_adapter(struct adapter *sc)
  2301. {
  2302. sx_xlock(&t4_list_lock);
  2303. SLIST_INSERT_HEAD(&t4_list, sc, link);
  2304. sx_xunlock(&t4_list_lock);
  2305. }
  2306. int
  2307. t4_map_bars_0_and_4(struct adapter *sc)
  2308. {
  2309. sc->regs_rid = PCIR_BAR(0);
  2310. sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
  2311. &sc->regs_rid, RF_ACTIVE);
  2312. if (sc->regs_res == NULL) {
  2313. device_printf(sc->dev, "cannot map registers.\n");
  2314. return (ENXIO);
  2315. }
  2316. sc->bt = rman_get_bustag(sc->regs_res);
  2317. sc->bh = rman_get_bushandle(sc->regs_res);
  2318. sc->mmio_len = rman_get_size(sc->regs_res);
  2319. setbit(&sc->doorbells, DOORBELL_KDB);
  2320. sc->msix_rid = PCIR_BAR(4);
  2321. sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
  2322. &sc->msix_rid, RF_ACTIVE);
  2323. if (sc->msix_res == NULL) {
  2324. device_printf(sc->dev, "cannot map MSI-X BAR.\n");
  2325. return (ENXIO);
  2326. }
  2327. return (0);
  2328. }
  2329. int
  2330. t4_map_bar_2(struct adapter *sc)
  2331. {
  2332. /*
  2333. * T4: only iWARP driver uses the userspace doorbells. There is no need
  2334. * to map it if RDMA is disabled.
  2335. */
  2336. if (is_t4(sc) && sc->rdmacaps == 0)
  2337. return (0);
  2338. sc->udbs_rid = PCIR_BAR(2);
  2339. sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
  2340. &sc->udbs_rid, RF_ACTIVE);
  2341. if (sc->udbs_res == NULL) {
  2342. device_printf(sc->dev, "cannot map doorbell BAR.\n");
  2343. return (ENXIO);
  2344. }
  2345. sc->udbs_base = rman_get_virtual(sc->udbs_res);
  2346. if (chip_id(sc) >= CHELSIO_T5) {
  2347. setbit(&sc->doorbells, DOORBELL_UDB);
  2348. #if defined(__i386__) || defined(__amd64__)
  2349. if (t5_write_combine) {
  2350. int rc, mode;
  2351. /*
  2352. * Enable write combining on BAR2. This is the
  2353. * userspace doorbell BAR and is split into 128B
  2354. * (UDBS_SEG_SIZE) doorbell regions, each associated
  2355. * with an egress queue. The first 64B has the doorbell
  2356. * and the second 64B can be used to submit a tx work
  2357. * request with an implicit doorbell.
  2358. */
  2359. rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
  2360. rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
  2361. if (rc == 0) {
  2362. clrbit(&sc->doorbells, DOORBELL_UDB);
  2363. setbit(&sc->doorbells, DOORBELL_WCWR);
  2364. setbit(&sc->doorbells, DOORBELL_UDBWC);
  2365. } else {
  2366. device_printf(sc->dev,
  2367. "couldn't enable write combining: %d\n",
  2368. rc);
  2369. }
  2370. mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0);
  2371. t4_write_reg(sc, A_SGE_STAT_CFG,
  2372. V_STATSOURCE_T5(7) | mode);
  2373. }
  2374. #endif
  2375. }
  2376. sc->iwt.wc_en = isset(&sc->doorbells, DOORBELL_UDBWC) ? 1 : 0;
  2377. return (0);
  2378. }
  2379. struct memwin_init {
  2380. uint32_t base;
  2381. uint32_t aperture;
  2382. };
  2383. static const struct memwin_init t4_memwin[NUM_MEMWIN] = {
  2384. { MEMWIN0_BASE, MEMWIN0_APERTURE },
  2385. { MEMWIN1_BASE, MEMWIN1_APERTURE },
  2386. { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
  2387. };
  2388. static const struct memwin_init t5_memwin[NUM_MEMWIN] = {
  2389. { MEMWIN0_BASE, MEMWIN0_APERTURE },
  2390. { MEMWIN1_BASE, MEMWIN1_APERTURE },
  2391. { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
  2392. };
  2393. static void
  2394. setup_memwin(struct adapter *sc)
  2395. {
  2396. const struct memwin_init *mw_init;
  2397. struct memwin *mw;
  2398. int i;
  2399. uint32_t bar0;
  2400. if (is_t4(sc)) {
  2401. /*
  2402. * Read low 32b of bar0 indirectly via the hardware backdoor
  2403. * mechanism. Works from within PCI passthrough environments
  2404. * too, where rman_get_start() can return a different value. We
  2405. * need to program the T4 memory window decoders with the actual
  2406. * addresses that will be coming across the PCIe link.
  2407. */
  2408. bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
  2409. bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
  2410. mw_init = &t4_memwin[0];
  2411. } else {
  2412. /* T5+ use the relative offset inside the PCIe BAR */
  2413. bar0 = 0;
  2414. mw_init = &t5_memwin[0];
  2415. }
  2416. for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) {
  2417. rw_init(&mw->mw_lock, "memory window access");
  2418. mw->mw_base = mw_init->base;
  2419. mw->mw_aperture = mw_init->aperture;
  2420. mw->mw_curpos = 0;
  2421. t4_write_reg(sc,
  2422. PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
  2423. (mw->mw_base + bar0) | V_BIR(0) |
  2424. V_WINDOW(ilog2(mw->mw_aperture) - 10));
  2425. rw_wlock(&mw->mw_lock);
  2426. position_memwin(sc, i, 0);
  2427. rw_wunlock(&mw->mw_lock);
  2428. }
  2429. /* flush */
  2430. t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
  2431. }
  2432. /*
  2433. * Positions the memory window at the given address in the card's address space.
  2434. * There are some alignment requirements and the actual position may be at an
  2435. * address prior to the requested address. mw->mw_curpos always has the actual
  2436. * position of the window.
  2437. */
  2438. static void
  2439. position_memwin(struct adapter *sc, int idx, uint32_t addr)
  2440. {
  2441. struct memwin *mw;
  2442. uint32_t pf;
  2443. uint32_t reg;
  2444. MPASS(idx >= 0 && idx < NUM_MEMWIN);
  2445. mw = &sc->memwin[idx];
  2446. rw_assert(&mw->mw_lock, RA_WLOCKED);
  2447. if (is_t4(sc)) {
  2448. pf = 0;
  2449. mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */
  2450. } else {
  2451. pf = V_PFNUM(sc->pf);
  2452. mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */
  2453. }
  2454. reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx);
  2455. t4_write_reg(sc, reg, mw->mw_curpos | pf);
  2456. t4_read_reg(sc, reg); /* flush */
  2457. }
  2458. int
  2459. rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val,
  2460. int len, int rw)
  2461. {
  2462. struct memwin *mw;
  2463. uint32_t mw_end, v;
  2464. MPASS(idx >= 0 && idx < NUM_MEMWIN);
  2465. /* Memory can only be accessed in naturally aligned 4 byte units */
  2466. if (addr & 3 || len & 3 || len <= 0)
  2467. return (EINVAL);
  2468. mw = &sc->memwin[idx];
  2469. while (len > 0) {
  2470. rw_rlock(&mw->mw_lock);
  2471. mw_end = mw->mw_curpos + mw->mw_aperture;
  2472. if (addr >= mw_end || addr < mw->mw_curpos) {
  2473. /* Will need to reposition the window */
  2474. if (!rw_try_upgrade(&mw->mw_lock)) {
  2475. rw_runlock(&mw->mw_lock);
  2476. rw_wlock(&mw->mw_lock);
  2477. }
  2478. rw_assert(&mw->mw_lock, RA_WLOCKED);
  2479. position_memwin(sc, idx, addr);
  2480. rw_downgrade(&mw->mw_lock);
  2481. mw_end = mw->mw_curpos + mw->mw_aperture;
  2482. }
  2483. rw_assert(&mw->mw_lock, RA_RLOCKED);
  2484. while (addr < mw_end && len > 0) {
  2485. if (rw == 0) {
  2486. v = t4_read_reg(sc, mw->mw_base + addr -
  2487. mw->mw_curpos);
  2488. *val++ = le32toh(v);
  2489. } else {
  2490. v = *val++;
  2491. t4_write_reg(sc, mw->mw_base + addr -
  2492. mw->mw_curpos, htole32(v));
  2493. }
  2494. addr += 4;
  2495. len -= 4;
  2496. }
  2497. rw_runlock(&mw->mw_lock);
  2498. }
  2499. return (0);
  2500. }
  2501. int
  2502. alloc_atid_tab(struct tid_info *t, int flags)
  2503. {
  2504. int i;
  2505. MPASS(t->natids > 0);
  2506. MPASS(t->atid_tab == NULL);
  2507. t->atid_tab = malloc(t->natids * sizeof(*t->atid_tab), M_CXGBE,
  2508. M_ZERO | flags);
  2509. if (t->atid_tab == NULL)
  2510. return (ENOMEM);
  2511. mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF);
  2512. t->afree = t->atid_tab;
  2513. t->atids_in_use = 0;
  2514. for (i = 1; i < t->natids; i++)
  2515. t->atid_tab[i - 1].next = &t->atid_tab[i];
  2516. t->atid_tab[t->natids - 1].next = NULL;
  2517. return (0);
  2518. }
  2519. void
  2520. free_atid_tab(struct tid_info *t)
  2521. {
  2522. KASSERT(t->atids_in_use == 0,
  2523. ("%s: %d atids still in use.", __func__, t->atids_in_use));
  2524. if (mtx_initialized(&t->atid_lock))
  2525. mtx_destroy(&t->atid_lock);
  2526. free(t->atid_tab, M_CXGBE);
  2527. t->atid_tab = NULL;
  2528. }
  2529. int
  2530. alloc_atid(struct adapter *sc, void *ctx)
  2531. {
  2532. struct tid_info *t = &sc->tids;
  2533. int atid = -1;
  2534. mtx_lock(&t->atid_lock);
  2535. if (t->afree) {
  2536. union aopen_entry *p = t->afree;
  2537. atid = p - t->atid_tab;
  2538. MPASS(atid <= M_TID_TID);
  2539. t->afree = p->next;
  2540. p->data = ctx;
  2541. t->atids_in_use++;
  2542. }
  2543. mtx_unlock(&t->atid_lock);
  2544. return (atid);
  2545. }
  2546. void *
  2547. lookup_atid(struct adapter *sc, int atid)
  2548. {
  2549. struct tid_info *t = &sc->tids;
  2550. return (t->atid_tab[atid].data);
  2551. }
  2552. void
  2553. free_atid(struct adapter *sc, int atid)
  2554. {
  2555. struct tid_info *t = &sc->tids;
  2556. union aopen_entry *p = &t->atid_tab[atid];
  2557. mtx_lock(&t->atid_lock);
  2558. p->next = t->afree;
  2559. t->afree = p;
  2560. t->atids_in_use--;
  2561. mtx_unlock(&t->atid_lock);
  2562. }
  2563. static void
  2564. queue_tid_release(struct adapter *sc, int tid)
  2565. {
  2566. CXGBE_UNIMPLEMENTED("deferred tid release");
  2567. }
  2568. void
  2569. release_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq)
  2570. {
  2571. struct wrqe *wr;
  2572. struct cpl_tid_release *req;
  2573. wr = alloc_wrqe(sizeof(*req), ctrlq);
  2574. if (wr == NULL) {
  2575. queue_tid_release(sc, tid); /* defer */
  2576. return;
  2577. }
  2578. req = wrtod(wr);
  2579. INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid);
  2580. t4_wrq_tx(sc, wr);
  2581. }
  2582. static int
  2583. t4_range_cmp(const void *a, const void *b)
  2584. {
  2585. return ((const struct t4_range *)a)->start -
  2586. ((const struct t4_range *)b)->start;
  2587. }
  2588. /*
  2589. * Verify that the memory range specified by the addr/len pair is valid within
  2590. * the card's address space.
  2591. */
  2592. static int
  2593. validate_mem_range(struct adapter *sc, uint32_t addr, uint32_t len)
  2594. {
  2595. struct t4_range mem_ranges[4], *r, *next;
  2596. uint32_t em, addr_len;
  2597. int i, n, remaining;
  2598. /* Memory can only be accessed in naturally aligned 4 byte units */
  2599. if (addr & 3 || len & 3 || len == 0)
  2600. return (EINVAL);
  2601. /* Enabled memories */
  2602. em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
  2603. r = &mem_ranges[0];
  2604. n = 0;
  2605. bzero(r, sizeof(mem_ranges));
  2606. if (em & F_EDRAM0_ENABLE) {
  2607. addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
  2608. r->size = G_EDRAM0_SIZE(addr_len) << 20;
  2609. if (r->size > 0) {
  2610. r->start = G_EDRAM0_BASE(addr_len) << 20;
  2611. if (addr >= r->start &&
  2612. addr + len <= r->start + r->size)
  2613. return (0);
  2614. r++;
  2615. n++;
  2616. }
  2617. }
  2618. if (em & F_EDRAM1_ENABLE) {
  2619. addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
  2620. r->size = G_EDRAM1_SIZE(addr_len) << 20;
  2621. if (r->size > 0) {
  2622. r->start = G_EDRAM1_BASE(addr_len) << 20;
  2623. if (addr >= r->start &&
  2624. addr + len <= r->start + r->size)
  2625. return (0);
  2626. r++;
  2627. n++;
  2628. }
  2629. }
  2630. if (em & F_EXT_MEM_ENABLE) {
  2631. addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
  2632. r->size = G_EXT_MEM_SIZE(addr_len) << 20;
  2633. if (r->size > 0) {
  2634. r->start = G_EXT_MEM_BASE(addr_len) << 20;
  2635. if (addr >= r->start &&
  2636. addr + len <= r->start + r->size)
  2637. return (0);
  2638. r++;
  2639. n++;
  2640. }
  2641. }
  2642. if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) {
  2643. addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
  2644. r->size = G_EXT_MEM1_SIZE(addr_len) << 20;
  2645. if (r->size > 0) {
  2646. r->start = G_EXT_MEM1_BASE(addr_len) << 20;
  2647. if (addr >= r->start &&
  2648. addr + len <= r->start + r->size)
  2649. return (0);
  2650. r++;
  2651. n++;
  2652. }
  2653. }
  2654. MPASS(n <= nitems(mem_ranges));
  2655. if (n > 1) {
  2656. /* Sort and merge the ranges. */
  2657. qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp);
  2658. /* Start from index 0 and examine the next n - 1 entries. */
  2659. r = &mem_ranges[0];
  2660. for (remaining = n - 1; remaining > 0; remaining--, r++) {
  2661. MPASS(r->size > 0); /* r is a valid entry. */
  2662. next = r + 1;
  2663. MPASS(next->size > 0); /* and so is the next one. */
  2664. while (r->start + r->size >= next->start) {
  2665. /* Merge the next one into the current entry. */
  2666. r->size = max(r->start + r->size,
  2667. next->start + next->size) - r->start;
  2668. n--; /* One fewer entry in total. */
  2669. if (--remaining == 0)
  2670. goto done; /* short circuit */
  2671. next++;
  2672. }
  2673. if (next != r + 1) {
  2674. /*
  2675. * Some entries were merged into r and next
  2676. * points to the first valid entry that couldn't
  2677. * be merged.
  2678. */
  2679. MPASS(next->size > 0); /* must be valid */
  2680. memcpy(r + 1, next, remaining * sizeof(*r));
  2681. #ifdef INVARIANTS
  2682. /*
  2683. * This so that the foo->size assertion in the
  2684. * next iteration of the loop do the right
  2685. * thing for entries that were pulled up and are
  2686. * no longer valid.
  2687. */
  2688. MPASS(n < nitems(mem_ranges));
  2689. bzero(&mem_ranges[n], (nitems(mem_ranges) - n) *
  2690. sizeof(struct t4_range));
  2691. #endif
  2692. }
  2693. }
  2694. done:
  2695. /* Done merging the ranges. */
  2696. MPASS(n > 0);
  2697. r = &mem_ranges[0];
  2698. for (i = 0; i < n; i++, r++) {
  2699. if (addr >= r->start &&
  2700. addr + len <= r->start + r->size)
  2701. return (0);
  2702. }
  2703. }
  2704. return (EFAULT);
  2705. }
  2706. static int
  2707. fwmtype_to_hwmtype(int mtype)
  2708. {
  2709. switch (mtype) {
  2710. case FW_MEMTYPE_EDC0:
  2711. return (MEM_EDC0);
  2712. case FW_MEMTYPE_EDC1:
  2713. return (MEM_EDC1);
  2714. case FW_MEMTYPE_EXTMEM:
  2715. return (MEM_MC0);
  2716. case FW_MEMTYPE_EXTMEM1:
  2717. return (MEM_MC1);
  2718. default:
  2719. panic("%s: cannot translate fw mtype %d.", __func__, mtype);
  2720. }
  2721. }
  2722. /*
  2723. * Verify that the memory range specified by the memtype/offset/len pair is
  2724. * valid and lies entirely within the memtype specified. The global address of
  2725. * the start of the range is returned in addr.
  2726. */
  2727. static int
  2728. validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, uint32_t len,
  2729. uint32_t *addr)
  2730. {
  2731. uint32_t em, addr_len, maddr;
  2732. /* Memory can only be accessed in naturally aligned 4 byte units */
  2733. if (off & 3 || len & 3 || len == 0)
  2734. return (EINVAL);
  2735. em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
  2736. switch (fwmtype_to_hwmtype(mtype)) {
  2737. case MEM_EDC0:
  2738. if (!(em & F_EDRAM0_ENABLE))
  2739. return (EINVAL);
  2740. addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
  2741. maddr = G_EDRAM0_BASE(addr_len) << 20;
  2742. break;
  2743. case MEM_EDC1:
  2744. if (!(em & F_EDRAM1_ENABLE))
  2745. return (EINVAL);
  2746. addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
  2747. maddr = G_EDRAM1_BASE(addr_len) << 20;
  2748. break;
  2749. case MEM_MC:
  2750. if (!(em & F_EXT_MEM_ENABLE))
  2751. return (EINVAL);
  2752. addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
  2753. maddr = G_EXT_MEM_BASE(addr_len) << 20;
  2754. break;
  2755. case MEM_MC1:
  2756. if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE))
  2757. return (EINVAL);
  2758. addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
  2759. maddr = G_EXT_MEM1_BASE(addr_len) << 20;
  2760. break;
  2761. default:
  2762. return (EINVAL);
  2763. }
  2764. *addr = maddr + off; /* global address */
  2765. return (validate_mem_range(sc, *addr, len));
  2766. }
  2767. static int
  2768. fixup_devlog_params(struct adapter *sc)
  2769. {
  2770. struct devlog_params *dparams = &sc->params.devlog;
  2771. int rc;
  2772. rc = validate_mt_off_len(sc, dparams->memtype, dparams->start,
  2773. dparams->size, &dparams->addr);
  2774. return (rc);
  2775. }
  2776. static void
  2777. update_nirq(struct intrs_and_queues *iaq, int nports)
  2778. {
  2779. iaq->nirq = T4_EXTRA_INTR;
  2780. iaq->nirq += nports * max(iaq->nrxq, iaq->nnmrxq);
  2781. iaq->nirq += nports * iaq->nofldrxq;
  2782. iaq->nirq += nports * (iaq->num_vis - 1) *
  2783. max(iaq->nrxq_vi, iaq->nnmrxq_vi);
  2784. iaq->nirq += nports * (iaq->num_vis - 1) * iaq->nofldrxq_vi;
  2785. }
  2786. /*
  2787. * Adjust requirements to fit the number of interrupts available.
  2788. */
  2789. static void
  2790. calculate_iaq(struct adapter *sc, struct intrs_and_queues *iaq, int itype,
  2791. int navail)
  2792. {
  2793. int old_nirq;
  2794. const int nports = sc->params.nports;
  2795. MPASS(nports > 0);
  2796. MPASS(navail > 0);
  2797. bzero(iaq, sizeof(*iaq));
  2798. iaq->intr_type = itype;
  2799. iaq->num_vis = t4_num_vis;
  2800. iaq->ntxq = t4_ntxq;
  2801. iaq->ntxq_vi = t4_ntxq_vi;
  2802. iaq->nrxq = t4_nrxq;
  2803. iaq->nrxq_vi = t4_nrxq_vi;
  2804. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  2805. if (is_offload(sc) || is_ethoffload(sc)) {
  2806. iaq->nofldtxq = t4_nofldtxq;
  2807. iaq->nofldtxq_vi = t4_nofldtxq_vi;
  2808. }
  2809. #endif
  2810. #ifdef TCP_OFFLOAD
  2811. if (is_offload(sc)) {
  2812. iaq->nofldrxq = t4_nofldrxq;
  2813. iaq->nofldrxq_vi = t4_nofldrxq_vi;
  2814. }
  2815. #endif
  2816. #ifdef DEV_NETMAP
  2817. if (t4_native_netmap & NN_MAIN_VI) {
  2818. iaq->nnmtxq = t4_nnmtxq;
  2819. iaq->nnmrxq = t4_nnmrxq;
  2820. }
  2821. if (t4_native_netmap & NN_EXTRA_VI) {
  2822. iaq->nnmtxq_vi = t4_nnmtxq_vi;
  2823. iaq->nnmrxq_vi = t4_nnmrxq_vi;
  2824. }
  2825. #endif
  2826. update_nirq(iaq, nports);
  2827. if (iaq->nirq <= navail &&
  2828. (itype != INTR_MSI || powerof2(iaq->nirq))) {
  2829. /*
  2830. * This is the normal case -- there are enough interrupts for
  2831. * everything.
  2832. */
  2833. goto done;
  2834. }
  2835. /*
  2836. * If extra VIs have been configured try reducing their count and see if
  2837. * that works.
  2838. */
  2839. while (iaq->num_vis > 1) {
  2840. iaq->num_vis--;
  2841. update_nirq(iaq, nports);
  2842. if (iaq->nirq <= navail &&
  2843. (itype != INTR_MSI || powerof2(iaq->nirq))) {
  2844. device_printf(sc->dev, "virtual interfaces per port "
  2845. "reduced to %d from %d. nrxq=%u, nofldrxq=%u, "
  2846. "nrxq_vi=%u nofldrxq_vi=%u, nnmrxq_vi=%u. "
  2847. "itype %d, navail %u, nirq %d.\n",
  2848. iaq->num_vis, t4_num_vis, iaq->nrxq, iaq->nofldrxq,
  2849. iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi,
  2850. itype, navail, iaq->nirq);
  2851. goto done;
  2852. }
  2853. }
  2854. /*
  2855. * Extra VIs will not be created. Log a message if they were requested.
  2856. */
  2857. MPASS(iaq->num_vis == 1);
  2858. iaq->ntxq_vi = iaq->nrxq_vi = 0;
  2859. iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0;
  2860. iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0;
  2861. if (iaq->num_vis != t4_num_vis) {
  2862. device_printf(sc->dev, "extra virtual interfaces disabled. "
  2863. "nrxq=%u, nofldrxq=%u, nrxq_vi=%u nofldrxq_vi=%u, "
  2864. "nnmrxq_vi=%u. itype %d, navail %u, nirq %d.\n",
  2865. iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi,
  2866. iaq->nnmrxq_vi, itype, navail, iaq->nirq);
  2867. }
  2868. /*
  2869. * Keep reducing the number of NIC rx queues to the next lower power of
  2870. * 2 (for even RSS distribution) and halving the TOE rx queues and see
  2871. * if that works.
  2872. */
  2873. do {
  2874. if (iaq->nrxq > 1) {
  2875. do {
  2876. iaq->nrxq--;
  2877. } while (!powerof2(iaq->nrxq));
  2878. if (iaq->nnmrxq > iaq->nrxq)
  2879. iaq->nnmrxq = iaq->nrxq;
  2880. }
  2881. if (iaq->nofldrxq > 1)
  2882. iaq->nofldrxq >>= 1;
  2883. old_nirq = iaq->nirq;
  2884. update_nirq(iaq, nports);
  2885. if (iaq->nirq <= navail &&
  2886. (itype != INTR_MSI || powerof2(iaq->nirq))) {
  2887. device_printf(sc->dev, "running with reduced number of "
  2888. "rx queues because of shortage of interrupts. "
  2889. "nrxq=%u, nofldrxq=%u. "
  2890. "itype %d, navail %u, nirq %d.\n", iaq->nrxq,
  2891. iaq->nofldrxq, itype, navail, iaq->nirq);
  2892. goto done;
  2893. }
  2894. } while (old_nirq != iaq->nirq);
  2895. /* One interrupt for everything. Ugh. */
  2896. device_printf(sc->dev, "running with minimal number of queues. "
  2897. "itype %d, navail %u.\n", itype, navail);
  2898. iaq->nirq = 1;
  2899. iaq->nrxq = 1;
  2900. iaq->ntxq = 1;
  2901. if (iaq->nofldrxq > 0) {
  2902. iaq->nofldrxq = 1;
  2903. iaq->nofldtxq = 1;
  2904. }
  2905. iaq->nnmtxq = 0;
  2906. iaq->nnmrxq = 0;
  2907. done:
  2908. MPASS(iaq->num_vis > 0);
  2909. if (iaq->num_vis > 1) {
  2910. MPASS(iaq->nrxq_vi > 0);
  2911. MPASS(iaq->ntxq_vi > 0);
  2912. }
  2913. MPASS(iaq->nirq > 0);
  2914. MPASS(iaq->nrxq > 0);
  2915. MPASS(iaq->ntxq > 0);
  2916. if (itype == INTR_MSI) {
  2917. MPASS(powerof2(iaq->nirq));
  2918. }
  2919. }
  2920. static int
  2921. cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq)
  2922. {
  2923. int rc, itype, navail, nalloc;
  2924. for (itype = INTR_MSIX; itype; itype >>= 1) {
  2925. if ((itype & t4_intr_types) == 0)
  2926. continue; /* not allowed */
  2927. if (itype == INTR_MSIX)
  2928. navail = pci_msix_count(sc->dev);
  2929. else if (itype == INTR_MSI)
  2930. navail = pci_msi_count(sc->dev);
  2931. else
  2932. navail = 1;
  2933. restart:
  2934. if (navail == 0)
  2935. continue;
  2936. calculate_iaq(sc, iaq, itype, navail);
  2937. nalloc = iaq->nirq;
  2938. rc = 0;
  2939. if (itype == INTR_MSIX)
  2940. rc = pci_alloc_msix(sc->dev, &nalloc);
  2941. else if (itype == INTR_MSI)
  2942. rc = pci_alloc_msi(sc->dev, &nalloc);
  2943. if (rc == 0 && nalloc > 0) {
  2944. if (nalloc == iaq->nirq)
  2945. return (0);
  2946. /*
  2947. * Didn't get the number requested. Use whatever number
  2948. * the kernel is willing to allocate.
  2949. */
  2950. device_printf(sc->dev, "fewer vectors than requested, "
  2951. "type=%d, req=%d, rcvd=%d; will downshift req.\n",
  2952. itype, iaq->nirq, nalloc);
  2953. pci_release_msi(sc->dev);
  2954. navail = nalloc;
  2955. goto restart;
  2956. }
  2957. device_printf(sc->dev,
  2958. "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
  2959. itype, rc, iaq->nirq, nalloc);
  2960. }
  2961. device_printf(sc->dev,
  2962. "failed to find a usable interrupt type. "
  2963. "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
  2964. pci_msix_count(sc->dev), pci_msi_count(sc->dev));
  2965. return (ENXIO);
  2966. }
  2967. #define FW_VERSION(chip) ( \
  2968. V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
  2969. V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
  2970. V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
  2971. V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
  2972. #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
  2973. /* Just enough of fw_hdr to cover all version info. */
  2974. struct fw_h {
  2975. __u8 ver;
  2976. __u8 chip;
  2977. __be16 len512;
  2978. __be32 fw_ver;
  2979. __be32 tp_microcode_ver;
  2980. __u8 intfver_nic;
  2981. __u8 intfver_vnic;
  2982. __u8 intfver_ofld;
  2983. __u8 intfver_ri;
  2984. __u8 intfver_iscsipdu;
  2985. __u8 intfver_iscsi;
  2986. __u8 intfver_fcoepdu;
  2987. __u8 intfver_fcoe;
  2988. };
  2989. /* Spot check a couple of fields. */
  2990. CTASSERT(offsetof(struct fw_h, fw_ver) == offsetof(struct fw_hdr, fw_ver));
  2991. CTASSERT(offsetof(struct fw_h, intfver_nic) == offsetof(struct fw_hdr, intfver_nic));
  2992. CTASSERT(offsetof(struct fw_h, intfver_fcoe) == offsetof(struct fw_hdr, intfver_fcoe));
  2993. struct fw_info {
  2994. uint8_t chip;
  2995. char *kld_name;
  2996. char *fw_mod_name;
  2997. struct fw_h fw_h;
  2998. } fw_info[] = {
  2999. {
  3000. .chip = CHELSIO_T4,
  3001. .kld_name = "t4fw_cfg",
  3002. .fw_mod_name = "t4fw",
  3003. .fw_h = {
  3004. .chip = FW_HDR_CHIP_T4,
  3005. .fw_ver = htobe32(FW_VERSION(T4)),
  3006. .intfver_nic = FW_INTFVER(T4, NIC),
  3007. .intfver_vnic = FW_INTFVER(T4, VNIC),
  3008. .intfver_ofld = FW_INTFVER(T4, OFLD),
  3009. .intfver_ri = FW_INTFVER(T4, RI),
  3010. .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
  3011. .intfver_iscsi = FW_INTFVER(T4, ISCSI),
  3012. .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
  3013. .intfver_fcoe = FW_INTFVER(T4, FCOE),
  3014. },
  3015. }, {
  3016. .chip = CHELSIO_T5,
  3017. .kld_name = "t5fw_cfg",
  3018. .fw_mod_name = "t5fw",
  3019. .fw_h = {
  3020. .chip = FW_HDR_CHIP_T5,
  3021. .fw_ver = htobe32(FW_VERSION(T5)),
  3022. .intfver_nic = FW_INTFVER(T5, NIC),
  3023. .intfver_vnic = FW_INTFVER(T5, VNIC),
  3024. .intfver_ofld = FW_INTFVER(T5, OFLD),
  3025. .intfver_ri = FW_INTFVER(T5, RI),
  3026. .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
  3027. .intfver_iscsi = FW_INTFVER(T5, ISCSI),
  3028. .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
  3029. .intfver_fcoe = FW_INTFVER(T5, FCOE),
  3030. },
  3031. }, {
  3032. .chip = CHELSIO_T6,
  3033. .kld_name = "t6fw_cfg",
  3034. .fw_mod_name = "t6fw",
  3035. .fw_h = {
  3036. .chip = FW_HDR_CHIP_T6,
  3037. .fw_ver = htobe32(FW_VERSION(T6)),
  3038. .intfver_nic = FW_INTFVER(T6, NIC),
  3039. .intfver_vnic = FW_INTFVER(T6, VNIC),
  3040. .intfver_ofld = FW_INTFVER(T6, OFLD),
  3041. .intfver_ri = FW_INTFVER(T6, RI),
  3042. .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU),
  3043. .intfver_iscsi = FW_INTFVER(T6, ISCSI),
  3044. .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU),
  3045. .intfver_fcoe = FW_INTFVER(T6, FCOE),
  3046. },
  3047. }
  3048. };
  3049. static struct fw_info *
  3050. find_fw_info(int chip)
  3051. {
  3052. int i;
  3053. for (i = 0; i < nitems(fw_info); i++) {
  3054. if (fw_info[i].chip == chip)
  3055. return (&fw_info[i]);
  3056. }
  3057. return (NULL);
  3058. }
  3059. /*
  3060. * Is the given firmware API compatible with the one the driver was compiled
  3061. * with?
  3062. */
  3063. static int
  3064. fw_compatible(const struct fw_h *hdr1, const struct fw_h *hdr2)
  3065. {
  3066. /* short circuit if it's the exact same firmware version */
  3067. if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
  3068. return (1);
  3069. /*
  3070. * XXX: Is this too conservative? Perhaps I should limit this to the
  3071. * features that are supported in the driver.
  3072. */
  3073. #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
  3074. if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
  3075. SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
  3076. SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
  3077. return (1);
  3078. #undef SAME_INTF
  3079. return (0);
  3080. }
  3081. static int
  3082. load_fw_module(struct adapter *sc, const struct firmware **dcfg,
  3083. const struct firmware **fw)
  3084. {
  3085. struct fw_info *fw_info;
  3086. *dcfg = NULL;
  3087. if (fw != NULL)
  3088. *fw = NULL;
  3089. fw_info = find_fw_info(chip_id(sc));
  3090. if (fw_info == NULL) {
  3091. device_printf(sc->dev,
  3092. "unable to look up firmware information for chip %d.\n",
  3093. chip_id(sc));
  3094. return (EINVAL);
  3095. }
  3096. *dcfg = firmware_get(fw_info->kld_name);
  3097. if (*dcfg != NULL) {
  3098. if (fw != NULL)
  3099. *fw = firmware_get(fw_info->fw_mod_name);
  3100. return (0);
  3101. }
  3102. return (ENOENT);
  3103. }
  3104. static void
  3105. unload_fw_module(struct adapter *sc, const struct firmware *dcfg,
  3106. const struct firmware *fw)
  3107. {
  3108. if (fw != NULL)
  3109. firmware_put(fw, FIRMWARE_UNLOAD);
  3110. if (dcfg != NULL)
  3111. firmware_put(dcfg, FIRMWARE_UNLOAD);
  3112. }
  3113. /*
  3114. * Return values:
  3115. * 0 means no firmware install attempted.
  3116. * ERESTART means a firmware install was attempted and was successful.
  3117. * +ve errno means a firmware install was attempted but failed.
  3118. */
  3119. static int
  3120. install_kld_firmware(struct adapter *sc, struct fw_h *card_fw,
  3121. const struct fw_h *drv_fw, const char *reason, int *already)
  3122. {
  3123. const struct firmware *cfg, *fw;
  3124. const uint32_t c = be32toh(card_fw->fw_ver);
  3125. uint32_t d, k;
  3126. int rc, fw_install;
  3127. struct fw_h bundled_fw;
  3128. bool load_attempted;
  3129. cfg = fw = NULL;
  3130. load_attempted = false;
  3131. fw_install = t4_fw_install < 0 ? -t4_fw_install : t4_fw_install;
  3132. memcpy(&bundled_fw, drv_fw, sizeof(bundled_fw));
  3133. if (t4_fw_install < 0) {
  3134. rc = load_fw_module(sc, &cfg, &fw);
  3135. if (rc != 0 || fw == NULL) {
  3136. device_printf(sc->dev,
  3137. "failed to load firmware module: %d. cfg %p, fw %p;"
  3138. " will use compiled-in firmware version for"
  3139. "hw.cxgbe.fw_install checks.\n",
  3140. rc, cfg, fw);
  3141. } else {
  3142. memcpy(&bundled_fw, fw->data, sizeof(bundled_fw));
  3143. }
  3144. load_attempted = true;
  3145. }
  3146. d = be32toh(bundled_fw.fw_ver);
  3147. if (reason != NULL)
  3148. goto install;
  3149. if ((sc->flags & FW_OK) == 0) {
  3150. if (c == 0xffffffff) {
  3151. reason = "missing";
  3152. goto install;
  3153. }
  3154. rc = 0;
  3155. goto done;
  3156. }
  3157. if (!fw_compatible(card_fw, &bundled_fw)) {
  3158. reason = "incompatible or unusable";
  3159. goto install;
  3160. }
  3161. if (d > c) {
  3162. reason = "older than the version bundled with this driver";
  3163. goto install;
  3164. }
  3165. if (fw_install == 2 && d != c) {
  3166. reason = "different than the version bundled with this driver";
  3167. goto install;
  3168. }
  3169. /* No reason to do anything to the firmware already on the card. */
  3170. rc = 0;
  3171. goto done;
  3172. install:
  3173. rc = 0;
  3174. if ((*already)++)
  3175. goto done;
  3176. if (fw_install == 0) {
  3177. device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
  3178. "but the driver is prohibited from installing a firmware "
  3179. "on the card.\n",
  3180. G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
  3181. G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
  3182. goto done;
  3183. }
  3184. /*
  3185. * We'll attempt to install a firmware. Load the module first (if it
  3186. * hasn't been loaded already).
  3187. */
  3188. if (!load_attempted) {
  3189. rc = load_fw_module(sc, &cfg, &fw);
  3190. if (rc != 0 || fw == NULL) {
  3191. device_printf(sc->dev,
  3192. "failed to load firmware module: %d. cfg %p, fw %p\n",
  3193. rc, cfg, fw);
  3194. /* carry on */
  3195. }
  3196. }
  3197. if (fw == NULL) {
  3198. device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
  3199. "but the driver cannot take corrective action because it "
  3200. "is unable to load the firmware module.\n",
  3201. G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
  3202. G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
  3203. rc = sc->flags & FW_OK ? 0 : ENOENT;
  3204. goto done;
  3205. }
  3206. k = be32toh(((const struct fw_hdr *)fw->data)->fw_ver);
  3207. if (k != d) {
  3208. MPASS(t4_fw_install > 0);
  3209. device_printf(sc->dev,
  3210. "firmware in KLD (%u.%u.%u.%u) is not what the driver was "
  3211. "expecting (%u.%u.%u.%u) and will not be used.\n",
  3212. G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
  3213. G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k),
  3214. G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
  3215. G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d));
  3216. rc = sc->flags & FW_OK ? 0 : EINVAL;
  3217. goto done;
  3218. }
  3219. device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
  3220. "installing firmware %u.%u.%u.%u on card.\n",
  3221. G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
  3222. G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
  3223. G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
  3224. G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d));
  3225. rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
  3226. if (rc != 0) {
  3227. device_printf(sc->dev, "failed to install firmware: %d\n", rc);
  3228. } else {
  3229. /* Installed successfully, update the cached header too. */
  3230. rc = ERESTART;
  3231. memcpy(card_fw, fw->data, sizeof(*card_fw));
  3232. }
  3233. done:
  3234. unload_fw_module(sc, cfg, fw);
  3235. return (rc);
  3236. }
  3237. /*
  3238. * Establish contact with the firmware and attempt to become the master driver.
  3239. *
  3240. * A firmware will be installed to the card if needed (if the driver is allowed
  3241. * to do so).
  3242. */
  3243. static int
  3244. contact_firmware(struct adapter *sc)
  3245. {
  3246. int rc, already = 0;
  3247. enum dev_state state;
  3248. struct fw_info *fw_info;
  3249. struct fw_hdr *card_fw; /* fw on the card */
  3250. const struct fw_h *drv_fw;
  3251. fw_info = find_fw_info(chip_id(sc));
  3252. if (fw_info == NULL) {
  3253. device_printf(sc->dev,
  3254. "unable to look up firmware information for chip %d.\n",
  3255. chip_id(sc));
  3256. return (EINVAL);
  3257. }
  3258. drv_fw = &fw_info->fw_h;
  3259. /* Read the header of the firmware on the card */
  3260. card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
  3261. restart:
  3262. rc = -t4_get_fw_hdr(sc, card_fw);
  3263. if (rc != 0) {
  3264. device_printf(sc->dev,
  3265. "unable to read firmware header from card's flash: %d\n",
  3266. rc);
  3267. goto done;
  3268. }
  3269. rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw, NULL,
  3270. &already);
  3271. if (rc == ERESTART)
  3272. goto restart;
  3273. if (rc != 0)
  3274. goto done;
  3275. rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
  3276. if (rc < 0 || state == DEV_STATE_ERR) {
  3277. rc = -rc;
  3278. device_printf(sc->dev,
  3279. "failed to connect to the firmware: %d, %d. "
  3280. "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW));
  3281. #if 0
  3282. if (install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw,
  3283. "not responding properly to HELLO", &already) == ERESTART)
  3284. goto restart;
  3285. #endif
  3286. goto done;
  3287. }
  3288. MPASS(be32toh(card_fw->flags) & FW_HDR_FLAGS_RESET_HALT);
  3289. sc->flags |= FW_OK; /* The firmware responded to the FW_HELLO. */
  3290. if (rc == sc->pf) {
  3291. sc->flags |= MASTER_PF;
  3292. rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw,
  3293. NULL, &already);
  3294. if (rc == ERESTART)
  3295. rc = 0;
  3296. else if (rc != 0)
  3297. goto done;
  3298. } else if (state == DEV_STATE_UNINIT) {
  3299. /*
  3300. * We didn't get to be the master so we definitely won't be
  3301. * configuring the chip. It's a bug if someone else hasn't
  3302. * configured it already.
  3303. */
  3304. device_printf(sc->dev, "couldn't be master(%d), "
  3305. "device not already initialized either(%d). "
  3306. "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW));
  3307. rc = EPROTO;
  3308. goto done;
  3309. } else {
  3310. /*
  3311. * Some other PF is the master and has configured the chip.
  3312. * This is allowed but untested.
  3313. */
  3314. device_printf(sc->dev, "PF%d is master, device state %d. "
  3315. "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW));
  3316. snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", rc);
  3317. sc->cfcsum = 0;
  3318. rc = 0;
  3319. }
  3320. done:
  3321. if (rc != 0 && sc->flags & FW_OK) {
  3322. t4_fw_bye(sc, sc->mbox);
  3323. sc->flags &= ~FW_OK;
  3324. }
  3325. free(card_fw, M_CXGBE);
  3326. return (rc);
  3327. }
  3328. static int
  3329. copy_cfg_file_to_card(struct adapter *sc, char *cfg_file,
  3330. uint32_t mtype, uint32_t moff)
  3331. {
  3332. struct fw_info *fw_info;
  3333. const struct firmware *dcfg, *rcfg = NULL;
  3334. const uint32_t *cfdata;
  3335. uint32_t cflen, addr;
  3336. int rc;
  3337. load_fw_module(sc, &dcfg, NULL);
  3338. /* Card specific interpretation of "default". */
  3339. if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
  3340. if (pci_get_device(sc->dev) == 0x440a)
  3341. snprintf(cfg_file, sizeof(t4_cfg_file), UWIRE_CF);
  3342. if (is_fpga(sc))
  3343. snprintf(cfg_file, sizeof(t4_cfg_file), FPGA_CF);
  3344. }
  3345. if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
  3346. if (dcfg == NULL) {
  3347. device_printf(sc->dev,
  3348. "KLD with default config is not available.\n");
  3349. rc = ENOENT;
  3350. goto done;
  3351. }
  3352. cfdata = dcfg->data;
  3353. cflen = dcfg->datasize & ~3;
  3354. } else {
  3355. char s[32];
  3356. fw_info = find_fw_info(chip_id(sc));
  3357. if (fw_info == NULL) {
  3358. device_printf(sc->dev,
  3359. "unable to look up firmware information for chip %d.\n",
  3360. chip_id(sc));
  3361. rc = EINVAL;
  3362. goto done;
  3363. }
  3364. snprintf(s, sizeof(s), "%s_%s", fw_info->kld_name, cfg_file);
  3365. rcfg = firmware_get(s);
  3366. if (rcfg == NULL) {
  3367. device_printf(sc->dev,
  3368. "unable to load module \"%s\" for configuration "
  3369. "profile \"%s\".\n", s, cfg_file);
  3370. rc = ENOENT;
  3371. goto done;
  3372. }
  3373. cfdata = rcfg->data;
  3374. cflen = rcfg->datasize & ~3;
  3375. }
  3376. if (cflen > FLASH_CFG_MAX_SIZE) {
  3377. device_printf(sc->dev,
  3378. "config file too long (%d, max allowed is %d).\n",
  3379. cflen, FLASH_CFG_MAX_SIZE);
  3380. rc = EINVAL;
  3381. goto done;
  3382. }
  3383. rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
  3384. if (rc != 0) {
  3385. device_printf(sc->dev,
  3386. "%s: addr (%d/0x%x) or len %d is not valid: %d.\n",
  3387. __func__, mtype, moff, cflen, rc);
  3388. rc = EINVAL;
  3389. goto done;
  3390. }
  3391. write_via_memwin(sc, 2, addr, cfdata, cflen);
  3392. done:
  3393. if (rcfg != NULL)
  3394. firmware_put(rcfg, FIRMWARE_UNLOAD);
  3395. unload_fw_module(sc, dcfg, NULL);
  3396. return (rc);
  3397. }
  3398. struct caps_allowed {
  3399. uint16_t nbmcaps;
  3400. uint16_t linkcaps;
  3401. uint16_t switchcaps;
  3402. uint16_t niccaps;
  3403. uint16_t toecaps;
  3404. uint16_t rdmacaps;
  3405. uint16_t cryptocaps;
  3406. uint16_t iscsicaps;
  3407. uint16_t fcoecaps;
  3408. };
  3409. #define FW_PARAM_DEV(param) \
  3410. (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
  3411. V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
  3412. #define FW_PARAM_PFVF(param) \
  3413. (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
  3414. V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
  3415. /*
  3416. * Provide a configuration profile to the firmware and have it initialize the
  3417. * chip accordingly. This may involve uploading a configuration file to the
  3418. * card.
  3419. */
  3420. static int
  3421. apply_cfg_and_initialize(struct adapter *sc, char *cfg_file,
  3422. const struct caps_allowed *caps_allowed)
  3423. {
  3424. int rc;
  3425. struct fw_caps_config_cmd caps;
  3426. uint32_t mtype, moff, finicsum, cfcsum, param, val;
  3427. rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
  3428. if (rc != 0) {
  3429. device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
  3430. return (rc);
  3431. }
  3432. bzero(&caps, sizeof(caps));
  3433. caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
  3434. F_FW_CMD_REQUEST | F_FW_CMD_READ);
  3435. if (strncmp(cfg_file, BUILTIN_CF, sizeof(t4_cfg_file)) == 0) {
  3436. mtype = 0;
  3437. moff = 0;
  3438. caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
  3439. } else if (strncmp(cfg_file, FLASH_CF, sizeof(t4_cfg_file)) == 0) {
  3440. mtype = FW_MEMTYPE_FLASH;
  3441. moff = t4_flash_cfg_addr(sc);
  3442. caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
  3443. V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
  3444. V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) |
  3445. FW_LEN16(caps));
  3446. } else {
  3447. /*
  3448. * Ask the firmware where it wants us to upload the config file.
  3449. */
  3450. param = FW_PARAM_DEV(CF);
  3451. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
  3452. if (rc != 0) {
  3453. /* No support for config file? Shouldn't happen. */
  3454. device_printf(sc->dev,
  3455. "failed to query config file location: %d.\n", rc);
  3456. goto done;
  3457. }
  3458. mtype = G_FW_PARAMS_PARAM_Y(val);
  3459. moff = G_FW_PARAMS_PARAM_Z(val) << 16;
  3460. caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
  3461. V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
  3462. V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) |
  3463. FW_LEN16(caps));
  3464. rc = copy_cfg_file_to_card(sc, cfg_file, mtype, moff);
  3465. if (rc != 0) {
  3466. device_printf(sc->dev,
  3467. "failed to upload config file to card: %d.\n", rc);
  3468. goto done;
  3469. }
  3470. }
  3471. rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
  3472. if (rc != 0) {
  3473. device_printf(sc->dev, "failed to pre-process config file: %d "
  3474. "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
  3475. goto done;
  3476. }
  3477. finicsum = be32toh(caps.finicsum);
  3478. cfcsum = be32toh(caps.cfcsum); /* actual */
  3479. if (finicsum != cfcsum) {
  3480. device_printf(sc->dev,
  3481. "WARNING: config file checksum mismatch: %08x %08x\n",
  3482. finicsum, cfcsum);
  3483. }
  3484. sc->cfcsum = cfcsum;
  3485. snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", cfg_file);
  3486. /*
  3487. * Let the firmware know what features will (not) be used so it can tune
  3488. * things accordingly.
  3489. */
  3490. #define LIMIT_CAPS(x) do { \
  3491. caps.x##caps &= htobe16(caps_allowed->x##caps); \
  3492. } while (0)
  3493. LIMIT_CAPS(nbm);
  3494. LIMIT_CAPS(link);
  3495. LIMIT_CAPS(switch);
  3496. LIMIT_CAPS(nic);
  3497. LIMIT_CAPS(toe);
  3498. LIMIT_CAPS(rdma);
  3499. LIMIT_CAPS(crypto);
  3500. LIMIT_CAPS(iscsi);
  3501. LIMIT_CAPS(fcoe);
  3502. #undef LIMIT_CAPS
  3503. if (caps.niccaps & htobe16(FW_CAPS_CONFIG_NIC_HASHFILTER)) {
  3504. /*
  3505. * TOE and hashfilters are mutually exclusive. It is a config
  3506. * file or firmware bug if both are reported as available. Try
  3507. * to cope with the situation in non-debug builds by disabling
  3508. * TOE.
  3509. */
  3510. MPASS(caps.toecaps == 0);
  3511. caps.toecaps = 0;
  3512. caps.rdmacaps = 0;
  3513. caps.iscsicaps = 0;
  3514. }
  3515. caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
  3516. F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
  3517. caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
  3518. rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
  3519. if (rc != 0) {
  3520. device_printf(sc->dev,
  3521. "failed to process config file: %d.\n", rc);
  3522. goto done;
  3523. }
  3524. t4_tweak_chip_settings(sc);
  3525. set_params__pre_init(sc);
  3526. /* get basic stuff going */
  3527. rc = -t4_fw_initialize(sc, sc->mbox);
  3528. if (rc != 0) {
  3529. device_printf(sc->dev, "fw_initialize failed: %d.\n", rc);
  3530. goto done;
  3531. }
  3532. done:
  3533. return (rc);
  3534. }
  3535. /*
  3536. * Partition chip resources for use between various PFs, VFs, etc.
  3537. */
  3538. static int
  3539. partition_resources(struct adapter *sc)
  3540. {
  3541. char cfg_file[sizeof(t4_cfg_file)];
  3542. struct caps_allowed caps_allowed;
  3543. int rc;
  3544. bool fallback;
  3545. /* Only the master driver gets to configure the chip resources. */
  3546. MPASS(sc->flags & MASTER_PF);
  3547. #define COPY_CAPS(x) do { \
  3548. caps_allowed.x##caps = t4_##x##caps_allowed; \
  3549. } while (0)
  3550. bzero(&caps_allowed, sizeof(caps_allowed));
  3551. COPY_CAPS(nbm);
  3552. COPY_CAPS(link);
  3553. COPY_CAPS(switch);
  3554. COPY_CAPS(nic);
  3555. COPY_CAPS(toe);
  3556. COPY_CAPS(rdma);
  3557. COPY_CAPS(crypto);
  3558. COPY_CAPS(iscsi);
  3559. COPY_CAPS(fcoe);
  3560. fallback = sc->debug_flags & DF_DISABLE_CFG_RETRY ? false : true;
  3561. snprintf(cfg_file, sizeof(cfg_file), "%s", t4_cfg_file);
  3562. retry:
  3563. rc = apply_cfg_and_initialize(sc, cfg_file, &caps_allowed);
  3564. if (rc != 0 && fallback) {
  3565. device_printf(sc->dev,
  3566. "failed (%d) to configure card with \"%s\" profile, "
  3567. "will fall back to a basic configuration and retry.\n",
  3568. rc, cfg_file);
  3569. snprintf(cfg_file, sizeof(cfg_file), "%s", BUILTIN_CF);
  3570. bzero(&caps_allowed, sizeof(caps_allowed));
  3571. COPY_CAPS(switch);
  3572. caps_allowed.niccaps = FW_CAPS_CONFIG_NIC;
  3573. fallback = false;
  3574. goto retry;
  3575. }
  3576. #undef COPY_CAPS
  3577. return (rc);
  3578. }
  3579. /*
  3580. * Retrieve parameters that are needed (or nice to have) very early.
  3581. */
  3582. static int
  3583. get_params__pre_init(struct adapter *sc)
  3584. {
  3585. int rc;
  3586. uint32_t param[2], val[2];
  3587. t4_get_version_info(sc);
  3588. snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
  3589. G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
  3590. G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
  3591. G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
  3592. G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
  3593. snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u",
  3594. G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers),
  3595. G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers),
  3596. G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers),
  3597. G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers));
  3598. snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u",
  3599. G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
  3600. G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
  3601. G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
  3602. G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
  3603. snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u",
  3604. G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers),
  3605. G_FW_HDR_FW_VER_MINOR(sc->params.er_vers),
  3606. G_FW_HDR_FW_VER_MICRO(sc->params.er_vers),
  3607. G_FW_HDR_FW_VER_BUILD(sc->params.er_vers));
  3608. param[0] = FW_PARAM_DEV(PORTVEC);
  3609. param[1] = FW_PARAM_DEV(CCLK);
  3610. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
  3611. if (rc != 0) {
  3612. device_printf(sc->dev,
  3613. "failed to query parameters (pre_init): %d.\n", rc);
  3614. return (rc);
  3615. }
  3616. sc->params.portvec = val[0];
  3617. sc->params.nports = bitcount32(val[0]);
  3618. sc->params.vpd.cclk = val[1];
  3619. /* Read device log parameters. */
  3620. rc = -t4_init_devlog_params(sc, 1);
  3621. if (rc == 0)
  3622. fixup_devlog_params(sc);
  3623. else {
  3624. device_printf(sc->dev,
  3625. "failed to get devlog parameters: %d.\n", rc);
  3626. rc = 0; /* devlog isn't critical for device operation */
  3627. }
  3628. return (rc);
  3629. }
  3630. /*
  3631. * Any params that need to be set before FW_INITIALIZE.
  3632. */
  3633. static int
  3634. set_params__pre_init(struct adapter *sc)
  3635. {
  3636. int rc = 0;
  3637. uint32_t param, val;
  3638. if (chip_id(sc) >= CHELSIO_T6) {
  3639. param = FW_PARAM_DEV(HPFILTER_REGION_SUPPORT);
  3640. val = 1;
  3641. rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
  3642. /* firmwares < 1.20.1.0 do not have this param. */
  3643. if (rc == FW_EINVAL &&
  3644. sc->params.fw_vers < FW_VERSION32(1, 20, 1, 0)) {
  3645. rc = 0;
  3646. }
  3647. if (rc != 0) {
  3648. device_printf(sc->dev,
  3649. "failed to enable high priority filters :%d.\n",
  3650. rc);
  3651. }
  3652. }
  3653. /* Enable opaque VIIDs with firmwares that support it. */
  3654. param = FW_PARAM_DEV(OPAQUE_VIID_SMT_EXTN);
  3655. val = 1;
  3656. rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
  3657. if (rc == 0 && val == 1)
  3658. sc->params.viid_smt_extn_support = true;
  3659. else
  3660. sc->params.viid_smt_extn_support = false;
  3661. return (rc);
  3662. }
  3663. /*
  3664. * Retrieve various parameters that are of interest to the driver. The device
  3665. * has been initialized by the firmware at this point.
  3666. */
  3667. static int
  3668. get_params__post_init(struct adapter *sc)
  3669. {
  3670. int rc;
  3671. uint32_t param[7], val[7];
  3672. struct fw_caps_config_cmd caps;
  3673. param[0] = FW_PARAM_PFVF(IQFLINT_START);
  3674. param[1] = FW_PARAM_PFVF(EQ_START);
  3675. param[2] = FW_PARAM_PFVF(FILTER_START);
  3676. param[3] = FW_PARAM_PFVF(FILTER_END);
  3677. param[4] = FW_PARAM_PFVF(L2T_START);
  3678. param[5] = FW_PARAM_PFVF(L2T_END);
  3679. param[6] = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
  3680. V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
  3681. V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD);
  3682. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 7, param, val);
  3683. if (rc != 0) {
  3684. device_printf(sc->dev,
  3685. "failed to query parameters (post_init): %d.\n", rc);
  3686. return (rc);
  3687. }
  3688. sc->sge.iq_start = val[0];
  3689. sc->sge.eq_start = val[1];
  3690. if ((int)val[3] > (int)val[2]) {
  3691. sc->tids.ftid_base = val[2];
  3692. sc->tids.ftid_end = val[3];
  3693. sc->tids.nftids = val[3] - val[2] + 1;
  3694. }
  3695. sc->vres.l2t.start = val[4];
  3696. sc->vres.l2t.size = val[5] - val[4] + 1;
  3697. KASSERT(sc->vres.l2t.size <= L2T_SIZE,
  3698. ("%s: L2 table size (%u) larger than expected (%u)",
  3699. __func__, sc->vres.l2t.size, L2T_SIZE));
  3700. sc->params.core_vdd = val[6];
  3701. if (chip_id(sc) >= CHELSIO_T6) {
  3702. sc->tids.tid_base = t4_read_reg(sc,
  3703. A_LE_DB_ACTIVE_TABLE_START_INDEX);
  3704. param[0] = FW_PARAM_PFVF(HPFILTER_START);
  3705. param[1] = FW_PARAM_PFVF(HPFILTER_END);
  3706. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
  3707. if (rc != 0) {
  3708. device_printf(sc->dev,
  3709. "failed to query hpfilter parameters: %d.\n", rc);
  3710. return (rc);
  3711. }
  3712. if ((int)val[1] > (int)val[0]) {
  3713. sc->tids.hpftid_base = val[0];
  3714. sc->tids.hpftid_end = val[1];
  3715. sc->tids.nhpftids = val[1] - val[0] + 1;
  3716. /*
  3717. * These should go off if the layout changes and the
  3718. * driver needs to catch up.
  3719. */
  3720. MPASS(sc->tids.hpftid_base == 0);
  3721. MPASS(sc->tids.tid_base == sc->tids.nhpftids);
  3722. }
  3723. }
  3724. /*
  3725. * MPSBGMAP is queried separately because only recent firmwares support
  3726. * it as a parameter and we don't want the compound query above to fail
  3727. * on older firmwares.
  3728. */
  3729. param[0] = FW_PARAM_DEV(MPSBGMAP);
  3730. val[0] = 0;
  3731. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
  3732. if (rc == 0)
  3733. sc->params.mps_bg_map = val[0];
  3734. else
  3735. sc->params.mps_bg_map = 0;
  3736. /*
  3737. * Determine whether the firmware supports the filter2 work request.
  3738. * This is queried separately for the same reason as MPSBGMAP above.
  3739. */
  3740. param[0] = FW_PARAM_DEV(FILTER2_WR);
  3741. val[0] = 0;
  3742. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
  3743. if (rc == 0)
  3744. sc->params.filter2_wr_support = val[0] != 0;
  3745. else
  3746. sc->params.filter2_wr_support = 0;
  3747. /*
  3748. * Find out whether we're allowed to use the ULPTX MEMWRITE DSGL.
  3749. * This is queried separately for the same reason as other params above.
  3750. */
  3751. param[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
  3752. val[0] = 0;
  3753. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
  3754. if (rc == 0)
  3755. sc->params.ulptx_memwrite_dsgl = val[0] != 0;
  3756. else
  3757. sc->params.ulptx_memwrite_dsgl = false;
  3758. /* FW_RI_FR_NSMR_TPTE_WR support */
  3759. param[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR);
  3760. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
  3761. if (rc == 0)
  3762. sc->params.fr_nsmr_tpte_wr_support = val[0] != 0;
  3763. else
  3764. sc->params.fr_nsmr_tpte_wr_support = false;
  3765. /* get capabilites */
  3766. bzero(&caps, sizeof(caps));
  3767. caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
  3768. F_FW_CMD_REQUEST | F_FW_CMD_READ);
  3769. caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
  3770. rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
  3771. if (rc != 0) {
  3772. device_printf(sc->dev,
  3773. "failed to get card capabilities: %d.\n", rc);
  3774. return (rc);
  3775. }
  3776. #define READ_CAPS(x) do { \
  3777. sc->x = htobe16(caps.x); \
  3778. } while (0)
  3779. READ_CAPS(nbmcaps);
  3780. READ_CAPS(linkcaps);
  3781. READ_CAPS(switchcaps);
  3782. READ_CAPS(niccaps);
  3783. READ_CAPS(toecaps);
  3784. READ_CAPS(rdmacaps);
  3785. READ_CAPS(cryptocaps);
  3786. READ_CAPS(iscsicaps);
  3787. READ_CAPS(fcoecaps);
  3788. if (sc->niccaps & FW_CAPS_CONFIG_NIC_HASHFILTER) {
  3789. MPASS(chip_id(sc) > CHELSIO_T4);
  3790. MPASS(sc->toecaps == 0);
  3791. sc->toecaps = 0;
  3792. param[0] = FW_PARAM_DEV(NTID);
  3793. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
  3794. if (rc != 0) {
  3795. device_printf(sc->dev,
  3796. "failed to query HASHFILTER parameters: %d.\n", rc);
  3797. return (rc);
  3798. }
  3799. sc->tids.ntids = val[0];
  3800. if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
  3801. MPASS(sc->tids.ntids >= sc->tids.nhpftids);
  3802. sc->tids.ntids -= sc->tids.nhpftids;
  3803. }
  3804. sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
  3805. sc->params.hash_filter = 1;
  3806. }
  3807. if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
  3808. param[0] = FW_PARAM_PFVF(ETHOFLD_START);
  3809. param[1] = FW_PARAM_PFVF(ETHOFLD_END);
  3810. param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
  3811. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
  3812. if (rc != 0) {
  3813. device_printf(sc->dev,
  3814. "failed to query NIC parameters: %d.\n", rc);
  3815. return (rc);
  3816. }
  3817. if ((int)val[1] > (int)val[0]) {
  3818. sc->tids.etid_base = val[0];
  3819. sc->tids.etid_end = val[1];
  3820. sc->tids.netids = val[1] - val[0] + 1;
  3821. sc->params.eo_wr_cred = val[2];
  3822. sc->params.ethoffload = 1;
  3823. }
  3824. }
  3825. if (sc->toecaps) {
  3826. /* query offload-related parameters */
  3827. param[0] = FW_PARAM_DEV(NTID);
  3828. param[1] = FW_PARAM_PFVF(SERVER_START);
  3829. param[2] = FW_PARAM_PFVF(SERVER_END);
  3830. param[3] = FW_PARAM_PFVF(TDDP_START);
  3831. param[4] = FW_PARAM_PFVF(TDDP_END);
  3832. param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
  3833. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
  3834. if (rc != 0) {
  3835. device_printf(sc->dev,
  3836. "failed to query TOE parameters: %d.\n", rc);
  3837. return (rc);
  3838. }
  3839. sc->tids.ntids = val[0];
  3840. if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
  3841. MPASS(sc->tids.ntids >= sc->tids.nhpftids);
  3842. sc->tids.ntids -= sc->tids.nhpftids;
  3843. }
  3844. sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
  3845. if ((int)val[2] > (int)val[1]) {
  3846. sc->tids.stid_base = val[1];
  3847. sc->tids.nstids = val[2] - val[1] + 1;
  3848. }
  3849. sc->vres.ddp.start = val[3];
  3850. sc->vres.ddp.size = val[4] - val[3] + 1;
  3851. sc->params.ofldq_wr_cred = val[5];
  3852. sc->params.offload = 1;
  3853. } else {
  3854. /*
  3855. * The firmware attempts memfree TOE configuration for -SO cards
  3856. * and will report toecaps=0 if it runs out of resources (this
  3857. * depends on the config file). It may not report 0 for other
  3858. * capabilities dependent on the TOE in this case. Set them to
  3859. * 0 here so that the driver doesn't bother tracking resources
  3860. * that will never be used.
  3861. */
  3862. sc->iscsicaps = 0;
  3863. sc->rdmacaps = 0;
  3864. }
  3865. if (sc->rdmacaps) {
  3866. param[0] = FW_PARAM_PFVF(STAG_START);
  3867. param[1] = FW_PARAM_PFVF(STAG_END);
  3868. param[2] = FW_PARAM_PFVF(RQ_START);
  3869. param[3] = FW_PARAM_PFVF(RQ_END);
  3870. param[4] = FW_PARAM_PFVF(PBL_START);
  3871. param[5] = FW_PARAM_PFVF(PBL_END);
  3872. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
  3873. if (rc != 0) {
  3874. device_printf(sc->dev,
  3875. "failed to query RDMA parameters(1): %d.\n", rc);
  3876. return (rc);
  3877. }
  3878. sc->vres.stag.start = val[0];
  3879. sc->vres.stag.size = val[1] - val[0] + 1;
  3880. sc->vres.rq.start = val[2];
  3881. sc->vres.rq.size = val[3] - val[2] + 1;
  3882. sc->vres.pbl.start = val[4];
  3883. sc->vres.pbl.size = val[5] - val[4] + 1;
  3884. param[0] = FW_PARAM_PFVF(SQRQ_START);
  3885. param[1] = FW_PARAM_PFVF(SQRQ_END);
  3886. param[2] = FW_PARAM_PFVF(CQ_START);
  3887. param[3] = FW_PARAM_PFVF(CQ_END);
  3888. param[4] = FW_PARAM_PFVF(OCQ_START);
  3889. param[5] = FW_PARAM_PFVF(OCQ_END);
  3890. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
  3891. if (rc != 0) {
  3892. device_printf(sc->dev,
  3893. "failed to query RDMA parameters(2): %d.\n", rc);
  3894. return (rc);
  3895. }
  3896. sc->vres.qp.start = val[0];
  3897. sc->vres.qp.size = val[1] - val[0] + 1;
  3898. sc->vres.cq.start = val[2];
  3899. sc->vres.cq.size = val[3] - val[2] + 1;
  3900. sc->vres.ocq.start = val[4];
  3901. sc->vres.ocq.size = val[5] - val[4] + 1;
  3902. param[0] = FW_PARAM_PFVF(SRQ_START);
  3903. param[1] = FW_PARAM_PFVF(SRQ_END);
  3904. param[2] = FW_PARAM_DEV(MAXORDIRD_QP);
  3905. param[3] = FW_PARAM_DEV(MAXIRD_ADAPTER);
  3906. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
  3907. if (rc != 0) {
  3908. device_printf(sc->dev,
  3909. "failed to query RDMA parameters(3): %d.\n", rc);
  3910. return (rc);
  3911. }
  3912. sc->vres.srq.start = val[0];
  3913. sc->vres.srq.size = val[1] - val[0] + 1;
  3914. sc->params.max_ordird_qp = val[2];
  3915. sc->params.max_ird_adapter = val[3];
  3916. }
  3917. if (sc->iscsicaps) {
  3918. param[0] = FW_PARAM_PFVF(ISCSI_START);
  3919. param[1] = FW_PARAM_PFVF(ISCSI_END);
  3920. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
  3921. if (rc != 0) {
  3922. device_printf(sc->dev,
  3923. "failed to query iSCSI parameters: %d.\n", rc);
  3924. return (rc);
  3925. }
  3926. sc->vres.iscsi.start = val[0];
  3927. sc->vres.iscsi.size = val[1] - val[0] + 1;
  3928. }
  3929. if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) {
  3930. param[0] = FW_PARAM_PFVF(TLS_START);
  3931. param[1] = FW_PARAM_PFVF(TLS_END);
  3932. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
  3933. if (rc != 0) {
  3934. device_printf(sc->dev,
  3935. "failed to query TLS parameters: %d.\n", rc);
  3936. return (rc);
  3937. }
  3938. sc->vres.key.start = val[0];
  3939. sc->vres.key.size = val[1] - val[0] + 1;
  3940. }
  3941. t4_init_sge_params(sc);
  3942. /*
  3943. * We've got the params we wanted to query via the firmware. Now grab
  3944. * some others directly from the chip.
  3945. */
  3946. rc = t4_read_chip_settings(sc);
  3947. return (rc);
  3948. }
  3949. static int
  3950. set_params__post_init(struct adapter *sc)
  3951. {
  3952. uint32_t param, val;
  3953. #ifdef TCP_OFFLOAD
  3954. int i, v, shift;
  3955. #endif
  3956. /* ask for encapsulated CPLs */
  3957. param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
  3958. val = 1;
  3959. (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
  3960. /* Enable 32b port caps if the firmware supports it. */
  3961. param = FW_PARAM_PFVF(PORT_CAPS32);
  3962. val = 1;
  3963. if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val) == 0)
  3964. sc->params.port_caps32 = 1;
  3965. /* Let filter + maskhash steer to a part of the VI's RSS region. */
  3966. val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1);
  3967. t4_set_reg_field(sc, A_TP_RSS_CONFIG_TNL, V_MASKFILTER(M_MASKFILTER),
  3968. V_MASKFILTER(val - 1));
  3969. #ifdef TCP_OFFLOAD
  3970. /*
  3971. * Override the TOE timers with user provided tunables. This is not the
  3972. * recommended way to change the timers (the firmware config file is) so
  3973. * these tunables are not documented.
  3974. *
  3975. * All the timer tunables are in microseconds.
  3976. */
  3977. if (t4_toe_keepalive_idle != 0) {
  3978. v = us_to_tcp_ticks(sc, t4_toe_keepalive_idle);
  3979. v &= M_KEEPALIVEIDLE;
  3980. t4_set_reg_field(sc, A_TP_KEEP_IDLE,
  3981. V_KEEPALIVEIDLE(M_KEEPALIVEIDLE), V_KEEPALIVEIDLE(v));
  3982. }
  3983. if (t4_toe_keepalive_interval != 0) {
  3984. v = us_to_tcp_ticks(sc, t4_toe_keepalive_interval);
  3985. v &= M_KEEPALIVEINTVL;
  3986. t4_set_reg_field(sc, A_TP_KEEP_INTVL,
  3987. V_KEEPALIVEINTVL(M_KEEPALIVEINTVL), V_KEEPALIVEINTVL(v));
  3988. }
  3989. if (t4_toe_keepalive_count != 0) {
  3990. v = t4_toe_keepalive_count & M_KEEPALIVEMAXR2;
  3991. t4_set_reg_field(sc, A_TP_SHIFT_CNT,
  3992. V_KEEPALIVEMAXR1(M_KEEPALIVEMAXR1) |
  3993. V_KEEPALIVEMAXR2(M_KEEPALIVEMAXR2),
  3994. V_KEEPALIVEMAXR1(1) | V_KEEPALIVEMAXR2(v));
  3995. }
  3996. if (t4_toe_rexmt_min != 0) {
  3997. v = us_to_tcp_ticks(sc, t4_toe_rexmt_min);
  3998. v &= M_RXTMIN;
  3999. t4_set_reg_field(sc, A_TP_RXT_MIN,
  4000. V_RXTMIN(M_RXTMIN), V_RXTMIN(v));
  4001. }
  4002. if (t4_toe_rexmt_max != 0) {
  4003. v = us_to_tcp_ticks(sc, t4_toe_rexmt_max);
  4004. v &= M_RXTMAX;
  4005. t4_set_reg_field(sc, A_TP_RXT_MAX,
  4006. V_RXTMAX(M_RXTMAX), V_RXTMAX(v));
  4007. }
  4008. if (t4_toe_rexmt_count != 0) {
  4009. v = t4_toe_rexmt_count & M_RXTSHIFTMAXR2;
  4010. t4_set_reg_field(sc, A_TP_SHIFT_CNT,
  4011. V_RXTSHIFTMAXR1(M_RXTSHIFTMAXR1) |
  4012. V_RXTSHIFTMAXR2(M_RXTSHIFTMAXR2),
  4013. V_RXTSHIFTMAXR1(1) | V_RXTSHIFTMAXR2(v));
  4014. }
  4015. for (i = 0; i < nitems(t4_toe_rexmt_backoff); i++) {
  4016. if (t4_toe_rexmt_backoff[i] != -1) {
  4017. v = t4_toe_rexmt_backoff[i] & M_TIMERBACKOFFINDEX0;
  4018. shift = (i & 3) << 3;
  4019. t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3),
  4020. M_TIMERBACKOFFINDEX0 << shift, v << shift);
  4021. }
  4022. }
  4023. #endif
  4024. return (0);
  4025. }
  4026. #undef FW_PARAM_PFVF
  4027. #undef FW_PARAM_DEV
  4028. static void
  4029. t4_set_desc(struct adapter *sc)
  4030. {
  4031. char buf[128];
  4032. struct adapter_params *p = &sc->params;
  4033. snprintf(buf, sizeof(buf), "Chelsio %s", p->vpd.id);
  4034. device_set_desc_copy(sc->dev, buf);
  4035. }
  4036. static inline void
  4037. ifmedia_add4(struct ifmedia *ifm, int m)
  4038. {
  4039. ifmedia_add(ifm, m, 0, NULL);
  4040. ifmedia_add(ifm, m | IFM_ETH_TXPAUSE, 0, NULL);
  4041. ifmedia_add(ifm, m | IFM_ETH_RXPAUSE, 0, NULL);
  4042. ifmedia_add(ifm, m | IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE, 0, NULL);
  4043. }
  4044. /*
  4045. * This is the selected media, which is not quite the same as the active media.
  4046. * The media line in ifconfig is "media: Ethernet selected (active)" if selected
  4047. * and active are not the same, and "media: Ethernet selected" otherwise.
  4048. */
  4049. static void
  4050. set_current_media(struct port_info *pi)
  4051. {
  4052. struct link_config *lc;
  4053. struct ifmedia *ifm;
  4054. int mword;
  4055. u_int speed;
  4056. PORT_LOCK_ASSERT_OWNED(pi);
  4057. /* Leave current media alone if it's already set to IFM_NONE. */
  4058. ifm = &pi->media;
  4059. if (ifm->ifm_cur != NULL &&
  4060. IFM_SUBTYPE(ifm->ifm_cur->ifm_media) == IFM_NONE)
  4061. return;
  4062. lc = &pi->link_cfg;
  4063. if (lc->requested_aneg != AUTONEG_DISABLE &&
  4064. lc->pcaps & FW_PORT_CAP32_ANEG) {
  4065. ifmedia_set(ifm, IFM_ETHER | IFM_AUTO);
  4066. return;
  4067. }
  4068. mword = IFM_ETHER | IFM_FDX;
  4069. if (lc->requested_fc & PAUSE_TX)
  4070. mword |= IFM_ETH_TXPAUSE;
  4071. if (lc->requested_fc & PAUSE_RX)
  4072. mword |= IFM_ETH_RXPAUSE;
  4073. if (lc->requested_speed == 0)
  4074. speed = port_top_speed(pi) * 1000; /* Gbps -> Mbps */
  4075. else
  4076. speed = lc->requested_speed;
  4077. mword |= port_mword(pi, speed_to_fwcap(speed));
  4078. ifmedia_set(ifm, mword);
  4079. }
  4080. /*
  4081. * Returns true if the ifmedia list for the port cannot change.
  4082. */
  4083. static bool
  4084. fixed_ifmedia(struct port_info *pi)
  4085. {
  4086. return (pi->port_type == FW_PORT_TYPE_BT_SGMII ||
  4087. pi->port_type == FW_PORT_TYPE_BT_XFI ||
  4088. pi->port_type == FW_PORT_TYPE_BT_XAUI ||
  4089. pi->port_type == FW_PORT_TYPE_KX4 ||
  4090. pi->port_type == FW_PORT_TYPE_KX ||
  4091. pi->port_type == FW_PORT_TYPE_KR ||
  4092. pi->port_type == FW_PORT_TYPE_BP_AP ||
  4093. pi->port_type == FW_PORT_TYPE_BP4_AP ||
  4094. pi->port_type == FW_PORT_TYPE_BP40_BA ||
  4095. pi->port_type == FW_PORT_TYPE_KR4_100G ||
  4096. pi->port_type == FW_PORT_TYPE_KR_SFP28 ||
  4097. pi->port_type == FW_PORT_TYPE_KR_XLAUI);
  4098. }
  4099. static void
  4100. build_medialist(struct port_info *pi)
  4101. {
  4102. uint32_t ss, speed;
  4103. int unknown, mword, bit;
  4104. struct link_config *lc;
  4105. struct ifmedia *ifm;
  4106. PORT_LOCK_ASSERT_OWNED(pi);
  4107. if (pi->flags & FIXED_IFMEDIA)
  4108. return;
  4109. /*
  4110. * Rebuild the ifmedia list.
  4111. */
  4112. ifm = &pi->media;
  4113. ifmedia_removeall(ifm);
  4114. lc = &pi->link_cfg;
  4115. ss = G_FW_PORT_CAP32_SPEED(lc->pcaps); /* Supported Speeds */
  4116. if (__predict_false(ss == 0)) { /* not supposed to happen. */
  4117. MPASS(ss != 0);
  4118. no_media:
  4119. MPASS(LIST_EMPTY(&ifm->ifm_list));
  4120. ifmedia_add(ifm, IFM_ETHER | IFM_NONE, 0, NULL);
  4121. ifmedia_set(ifm, IFM_ETHER | IFM_NONE);
  4122. return;
  4123. }
  4124. unknown = 0;
  4125. for (bit = S_FW_PORT_CAP32_SPEED; bit < fls(ss); bit++) {
  4126. speed = 1 << bit;
  4127. MPASS(speed & M_FW_PORT_CAP32_SPEED);
  4128. if (ss & speed) {
  4129. mword = port_mword(pi, speed);
  4130. if (mword == IFM_NONE) {
  4131. goto no_media;
  4132. } else if (mword == IFM_UNKNOWN)
  4133. unknown++;
  4134. else
  4135. ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | mword);
  4136. }
  4137. }
  4138. if (unknown > 0) /* Add one unknown for all unknown media types. */
  4139. ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | IFM_UNKNOWN);
  4140. if (lc->pcaps & FW_PORT_CAP32_ANEG)
  4141. ifmedia_add(ifm, IFM_ETHER | IFM_AUTO, 0, NULL);
  4142. set_current_media(pi);
  4143. }
  4144. /*
  4145. * Initialize the requested fields in the link config based on driver tunables.
  4146. */
  4147. static void
  4148. init_link_config(struct port_info *pi)
  4149. {
  4150. struct link_config *lc = &pi->link_cfg;
  4151. PORT_LOCK_ASSERT_OWNED(pi);
  4152. lc->requested_speed = 0;
  4153. if (t4_autoneg == 0)
  4154. lc->requested_aneg = AUTONEG_DISABLE;
  4155. else if (t4_autoneg == 1)
  4156. lc->requested_aneg = AUTONEG_ENABLE;
  4157. else
  4158. lc->requested_aneg = AUTONEG_AUTO;
  4159. lc->requested_fc = t4_pause_settings & (PAUSE_TX | PAUSE_RX |
  4160. PAUSE_AUTONEG);
  4161. if (t4_fec & FEC_AUTO)
  4162. lc->requested_fec = FEC_AUTO;
  4163. else if (t4_fec == 0)
  4164. lc->requested_fec = FEC_NONE;
  4165. else {
  4166. /* -1 is handled by the FEC_AUTO block above and not here. */
  4167. lc->requested_fec = t4_fec &
  4168. (FEC_RS | FEC_BASER_RS | FEC_NONE | FEC_MODULE);
  4169. if (lc->requested_fec == 0)
  4170. lc->requested_fec = FEC_AUTO;
  4171. }
  4172. }
  4173. /*
  4174. * Makes sure that all requested settings comply with what's supported by the
  4175. * port. Returns the number of settings that were invalid and had to be fixed.
  4176. */
  4177. static int
  4178. fixup_link_config(struct port_info *pi)
  4179. {
  4180. int n = 0;
  4181. struct link_config *lc = &pi->link_cfg;
  4182. uint32_t fwspeed;
  4183. PORT_LOCK_ASSERT_OWNED(pi);
  4184. /* Speed (when not autonegotiating) */
  4185. if (lc->requested_speed != 0) {
  4186. fwspeed = speed_to_fwcap(lc->requested_speed);
  4187. if ((fwspeed & lc->pcaps) == 0) {
  4188. n++;
  4189. lc->requested_speed = 0;
  4190. }
  4191. }
  4192. /* Link autonegotiation */
  4193. MPASS(lc->requested_aneg == AUTONEG_ENABLE ||
  4194. lc->requested_aneg == AUTONEG_DISABLE ||
  4195. lc->requested_aneg == AUTONEG_AUTO);
  4196. if (lc->requested_aneg == AUTONEG_ENABLE &&
  4197. !(lc->pcaps & FW_PORT_CAP32_ANEG)) {
  4198. n++;
  4199. lc->requested_aneg = AUTONEG_AUTO;
  4200. }
  4201. /* Flow control */
  4202. MPASS((lc->requested_fc & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) == 0);
  4203. if (lc->requested_fc & PAUSE_TX &&
  4204. !(lc->pcaps & FW_PORT_CAP32_FC_TX)) {
  4205. n++;
  4206. lc->requested_fc &= ~PAUSE_TX;
  4207. }
  4208. if (lc->requested_fc & PAUSE_RX &&
  4209. !(lc->pcaps & FW_PORT_CAP32_FC_RX)) {
  4210. n++;
  4211. lc->requested_fc &= ~PAUSE_RX;
  4212. }
  4213. if (!(lc->requested_fc & PAUSE_AUTONEG) &&
  4214. !(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE)) {
  4215. n++;
  4216. lc->requested_fc |= PAUSE_AUTONEG;
  4217. }
  4218. /* FEC */
  4219. if ((lc->requested_fec & FEC_RS &&
  4220. !(lc->pcaps & FW_PORT_CAP32_FEC_RS)) ||
  4221. (lc->requested_fec & FEC_BASER_RS &&
  4222. !(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS))) {
  4223. n++;
  4224. lc->requested_fec = FEC_AUTO;
  4225. }
  4226. return (n);
  4227. }
  4228. /*
  4229. * Apply the requested L1 settings, which are expected to be valid, to the
  4230. * hardware.
  4231. */
  4232. static int
  4233. apply_link_config(struct port_info *pi)
  4234. {
  4235. struct adapter *sc = pi->adapter;
  4236. struct link_config *lc = &pi->link_cfg;
  4237. int rc;
  4238. #ifdef INVARIANTS
  4239. ASSERT_SYNCHRONIZED_OP(sc);
  4240. PORT_LOCK_ASSERT_OWNED(pi);
  4241. if (lc->requested_aneg == AUTONEG_ENABLE)
  4242. MPASS(lc->pcaps & FW_PORT_CAP32_ANEG);
  4243. if (!(lc->requested_fc & PAUSE_AUTONEG))
  4244. MPASS(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE);
  4245. if (lc->requested_fc & PAUSE_TX)
  4246. MPASS(lc->pcaps & FW_PORT_CAP32_FC_TX);
  4247. if (lc->requested_fc & PAUSE_RX)
  4248. MPASS(lc->pcaps & FW_PORT_CAP32_FC_RX);
  4249. if (lc->requested_fec & FEC_RS)
  4250. MPASS(lc->pcaps & FW_PORT_CAP32_FEC_RS);
  4251. if (lc->requested_fec & FEC_BASER_RS)
  4252. MPASS(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS);
  4253. #endif
  4254. rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
  4255. if (rc != 0) {
  4256. /* Don't complain if the VF driver gets back an EPERM. */
  4257. if (!(sc->flags & IS_VF) || rc != FW_EPERM)
  4258. device_printf(pi->dev, "l1cfg failed: %d\n", rc);
  4259. } else {
  4260. /*
  4261. * An L1_CFG will almost always result in a link-change event if
  4262. * the link is up, and the driver will refresh the actual
  4263. * fec/fc/etc. when the notification is processed. If the link
  4264. * is down then the actual settings are meaningless.
  4265. *
  4266. * This takes care of the case where a change in the L1 settings
  4267. * may not result in a notification.
  4268. */
  4269. if (lc->link_ok && !(lc->requested_fc & PAUSE_AUTONEG))
  4270. lc->fc = lc->requested_fc & (PAUSE_TX | PAUSE_RX);
  4271. }
  4272. return (rc);
  4273. }
  4274. #define FW_MAC_EXACT_CHUNK 7
  4275. /*
  4276. * Program the port's XGMAC based on parameters in ifnet. The caller also
  4277. * indicates which parameters should be programmed (the rest are left alone).
  4278. */
  4279. int
  4280. update_mac_settings(struct ifnet *ifp, int flags)
  4281. {
  4282. int rc = 0;
  4283. struct vi_info *vi = ifp->if_softc;
  4284. struct port_info *pi = vi->pi;
  4285. struct adapter *sc = pi->adapter;
  4286. int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
  4287. ASSERT_SYNCHRONIZED_OP(sc);
  4288. KASSERT(flags, ("%s: not told what to update.", __func__));
  4289. if (flags & XGMAC_MTU)
  4290. mtu = ifp->if_mtu;
  4291. if (flags & XGMAC_PROMISC)
  4292. promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
  4293. if (flags & XGMAC_ALLMULTI)
  4294. allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
  4295. if (flags & XGMAC_VLANEX)
  4296. vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
  4297. if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
  4298. rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
  4299. allmulti, 1, vlanex, false);
  4300. if (rc) {
  4301. if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
  4302. rc);
  4303. return (rc);
  4304. }
  4305. }
  4306. if (flags & XGMAC_UCADDR) {
  4307. uint8_t ucaddr[ETHER_ADDR_LEN];
  4308. bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
  4309. rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
  4310. ucaddr, true, &vi->smt_idx);
  4311. if (rc < 0) {
  4312. rc = -rc;
  4313. if_printf(ifp, "change_mac failed: %d\n", rc);
  4314. return (rc);
  4315. } else {
  4316. vi->xact_addr_filt = rc;
  4317. rc = 0;
  4318. }
  4319. }
  4320. if (flags & XGMAC_MCADDRS) {
  4321. const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
  4322. int del = 1;
  4323. uint64_t hash = 0;
  4324. struct ifmultiaddr *ifma;
  4325. int i = 0, j;
  4326. if_maddr_rlock(ifp);
  4327. CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
  4328. if (ifma->ifma_addr->sa_family != AF_LINK)
  4329. continue;
  4330. mcaddr[i] =
  4331. LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
  4332. MPASS(ETHER_IS_MULTICAST(mcaddr[i]));
  4333. i++;
  4334. if (i == FW_MAC_EXACT_CHUNK) {
  4335. rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
  4336. del, i, mcaddr, NULL, &hash, 0);
  4337. if (rc < 0) {
  4338. rc = -rc;
  4339. for (j = 0; j < i; j++) {
  4340. if_printf(ifp,
  4341. "failed to add mc address"
  4342. " %02x:%02x:%02x:"
  4343. "%02x:%02x:%02x rc=%d\n",
  4344. mcaddr[j][0], mcaddr[j][1],
  4345. mcaddr[j][2], mcaddr[j][3],
  4346. mcaddr[j][4], mcaddr[j][5],
  4347. rc);
  4348. }
  4349. goto mcfail;
  4350. }
  4351. del = 0;
  4352. i = 0;
  4353. }
  4354. }
  4355. if (i > 0) {
  4356. rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i,
  4357. mcaddr, NULL, &hash, 0);
  4358. if (rc < 0) {
  4359. rc = -rc;
  4360. for (j = 0; j < i; j++) {
  4361. if_printf(ifp,
  4362. "failed to add mc address"
  4363. " %02x:%02x:%02x:"
  4364. "%02x:%02x:%02x rc=%d\n",
  4365. mcaddr[j][0], mcaddr[j][1],
  4366. mcaddr[j][2], mcaddr[j][3],
  4367. mcaddr[j][4], mcaddr[j][5],
  4368. rc);
  4369. }
  4370. goto mcfail;
  4371. }
  4372. }
  4373. rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0);
  4374. if (rc != 0)
  4375. if_printf(ifp, "failed to set mc address hash: %d", rc);
  4376. mcfail:
  4377. if_maddr_runlock(ifp);
  4378. }
  4379. return (rc);
  4380. }
  4381. /*
  4382. * {begin|end}_synchronized_op must be called from the same thread.
  4383. */
  4384. int
  4385. begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags,
  4386. char *wmesg)
  4387. {
  4388. int rc, pri;
  4389. #ifdef WITNESS
  4390. /* the caller thinks it's ok to sleep, but is it really? */
  4391. if (flags & SLEEP_OK)
  4392. WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
  4393. "begin_synchronized_op");
  4394. #endif
  4395. if (INTR_OK)
  4396. pri = PCATCH;
  4397. else
  4398. pri = 0;
  4399. ADAPTER_LOCK(sc);
  4400. for (;;) {
  4401. if (vi && IS_DOOMED(vi)) {
  4402. rc = ENXIO;
  4403. goto done;
  4404. }
  4405. if (!IS_BUSY(sc)) {
  4406. rc = 0;
  4407. break;
  4408. }
  4409. if (!(flags & SLEEP_OK)) {
  4410. rc = EBUSY;
  4411. goto done;
  4412. }
  4413. if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
  4414. rc = EINTR;
  4415. goto done;
  4416. }
  4417. }
  4418. KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
  4419. SET_BUSY(sc);
  4420. #ifdef INVARIANTS
  4421. sc->last_op = wmesg;
  4422. sc->last_op_thr = curthread;
  4423. sc->last_op_flags = flags;
  4424. #endif
  4425. done:
  4426. if (!(flags & HOLD_LOCK) || rc)
  4427. ADAPTER_UNLOCK(sc);
  4428. return (rc);
  4429. }
  4430. /*
  4431. * Tell if_ioctl and if_init that the VI is going away. This is
  4432. * special variant of begin_synchronized_op and must be paired with a
  4433. * call to end_synchronized_op.
  4434. */
  4435. void
  4436. doom_vi(struct adapter *sc, struct vi_info *vi)
  4437. {
  4438. ADAPTER_LOCK(sc);
  4439. SET_DOOMED(vi);
  4440. wakeup(&sc->flags);
  4441. while (IS_BUSY(sc))
  4442. mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
  4443. SET_BUSY(sc);
  4444. #ifdef INVARIANTS
  4445. sc->last_op = "t4detach";
  4446. sc->last_op_thr = curthread;
  4447. sc->last_op_flags = 0;
  4448. #endif
  4449. ADAPTER_UNLOCK(sc);
  4450. }
  4451. /*
  4452. * {begin|end}_synchronized_op must be called from the same thread.
  4453. */
  4454. void
  4455. end_synchronized_op(struct adapter *sc, int flags)
  4456. {
  4457. if (flags & LOCK_HELD)
  4458. ADAPTER_LOCK_ASSERT_OWNED(sc);
  4459. else
  4460. ADAPTER_LOCK(sc);
  4461. KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
  4462. CLR_BUSY(sc);
  4463. wakeup(&sc->flags);
  4464. ADAPTER_UNLOCK(sc);
  4465. }
  4466. static int
  4467. cxgbe_init_synchronized(struct vi_info *vi)
  4468. {
  4469. struct port_info *pi = vi->pi;
  4470. struct adapter *sc = pi->adapter;
  4471. struct ifnet *ifp = vi->ifp;
  4472. int rc = 0, i;
  4473. struct sge_txq *txq;
  4474. ASSERT_SYNCHRONIZED_OP(sc);
  4475. if (ifp->if_drv_flags & IFF_DRV_RUNNING)
  4476. return (0); /* already running */
  4477. if (!(sc->flags & FULL_INIT_DONE) &&
  4478. ((rc = adapter_full_init(sc)) != 0))
  4479. return (rc); /* error message displayed already */
  4480. if (!(vi->flags & VI_INIT_DONE) &&
  4481. ((rc = vi_full_init(vi)) != 0))
  4482. return (rc); /* error message displayed already */
  4483. rc = update_mac_settings(ifp, XGMAC_ALL);
  4484. if (rc)
  4485. goto done; /* error message displayed already */
  4486. PORT_LOCK(pi);
  4487. if (pi->up_vis == 0) {
  4488. t4_update_port_info(pi);
  4489. fixup_link_config(pi);
  4490. build_medialist(pi);
  4491. apply_link_config(pi);
  4492. }
  4493. rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
  4494. if (rc != 0) {
  4495. if_printf(ifp, "enable_vi failed: %d\n", rc);
  4496. PORT_UNLOCK(pi);
  4497. goto done;
  4498. }
  4499. /*
  4500. * Can't fail from this point onwards. Review cxgbe_uninit_synchronized
  4501. * if this changes.
  4502. */
  4503. for_each_txq(vi, i, txq) {
  4504. TXQ_LOCK(txq);
  4505. txq->eq.flags |= EQ_ENABLED;
  4506. TXQ_UNLOCK(txq);
  4507. }
  4508. /*
  4509. * The first iq of the first port to come up is used for tracing.
  4510. */
  4511. if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
  4512. sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
  4513. t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL :
  4514. A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
  4515. V_QUEUENUMBER(sc->traceq));
  4516. pi->flags |= HAS_TRACEQ;
  4517. }
  4518. /* all ok */
  4519. pi->up_vis++;
  4520. ifp->if_drv_flags |= IFF_DRV_RUNNING;
  4521. if (pi->nvi > 1 || sc->flags & IS_VF)
  4522. callout_reset(&vi->tick, hz, vi_tick, vi);
  4523. else
  4524. callout_reset(&pi->tick, hz, cxgbe_tick, pi);
  4525. if (pi->link_cfg.link_ok)
  4526. t4_os_link_changed(pi);
  4527. PORT_UNLOCK(pi);
  4528. done:
  4529. if (rc != 0)
  4530. cxgbe_uninit_synchronized(vi);
  4531. return (rc);
  4532. }
  4533. /*
  4534. * Idempotent.
  4535. */
  4536. static int
  4537. cxgbe_uninit_synchronized(struct vi_info *vi)
  4538. {
  4539. struct port_info *pi = vi->pi;
  4540. struct adapter *sc = pi->adapter;
  4541. struct ifnet *ifp = vi->ifp;
  4542. int rc, i;
  4543. struct sge_txq *txq;
  4544. ASSERT_SYNCHRONIZED_OP(sc);
  4545. if (!(vi->flags & VI_INIT_DONE)) {
  4546. if (__predict_false(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
  4547. KASSERT(0, ("uninited VI is running"));
  4548. if_printf(ifp, "uninited VI with running ifnet. "
  4549. "vi->flags 0x%016lx, if_flags 0x%08x, "
  4550. "if_drv_flags 0x%08x\n", vi->flags, ifp->if_flags,
  4551. ifp->if_drv_flags);
  4552. }
  4553. return (0);
  4554. }
  4555. /*
  4556. * Disable the VI so that all its data in either direction is discarded
  4557. * by the MPS. Leave everything else (the queues, interrupts, and 1Hz
  4558. * tick) intact as the TP can deliver negative advice or data that it's
  4559. * holding in its RAM (for an offloaded connection) even after the VI is
  4560. * disabled.
  4561. */
  4562. rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
  4563. if (rc) {
  4564. if_printf(ifp, "disable_vi failed: %d\n", rc);
  4565. return (rc);
  4566. }
  4567. for_each_txq(vi, i, txq) {
  4568. TXQ_LOCK(txq);
  4569. txq->eq.flags &= ~EQ_ENABLED;
  4570. TXQ_UNLOCK(txq);
  4571. }
  4572. PORT_LOCK(pi);
  4573. if (pi->nvi > 1 || sc->flags & IS_VF)
  4574. callout_stop(&vi->tick);
  4575. else
  4576. callout_stop(&pi->tick);
  4577. if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
  4578. PORT_UNLOCK(pi);
  4579. return (0);
  4580. }
  4581. ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
  4582. pi->up_vis--;
  4583. if (pi->up_vis > 0) {
  4584. PORT_UNLOCK(pi);
  4585. return (0);
  4586. }
  4587. pi->link_cfg.link_ok = false;
  4588. pi->link_cfg.speed = 0;
  4589. pi->link_cfg.link_down_rc = 255;
  4590. t4_os_link_changed(pi);
  4591. PORT_UNLOCK(pi);
  4592. return (0);
  4593. }
  4594. /*
  4595. * It is ok for this function to fail midway and return right away. t4_detach
  4596. * will walk the entire sc->irq list and clean up whatever is valid.
  4597. */
  4598. int
  4599. t4_setup_intr_handlers(struct adapter *sc)
  4600. {
  4601. int rc, rid, p, q, v;
  4602. char s[8];
  4603. struct irq *irq;
  4604. struct port_info *pi;
  4605. struct vi_info *vi;
  4606. struct sge *sge = &sc->sge;
  4607. struct sge_rxq *rxq;
  4608. #ifdef TCP_OFFLOAD
  4609. struct sge_ofld_rxq *ofld_rxq;
  4610. #endif
  4611. #ifdef DEV_NETMAP
  4612. struct sge_nm_rxq *nm_rxq;
  4613. #endif
  4614. #ifdef RSS
  4615. int nbuckets = rss_getnumbuckets();
  4616. #endif
  4617. /*
  4618. * Setup interrupts.
  4619. */
  4620. irq = &sc->irq[0];
  4621. rid = sc->intr_type == INTR_INTX ? 0 : 1;
  4622. if (forwarding_intr_to_fwq(sc))
  4623. return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
  4624. /* Multiple interrupts. */
  4625. if (sc->flags & IS_VF)
  4626. KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports,
  4627. ("%s: too few intr.", __func__));
  4628. else
  4629. KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
  4630. ("%s: too few intr.", __func__));
  4631. /* The first one is always error intr on PFs */
  4632. if (!(sc->flags & IS_VF)) {
  4633. rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
  4634. if (rc != 0)
  4635. return (rc);
  4636. irq++;
  4637. rid++;
  4638. }
  4639. /* The second one is always the firmware event queue (first on VFs) */
  4640. rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt");
  4641. if (rc != 0)
  4642. return (rc);
  4643. irq++;
  4644. rid++;
  4645. for_each_port(sc, p) {
  4646. pi = sc->port[p];
  4647. for_each_vi(pi, v, vi) {
  4648. vi->first_intr = rid - 1;
  4649. if (vi->nnmrxq > 0) {
  4650. int n = max(vi->nrxq, vi->nnmrxq);
  4651. rxq = &sge->rxq[vi->first_rxq];
  4652. #ifdef DEV_NETMAP
  4653. nm_rxq = &sge->nm_rxq[vi->first_nm_rxq];
  4654. #endif
  4655. for (q = 0; q < n; q++) {
  4656. snprintf(s, sizeof(s), "%x%c%x", p,
  4657. 'a' + v, q);
  4658. if (q < vi->nrxq)
  4659. irq->rxq = rxq++;
  4660. #ifdef DEV_NETMAP
  4661. if (q < vi->nnmrxq)
  4662. irq->nm_rxq = nm_rxq++;
  4663. if (irq->nm_rxq != NULL &&
  4664. irq->rxq == NULL) {
  4665. /* Netmap rx only */
  4666. rc = t4_alloc_irq(sc, irq, rid,
  4667. t4_nm_intr, irq->nm_rxq, s);
  4668. }
  4669. if (irq->nm_rxq != NULL &&
  4670. irq->rxq != NULL) {
  4671. /* NIC and Netmap rx */
  4672. rc = t4_alloc_irq(sc, irq, rid,
  4673. t4_vi_intr, irq, s);
  4674. }
  4675. #endif
  4676. if (irq->rxq != NULL &&
  4677. irq->nm_rxq == NULL) {
  4678. /* NIC rx only */
  4679. rc = t4_alloc_irq(sc, irq, rid,
  4680. t4_intr, irq->rxq, s);
  4681. }
  4682. if (rc != 0)
  4683. return (rc);
  4684. #ifdef RSS
  4685. if (q < vi->nrxq) {
  4686. bus_bind_intr(sc->dev, irq->res,
  4687. rss_getcpu(q % nbuckets));
  4688. }
  4689. #endif
  4690. irq++;
  4691. rid++;
  4692. vi->nintr++;
  4693. }
  4694. } else {
  4695. for_each_rxq(vi, q, rxq) {
  4696. snprintf(s, sizeof(s), "%x%c%x", p,
  4697. 'a' + v, q);
  4698. rc = t4_alloc_irq(sc, irq, rid,
  4699. t4_intr, rxq, s);
  4700. if (rc != 0)
  4701. return (rc);
  4702. #ifdef RSS
  4703. bus_bind_intr(sc->dev, irq->res,
  4704. rss_getcpu(q % nbuckets));
  4705. #endif
  4706. irq++;
  4707. rid++;
  4708. vi->nintr++;
  4709. }
  4710. }
  4711. #ifdef TCP_OFFLOAD
  4712. for_each_ofld_rxq(vi, q, ofld_rxq) {
  4713. snprintf(s, sizeof(s), "%x%c%x", p, 'A' + v, q);
  4714. rc = t4_alloc_irq(sc, irq, rid, t4_intr,
  4715. ofld_rxq, s);
  4716. if (rc != 0)
  4717. return (rc);
  4718. irq++;
  4719. rid++;
  4720. vi->nintr++;
  4721. }
  4722. #endif
  4723. }
  4724. }
  4725. MPASS(irq == &sc->irq[sc->intr_count]);
  4726. return (0);
  4727. }
  4728. int
  4729. adapter_full_init(struct adapter *sc)
  4730. {
  4731. int rc, i;
  4732. #ifdef RSS
  4733. uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
  4734. uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
  4735. #endif
  4736. ASSERT_SYNCHRONIZED_OP(sc);
  4737. ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
  4738. KASSERT((sc->flags & FULL_INIT_DONE) == 0,
  4739. ("%s: FULL_INIT_DONE already", __func__));
  4740. /*
  4741. * queues that belong to the adapter (not any particular port).
  4742. */
  4743. rc = t4_setup_adapter_queues(sc);
  4744. if (rc != 0)
  4745. goto done;
  4746. for (i = 0; i < nitems(sc->tq); i++) {
  4747. sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
  4748. taskqueue_thread_enqueue, &sc->tq[i]);
  4749. if (sc->tq[i] == NULL) {
  4750. device_printf(sc->dev,
  4751. "failed to allocate task queue %d\n", i);
  4752. rc = ENOMEM;
  4753. goto done;
  4754. }
  4755. taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
  4756. device_get_nameunit(sc->dev), i);
  4757. }
  4758. #ifdef RSS
  4759. MPASS(RSS_KEYSIZE == 40);
  4760. rss_getkey((void *)&raw_rss_key[0]);
  4761. for (i = 0; i < nitems(rss_key); i++) {
  4762. rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
  4763. }
  4764. t4_write_rss_key(sc, &rss_key[0], -1, 1);
  4765. #endif
  4766. if (!(sc->flags & IS_VF))
  4767. t4_intr_enable(sc);
  4768. sc->flags |= FULL_INIT_DONE;
  4769. done:
  4770. if (rc != 0)
  4771. adapter_full_uninit(sc);
  4772. return (rc);
  4773. }
  4774. int
  4775. adapter_full_uninit(struct adapter *sc)
  4776. {
  4777. int i;
  4778. ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
  4779. t4_teardown_adapter_queues(sc);
  4780. for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
  4781. taskqueue_free(sc->tq[i]);
  4782. sc->tq[i] = NULL;
  4783. }
  4784. sc->flags &= ~FULL_INIT_DONE;
  4785. return (0);
  4786. }
  4787. #ifdef RSS
  4788. #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
  4789. RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
  4790. RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
  4791. RSS_HASHTYPE_RSS_UDP_IPV6)
  4792. /* Translates kernel hash types to hardware. */
  4793. static int
  4794. hashconfig_to_hashen(int hashconfig)
  4795. {
  4796. int hashen = 0;
  4797. if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
  4798. hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
  4799. if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
  4800. hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
  4801. if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
  4802. hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
  4803. F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
  4804. }
  4805. if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
  4806. hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
  4807. F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
  4808. }
  4809. if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
  4810. hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
  4811. if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
  4812. hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
  4813. return (hashen);
  4814. }
  4815. /* Translates hardware hash types to kernel. */
  4816. static int
  4817. hashen_to_hashconfig(int hashen)
  4818. {
  4819. int hashconfig = 0;
  4820. if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
  4821. /*
  4822. * If UDP hashing was enabled it must have been enabled for
  4823. * either IPv4 or IPv6 (inclusive or). Enabling UDP without
  4824. * enabling any 4-tuple hash is nonsense configuration.
  4825. */
  4826. MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
  4827. F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
  4828. if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
  4829. hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
  4830. if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
  4831. hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
  4832. }
  4833. if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
  4834. hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
  4835. if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
  4836. hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
  4837. if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
  4838. hashconfig |= RSS_HASHTYPE_RSS_IPV4;
  4839. if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
  4840. hashconfig |= RSS_HASHTYPE_RSS_IPV6;
  4841. return (hashconfig);
  4842. }
  4843. #endif
  4844. int
  4845. vi_full_init(struct vi_info *vi)
  4846. {
  4847. struct adapter *sc = vi->pi->adapter;
  4848. struct ifnet *ifp = vi->ifp;
  4849. uint16_t *rss;
  4850. struct sge_rxq *rxq;
  4851. int rc, i, j;
  4852. #ifdef RSS
  4853. int nbuckets = rss_getnumbuckets();
  4854. int hashconfig = rss_gethashconfig();
  4855. int extra;
  4856. #endif
  4857. ASSERT_SYNCHRONIZED_OP(sc);
  4858. KASSERT((vi->flags & VI_INIT_DONE) == 0,
  4859. ("%s: VI_INIT_DONE already", __func__));
  4860. sysctl_ctx_init(&vi->ctx);
  4861. vi->flags |= VI_SYSCTL_CTX;
  4862. /*
  4863. * Allocate tx/rx/fl queues for this VI.
  4864. */
  4865. rc = t4_setup_vi_queues(vi);
  4866. if (rc != 0)
  4867. goto done; /* error message displayed already */
  4868. /*
  4869. * Setup RSS for this VI. Save a copy of the RSS table for later use.
  4870. */
  4871. if (vi->nrxq > vi->rss_size) {
  4872. if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); "
  4873. "some queues will never receive traffic.\n", vi->nrxq,
  4874. vi->rss_size);
  4875. } else if (vi->rss_size % vi->nrxq) {
  4876. if_printf(ifp, "nrxq (%d), hw RSS table size (%d); "
  4877. "expect uneven traffic distribution.\n", vi->nrxq,
  4878. vi->rss_size);
  4879. }
  4880. #ifdef RSS
  4881. if (vi->nrxq != nbuckets) {
  4882. if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);"
  4883. "performance will be impacted.\n", vi->nrxq, nbuckets);
  4884. }
  4885. #endif
  4886. rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
  4887. for (i = 0; i < vi->rss_size;) {
  4888. #ifdef RSS
  4889. j = rss_get_indirection_to_bucket(i);
  4890. j %= vi->nrxq;
  4891. rxq = &sc->sge.rxq[vi->first_rxq + j];
  4892. rss[i++] = rxq->iq.abs_id;
  4893. #else
  4894. for_each_rxq(vi, j, rxq) {
  4895. rss[i++] = rxq->iq.abs_id;
  4896. if (i == vi->rss_size)
  4897. break;
  4898. }
  4899. #endif
  4900. }
  4901. rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss,
  4902. vi->rss_size);
  4903. if (rc != 0) {
  4904. free(rss, M_CXGBE);
  4905. if_printf(ifp, "rss_config failed: %d\n", rc);
  4906. goto done;
  4907. }
  4908. #ifdef RSS
  4909. vi->hashen = hashconfig_to_hashen(hashconfig);
  4910. /*
  4911. * We may have had to enable some hashes even though the global config
  4912. * wants them disabled. This is a potential problem that must be
  4913. * reported to the user.
  4914. */
  4915. extra = hashen_to_hashconfig(vi->hashen) ^ hashconfig;
  4916. /*
  4917. * If we consider only the supported hash types, then the enabled hashes
  4918. * are a superset of the requested hashes. In other words, there cannot
  4919. * be any supported hash that was requested but not enabled, but there
  4920. * can be hashes that were not requested but had to be enabled.
  4921. */
  4922. extra &= SUPPORTED_RSS_HASHTYPES;
  4923. MPASS((extra & hashconfig) == 0);
  4924. if (extra) {
  4925. if_printf(ifp,
  4926. "global RSS config (0x%x) cannot be accommodated.\n",
  4927. hashconfig);
  4928. }
  4929. if (extra & RSS_HASHTYPE_RSS_IPV4)
  4930. if_printf(ifp, "IPv4 2-tuple hashing forced on.\n");
  4931. if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
  4932. if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n");
  4933. if (extra & RSS_HASHTYPE_RSS_IPV6)
  4934. if_printf(ifp, "IPv6 2-tuple hashing forced on.\n");
  4935. if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
  4936. if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n");
  4937. if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
  4938. if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n");
  4939. if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
  4940. if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n");
  4941. #else
  4942. vi->hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
  4943. F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
  4944. F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
  4945. F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
  4946. #endif
  4947. rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, rss[0], 0, 0);
  4948. if (rc != 0) {
  4949. free(rss, M_CXGBE);
  4950. if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc);
  4951. goto done;
  4952. }
  4953. vi->rss = rss;
  4954. vi->flags |= VI_INIT_DONE;
  4955. done:
  4956. if (rc != 0)
  4957. vi_full_uninit(vi);
  4958. return (rc);
  4959. }
  4960. /*
  4961. * Idempotent.
  4962. */
  4963. int
  4964. vi_full_uninit(struct vi_info *vi)
  4965. {
  4966. struct port_info *pi = vi->pi;
  4967. struct adapter *sc = pi->adapter;
  4968. int i;
  4969. struct sge_rxq *rxq;
  4970. struct sge_txq *txq;
  4971. #ifdef TCP_OFFLOAD
  4972. struct sge_ofld_rxq *ofld_rxq;
  4973. #endif
  4974. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  4975. struct sge_wrq *ofld_txq;
  4976. #endif
  4977. if (vi->flags & VI_INIT_DONE) {
  4978. /* Need to quiesce queues. */
  4979. /* XXX: Only for the first VI? */
  4980. if (IS_MAIN_VI(vi) && !(sc->flags & IS_VF))
  4981. quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
  4982. for_each_txq(vi, i, txq) {
  4983. quiesce_txq(sc, txq);
  4984. }
  4985. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  4986. for_each_ofld_txq(vi, i, ofld_txq) {
  4987. quiesce_wrq(sc, ofld_txq);
  4988. }
  4989. #endif
  4990. for_each_rxq(vi, i, rxq) {
  4991. quiesce_iq(sc, &rxq->iq);
  4992. quiesce_fl(sc, &rxq->fl);
  4993. }
  4994. #ifdef TCP_OFFLOAD
  4995. for_each_ofld_rxq(vi, i, ofld_rxq) {
  4996. quiesce_iq(sc, &ofld_rxq->iq);
  4997. quiesce_fl(sc, &ofld_rxq->fl);
  4998. }
  4999. #endif
  5000. free(vi->rss, M_CXGBE);
  5001. free(vi->nm_rss, M_CXGBE);
  5002. }
  5003. t4_teardown_vi_queues(vi);
  5004. vi->flags &= ~VI_INIT_DONE;
  5005. return (0);
  5006. }
  5007. static void
  5008. quiesce_txq(struct adapter *sc, struct sge_txq *txq)
  5009. {
  5010. struct sge_eq *eq = &txq->eq;
  5011. struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
  5012. (void) sc; /* unused */
  5013. #ifdef INVARIANTS
  5014. TXQ_LOCK(txq);
  5015. MPASS((eq->flags & EQ_ENABLED) == 0);
  5016. TXQ_UNLOCK(txq);
  5017. #endif
  5018. /* Wait for the mp_ring to empty. */
  5019. while (!mp_ring_is_idle(txq->r)) {
  5020. mp_ring_check_drainage(txq->r, 0);
  5021. pause("rquiesce", 1);
  5022. }
  5023. /* Then wait for the hardware to finish. */
  5024. while (spg->cidx != htobe16(eq->pidx))
  5025. pause("equiesce", 1);
  5026. /* Finally, wait for the driver to reclaim all descriptors. */
  5027. while (eq->cidx != eq->pidx)
  5028. pause("dquiesce", 1);
  5029. }
  5030. static void
  5031. quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq)
  5032. {
  5033. /* XXXTX */
  5034. }
  5035. static void
  5036. quiesce_iq(struct adapter *sc, struct sge_iq *iq)
  5037. {
  5038. (void) sc; /* unused */
  5039. /* Synchronize with the interrupt handler */
  5040. while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
  5041. pause("iqfree", 1);
  5042. }
  5043. static void
  5044. quiesce_fl(struct adapter *sc, struct sge_fl *fl)
  5045. {
  5046. mtx_lock(&sc->sfl_lock);
  5047. FL_LOCK(fl);
  5048. fl->flags |= FL_DOOMED;
  5049. FL_UNLOCK(fl);
  5050. callout_stop(&sc->sfl_callout);
  5051. mtx_unlock(&sc->sfl_lock);
  5052. KASSERT((fl->flags & FL_STARVING) == 0,
  5053. ("%s: still starving", __func__));
  5054. }
  5055. static int
  5056. t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
  5057. driver_intr_t *handler, void *arg, char *name)
  5058. {
  5059. int rc;
  5060. irq->rid = rid;
  5061. irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
  5062. RF_SHAREABLE | RF_ACTIVE);
  5063. if (irq->res == NULL) {
  5064. device_printf(sc->dev,
  5065. "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
  5066. return (ENOMEM);
  5067. }
  5068. rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
  5069. NULL, handler, arg, &irq->tag);
  5070. if (rc != 0) {
  5071. device_printf(sc->dev,
  5072. "failed to setup interrupt for rid %d, name %s: %d\n",
  5073. rid, name, rc);
  5074. } else if (name)
  5075. bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name);
  5076. return (rc);
  5077. }
  5078. static int
  5079. t4_free_irq(struct adapter *sc, struct irq *irq)
  5080. {
  5081. if (irq->tag)
  5082. bus_teardown_intr(sc->dev, irq->res, irq->tag);
  5083. if (irq->res)
  5084. bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
  5085. bzero(irq, sizeof(*irq));
  5086. return (0);
  5087. }
  5088. static void
  5089. get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
  5090. {
  5091. regs->version = chip_id(sc) | chip_rev(sc) << 10;
  5092. t4_get_regs(sc, buf, regs->len);
  5093. }
  5094. #define A_PL_INDIR_CMD 0x1f8
  5095. #define S_PL_AUTOINC 31
  5096. #define M_PL_AUTOINC 0x1U
  5097. #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC)
  5098. #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC)
  5099. #define S_PL_VFID 20
  5100. #define M_PL_VFID 0xffU
  5101. #define V_PL_VFID(x) ((x) << S_PL_VFID)
  5102. #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID)
  5103. #define S_PL_ADDR 0
  5104. #define M_PL_ADDR 0xfffffU
  5105. #define V_PL_ADDR(x) ((x) << S_PL_ADDR)
  5106. #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR)
  5107. #define A_PL_INDIR_DATA 0x1fc
  5108. static uint64_t
  5109. read_vf_stat(struct adapter *sc, u_int vin, int reg)
  5110. {
  5111. u32 stats[2];
  5112. mtx_assert(&sc->reg_lock, MA_OWNED);
  5113. if (sc->flags & IS_VF) {
  5114. stats[0] = t4_read_reg(sc, VF_MPS_REG(reg));
  5115. stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4));
  5116. } else {
  5117. t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
  5118. V_PL_VFID(vin) | V_PL_ADDR(VF_MPS_REG(reg)));
  5119. stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA);
  5120. stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA);
  5121. }
  5122. return (((uint64_t)stats[1]) << 32 | stats[0]);
  5123. }
  5124. static void
  5125. t4_get_vi_stats(struct adapter *sc, u_int vin, struct fw_vi_stats_vf *stats)
  5126. {
  5127. #define GET_STAT(name) \
  5128. read_vf_stat(sc, vin, A_MPS_VF_STAT_##name##_L)
  5129. stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES);
  5130. stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES);
  5131. stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES);
  5132. stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES);
  5133. stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES);
  5134. stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES);
  5135. stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES);
  5136. stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES);
  5137. stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
  5138. stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES);
  5139. stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES);
  5140. stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES);
  5141. stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES);
  5142. stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES);
  5143. stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES);
  5144. stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES);
  5145. #undef GET_STAT
  5146. }
  5147. static void
  5148. t4_clr_vi_stats(struct adapter *sc, u_int vin)
  5149. {
  5150. int reg;
  5151. t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | V_PL_VFID(vin) |
  5152. V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L)));
  5153. for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
  5154. reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
  5155. t4_write_reg(sc, A_PL_INDIR_DATA, 0);
  5156. }
  5157. static void
  5158. vi_refresh_stats(struct adapter *sc, struct vi_info *vi)
  5159. {
  5160. struct timeval tv;
  5161. const struct timeval interval = {0, 250000}; /* 250ms */
  5162. if (!(vi->flags & VI_INIT_DONE))
  5163. return;
  5164. getmicrotime(&tv);
  5165. timevalsub(&tv, &interval);
  5166. if (timevalcmp(&tv, &vi->last_refreshed, <))
  5167. return;
  5168. mtx_lock(&sc->reg_lock);
  5169. t4_get_vi_stats(sc, vi->vin, &vi->stats);
  5170. getmicrotime(&vi->last_refreshed);
  5171. mtx_unlock(&sc->reg_lock);
  5172. }
  5173. static void
  5174. cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi)
  5175. {
  5176. u_int i, v, tnl_cong_drops, chan_map;
  5177. struct timeval tv;
  5178. const struct timeval interval = {0, 250000}; /* 250ms */
  5179. getmicrotime(&tv);
  5180. timevalsub(&tv, &interval);
  5181. if (timevalcmp(&tv, &pi->last_refreshed, <))
  5182. return;
  5183. tnl_cong_drops = 0;
  5184. t4_get_port_stats(sc, pi->tx_chan, &pi->stats);
  5185. chan_map = pi->rx_e_chan_map;
  5186. while (chan_map) {
  5187. i = ffs(chan_map) - 1;
  5188. mtx_lock(&sc->reg_lock);
  5189. t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 1,
  5190. A_TP_MIB_TNL_CNG_DROP_0 + i);
  5191. mtx_unlock(&sc->reg_lock);
  5192. tnl_cong_drops += v;
  5193. chan_map &= ~(1 << i);
  5194. }
  5195. pi->tnl_cong_drops = tnl_cong_drops;
  5196. getmicrotime(&pi->last_refreshed);
  5197. }
  5198. static void
  5199. cxgbe_tick(void *arg)
  5200. {
  5201. struct port_info *pi = arg;
  5202. struct adapter *sc = pi->adapter;
  5203. PORT_LOCK_ASSERT_OWNED(pi);
  5204. cxgbe_refresh_stats(sc, pi);
  5205. callout_schedule(&pi->tick, hz);
  5206. }
  5207. void
  5208. vi_tick(void *arg)
  5209. {
  5210. struct vi_info *vi = arg;
  5211. struct adapter *sc = vi->pi->adapter;
  5212. vi_refresh_stats(sc, vi);
  5213. callout_schedule(&vi->tick, hz);
  5214. }
  5215. /*
  5216. * Should match fw_caps_config_<foo> enums in t4fw_interface.h
  5217. */
  5218. static char *caps_decoder[] = {
  5219. "\20\001IPMI\002NCSI", /* 0: NBM */
  5220. "\20\001PPP\002QFC\003DCBX", /* 1: link */
  5221. "\20\001INGRESS\002EGRESS", /* 2: switch */
  5222. "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL" /* 3: NIC */
  5223. "\006HASHFILTER\007ETHOFLD",
  5224. "\20\001TOE", /* 4: TOE */
  5225. "\20\001RDDP\002RDMAC", /* 5: RDMA */
  5226. "\20\001INITIATOR_PDU\002TARGET_PDU" /* 6: iSCSI */
  5227. "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD"
  5228. "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD"
  5229. "\007T10DIF"
  5230. "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD",
  5231. "\20\001LOOKASIDE\002TLSKEYS", /* 7: Crypto */
  5232. "\20\001INITIATOR\002TARGET\003CTRL_OFLD" /* 8: FCoE */
  5233. "\004PO_INITIATOR\005PO_TARGET",
  5234. };
  5235. void
  5236. t4_sysctls(struct adapter *sc)
  5237. {
  5238. struct sysctl_ctx_list *ctx;
  5239. struct sysctl_oid *oid;
  5240. struct sysctl_oid_list *children, *c0;
  5241. static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
  5242. ctx = device_get_sysctl_ctx(sc->dev);
  5243. /*
  5244. * dev.t4nex.X.
  5245. */
  5246. oid = device_get_sysctl_tree(sc->dev);
  5247. c0 = children = SYSCTL_CHILDREN(oid);
  5248. sc->sc_do_rxcopy = 1;
  5249. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
  5250. &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
  5251. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
  5252. sc->params.nports, "# of ports");
  5253. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
  5254. CTLTYPE_STRING | CTLFLAG_RD, doorbells, (uintptr_t)&sc->doorbells,
  5255. sysctl_bitfield_8b, "A", "available doorbells");
  5256. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
  5257. sc->params.vpd.cclk, "core clock frequency (in KHz)");
  5258. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
  5259. CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.timer_val,
  5260. sizeof(sc->params.sge.timer_val), sysctl_int_array, "A",
  5261. "interrupt holdoff timer values (us)");
  5262. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
  5263. CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.counter_val,
  5264. sizeof(sc->params.sge.counter_val), sysctl_int_array, "A",
  5265. "interrupt holdoff packet counter values");
  5266. t4_sge_sysctls(sc, ctx, children);
  5267. sc->lro_timeout = 100;
  5268. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
  5269. &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
  5270. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW,
  5271. &sc->debug_flags, 0, "flags to enable runtime debugging");
  5272. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version",
  5273. CTLFLAG_RD, sc->tp_version, 0, "TP microcode version");
  5274. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
  5275. CTLFLAG_RD, sc->fw_version, 0, "firmware version");
  5276. if (sc->flags & IS_VF)
  5277. return;
  5278. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
  5279. NULL, chip_rev(sc), "chip hardware revision");
  5280. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn",
  5281. CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number");
  5282. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn",
  5283. CTLFLAG_RD, sc->params.vpd.pn, 0, "part number");
  5284. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec",
  5285. CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change");
  5286. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "md_version",
  5287. CTLFLAG_RD, sc->params.vpd.md, 0, "manufacturing diags version");
  5288. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na",
  5289. CTLFLAG_RD, sc->params.vpd.na, 0, "network address");
  5290. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD,
  5291. sc->er_version, 0, "expansion ROM version");
  5292. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD,
  5293. sc->bs_version, 0, "bootstrap firmware version");
  5294. SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD,
  5295. NULL, sc->params.scfg_vers, "serial config version");
  5296. SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD,
  5297. NULL, sc->params.vpd_vers, "VPD version");
  5298. SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
  5299. CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
  5300. SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
  5301. sc->cfcsum, "config file checksum");
  5302. #define SYSCTL_CAP(name, n, text) \
  5303. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \
  5304. CTLTYPE_STRING | CTLFLAG_RD, caps_decoder[n], (uintptr_t)&sc->name, \
  5305. sysctl_bitfield_16b, "A", "available " text " capabilities")
  5306. SYSCTL_CAP(nbmcaps, 0, "NBM");
  5307. SYSCTL_CAP(linkcaps, 1, "link");
  5308. SYSCTL_CAP(switchcaps, 2, "switch");
  5309. SYSCTL_CAP(niccaps, 3, "NIC");
  5310. SYSCTL_CAP(toecaps, 4, "TCP offload");
  5311. SYSCTL_CAP(rdmacaps, 5, "RDMA");
  5312. SYSCTL_CAP(iscsicaps, 6, "iSCSI");
  5313. SYSCTL_CAP(cryptocaps, 7, "crypto");
  5314. SYSCTL_CAP(fcoecaps, 8, "FCoE");
  5315. #undef SYSCTL_CAP
  5316. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
  5317. NULL, sc->tids.nftids, "number of filters");
  5318. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT |
  5319. CTLFLAG_RD, sc, 0, sysctl_temperature, "I",
  5320. "chip temperature (in Celsius)");
  5321. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "reset_sensor", CTLTYPE_INT |
  5322. CTLFLAG_RW, sc, 0, sysctl_reset_sensor, "I",
  5323. "reset the chip's temperature sensor.");
  5324. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "loadavg", CTLTYPE_STRING |
  5325. CTLFLAG_RD, sc, 0, sysctl_loadavg, "A",
  5326. "microprocessor load averages (debug firmwares only)");
  5327. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "core_vdd", CTLTYPE_INT |
  5328. CTLFLAG_RD, sc, 0, sysctl_vdd, "I", "core Vdd (in mV)");
  5329. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "local_cpus",
  5330. CTLTYPE_STRING | CTLFLAG_RD, sc, LOCAL_CPUS,
  5331. sysctl_cpus, "A", "local CPUs");
  5332. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_cpus",
  5333. CTLTYPE_STRING | CTLFLAG_RD, sc, INTR_CPUS,
  5334. sysctl_cpus, "A", "preferred CPUs for interrupts");
  5335. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "swintr", CTLFLAG_RW,
  5336. &sc->swintr, 0, "software triggered interrupts");
  5337. /*
  5338. * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload.
  5339. */
  5340. oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
  5341. CTLFLAG_RD | CTLFLAG_SKIP, NULL,
  5342. "logs and miscellaneous information");
  5343. children = SYSCTL_CHILDREN(oid);
  5344. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
  5345. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5346. sysctl_cctrl, "A", "congestion control");
  5347. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
  5348. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5349. sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
  5350. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
  5351. CTLTYPE_STRING | CTLFLAG_RD, sc, 1,
  5352. sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
  5353. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
  5354. CTLTYPE_STRING | CTLFLAG_RD, sc, 2,
  5355. sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
  5356. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
  5357. CTLTYPE_STRING | CTLFLAG_RD, sc, 3,
  5358. sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
  5359. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
  5360. CTLTYPE_STRING | CTLFLAG_RD, sc, 4,
  5361. sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
  5362. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
  5363. CTLTYPE_STRING | CTLFLAG_RD, sc, 5,
  5364. sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
  5365. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
  5366. CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_cim_la,
  5367. "A", "CIM logic analyzer");
  5368. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
  5369. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5370. sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
  5371. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
  5372. CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ,
  5373. sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
  5374. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
  5375. CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ,
  5376. sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
  5377. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
  5378. CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ,
  5379. sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
  5380. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
  5381. CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ,
  5382. sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
  5383. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
  5384. CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ,
  5385. sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
  5386. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
  5387. CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ,
  5388. sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
  5389. if (chip_id(sc) > CHELSIO_T4) {
  5390. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
  5391. CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ,
  5392. sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)");
  5393. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
  5394. CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ,
  5395. sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)");
  5396. }
  5397. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
  5398. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5399. sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
  5400. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
  5401. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5402. sysctl_cim_qcfg, "A", "CIM queue configuration");
  5403. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
  5404. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5405. sysctl_cpl_stats, "A", "CPL statistics");
  5406. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
  5407. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5408. sysctl_ddp_stats, "A", "non-TCP DDP statistics");
  5409. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
  5410. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5411. sysctl_devlog, "A", "firmware's device log");
  5412. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
  5413. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5414. sysctl_fcoe_stats, "A", "FCoE statistics");
  5415. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
  5416. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5417. sysctl_hw_sched, "A", "hardware scheduler ");
  5418. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
  5419. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5420. sysctl_l2t, "A", "hardware L2 table");
  5421. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "smt",
  5422. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5423. sysctl_smt, "A", "hardware source MAC table");
  5424. #ifdef INET6
  5425. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "clip",
  5426. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5427. sysctl_clip, "A", "active CLIP table entries");
  5428. #endif
  5429. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
  5430. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5431. sysctl_lb_stats, "A", "loopback statistics");
  5432. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
  5433. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5434. sysctl_meminfo, "A", "memory regions");
  5435. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
  5436. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5437. chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6,
  5438. "A", "MPS TCAM entries");
  5439. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
  5440. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5441. sysctl_path_mtus, "A", "path MTUs");
  5442. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
  5443. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5444. sysctl_pm_stats, "A", "PM statistics");
  5445. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
  5446. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5447. sysctl_rdma_stats, "A", "RDMA statistics");
  5448. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
  5449. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5450. sysctl_tcp_stats, "A", "TCP statistics");
  5451. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
  5452. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5453. sysctl_tids, "A", "TID information");
  5454. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
  5455. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5456. sysctl_tp_err_stats, "A", "TP error statistics");
  5457. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask",
  5458. CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tp_la_mask, "I",
  5459. "TP logic analyzer event capture mask");
  5460. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
  5461. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5462. sysctl_tp_la, "A", "TP logic analyzer");
  5463. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
  5464. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5465. sysctl_tx_rate, "A", "Tx rate");
  5466. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
  5467. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5468. sysctl_ulprx_la, "A", "ULPRX logic analyzer");
  5469. if (chip_id(sc) >= CHELSIO_T5) {
  5470. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
  5471. CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
  5472. sysctl_wcwr_stats, "A", "write combined work requests");
  5473. }
  5474. #ifdef TCP_OFFLOAD
  5475. if (is_offload(sc)) {
  5476. int i;
  5477. char s[4];
  5478. /*
  5479. * dev.t4nex.X.toe.
  5480. */
  5481. oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
  5482. NULL, "TOE parameters");
  5483. children = SYSCTL_CHILDREN(oid);
  5484. sc->tt.cong_algorithm = -1;
  5485. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_algorithm",
  5486. CTLFLAG_RW, &sc->tt.cong_algorithm, 0, "congestion control "
  5487. "(-1 = default, 0 = reno, 1 = tahoe, 2 = newreno, "
  5488. "3 = highspeed)");
  5489. sc->tt.sndbuf = -1;
  5490. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
  5491. &sc->tt.sndbuf, 0, "hardware send buffer");
  5492. sc->tt.ddp = 0;
  5493. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
  5494. &sc->tt.ddp, 0, "DDP allowed");
  5495. sc->tt.rx_coalesce = -1;
  5496. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
  5497. CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
  5498. sc->tt.tls = 0;
  5499. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tls", CTLFLAG_RW,
  5500. &sc->tt.tls, 0, "Inline TLS allowed");
  5501. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls_rx_ports",
  5502. CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tls_rx_ports,
  5503. "I", "TCP ports that use inline TLS+TOE RX");
  5504. sc->tt.tx_align = -1;
  5505. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
  5506. CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
  5507. sc->tt.tx_zcopy = 0;
  5508. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy",
  5509. CTLFLAG_RW, &sc->tt.tx_zcopy, 0,
  5510. "Enable zero-copy aio_write(2)");
  5511. sc->tt.cop_managed_offloading = !!t4_cop_managed_offloading;
  5512. SYSCTL_ADD_INT(ctx, children, OID_AUTO,
  5513. "cop_managed_offloading", CTLFLAG_RW,
  5514. &sc->tt.cop_managed_offloading, 0,
  5515. "COP (Connection Offload Policy) controls all TOE offload");
  5516. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick",
  5517. CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_tick, "A",
  5518. "TP timer tick (us)");
  5519. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick",
  5520. CTLTYPE_STRING | CTLFLAG_RD, sc, 1, sysctl_tp_tick, "A",
  5521. "TCP timestamp tick (us)");
  5522. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick",
  5523. CTLTYPE_STRING | CTLFLAG_RD, sc, 2, sysctl_tp_tick, "A",
  5524. "DACK tick (us)");
  5525. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer",
  5526. CTLTYPE_UINT | CTLFLAG_RD, sc, 0, sysctl_tp_dack_timer,
  5527. "IU", "DACK timer (us)");
  5528. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min",
  5529. CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MIN,
  5530. sysctl_tp_timer, "LU", "Minimum retransmit interval (us)");
  5531. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max",
  5532. CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MAX,
  5533. sysctl_tp_timer, "LU", "Maximum retransmit interval (us)");
  5534. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min",
  5535. CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MIN,
  5536. sysctl_tp_timer, "LU", "Persist timer min (us)");
  5537. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max",
  5538. CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MAX,
  5539. sysctl_tp_timer, "LU", "Persist timer max (us)");
  5540. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle",
  5541. CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_IDLE,
  5542. sysctl_tp_timer, "LU", "Keepalive idle timer (us)");
  5543. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_interval",
  5544. CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_INTVL,
  5545. sysctl_tp_timer, "LU", "Keepalive interval timer (us)");
  5546. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt",
  5547. CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_INIT_SRTT,
  5548. sysctl_tp_timer, "LU", "Initial SRTT (us)");
  5549. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer",
  5550. CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_FINWAIT2_TIMER,
  5551. sysctl_tp_timer, "LU", "FINWAIT2 timer (us)");
  5552. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "syn_rexmt_count",
  5553. CTLTYPE_UINT | CTLFLAG_RD, sc, S_SYNSHIFTMAX,
  5554. sysctl_tp_shift_cnt, "IU",
  5555. "Number of SYN retransmissions before abort");
  5556. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_count",
  5557. CTLTYPE_UINT | CTLFLAG_RD, sc, S_RXTSHIFTMAXR2,
  5558. sysctl_tp_shift_cnt, "IU",
  5559. "Number of retransmissions before abort");
  5560. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_count",
  5561. CTLTYPE_UINT | CTLFLAG_RD, sc, S_KEEPALIVEMAXR2,
  5562. sysctl_tp_shift_cnt, "IU",
  5563. "Number of keepalive probes before abort");
  5564. oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rexmt_backoff",
  5565. CTLFLAG_RD, NULL, "TOE retransmit backoffs");
  5566. children = SYSCTL_CHILDREN(oid);
  5567. for (i = 0; i < 16; i++) {
  5568. snprintf(s, sizeof(s), "%u", i);
  5569. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, s,
  5570. CTLTYPE_UINT | CTLFLAG_RD, sc, i, sysctl_tp_backoff,
  5571. "IU", "TOE retransmit backoff");
  5572. }
  5573. }
  5574. #endif
  5575. }
  5576. void
  5577. vi_sysctls(struct vi_info *vi)
  5578. {
  5579. struct sysctl_ctx_list *ctx;
  5580. struct sysctl_oid *oid;
  5581. struct sysctl_oid_list *children;
  5582. ctx = device_get_sysctl_ctx(vi->dev);
  5583. /*
  5584. * dev.v?(cxgbe|cxl).X.
  5585. */
  5586. oid = device_get_sysctl_tree(vi->dev);
  5587. children = SYSCTL_CHILDREN(oid);
  5588. SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL,
  5589. vi->viid, "VI identifer");
  5590. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
  5591. &vi->nrxq, 0, "# of rx queues");
  5592. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
  5593. &vi->ntxq, 0, "# of tx queues");
  5594. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
  5595. &vi->first_rxq, 0, "index of first rx queue");
  5596. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
  5597. &vi->first_txq, 0, "index of first tx queue");
  5598. SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_base", CTLFLAG_RD, NULL,
  5599. vi->rss_base, "start of RSS indirection table");
  5600. SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL,
  5601. vi->rss_size, "size of RSS indirection table");
  5602. if (IS_MAIN_VI(vi)) {
  5603. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq",
  5604. CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU",
  5605. "Reserve queue 0 for non-flowid packets");
  5606. }
  5607. #ifdef TCP_OFFLOAD
  5608. if (vi->nofldrxq != 0) {
  5609. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
  5610. &vi->nofldrxq, 0,
  5611. "# of rx queues for offloaded TCP connections");
  5612. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
  5613. CTLFLAG_RD, &vi->first_ofld_rxq, 0,
  5614. "index of first TOE rx queue");
  5615. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx_ofld",
  5616. CTLTYPE_INT | CTLFLAG_RW, vi, 0,
  5617. sysctl_holdoff_tmr_idx_ofld, "I",
  5618. "holdoff timer index for TOE queues");
  5619. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx_ofld",
  5620. CTLTYPE_INT | CTLFLAG_RW, vi, 0,
  5621. sysctl_holdoff_pktc_idx_ofld, "I",
  5622. "holdoff packet counter index for TOE queues");
  5623. }
  5624. #endif
  5625. #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
  5626. if (vi->nofldtxq != 0) {
  5627. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
  5628. &vi->nofldtxq, 0,
  5629. "# of tx queues for TOE/ETHOFLD");
  5630. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
  5631. CTLFLAG_RD, &vi->first_ofld_txq, 0,
  5632. "index of first TOE/ETHOFLD tx queue");
  5633. }
  5634. #endif
  5635. #ifdef DEV_NETMAP
  5636. if (vi->nnmrxq != 0) {
  5637. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
  5638. &vi->nnmrxq, 0, "# of netmap rx queues");
  5639. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
  5640. &vi->nnmtxq, 0, "# of netmap tx queues");
  5641. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
  5642. CTLFLAG_RD, &vi->first_nm_rxq, 0,
  5643. "index of first netmap rx queue");
  5644. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
  5645. CTLFLAG_RD, &vi->first_nm_txq, 0,
  5646. "index of first netmap tx queue");
  5647. }
  5648. #endif
  5649. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
  5650. CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I",
  5651. "holdoff timer index");
  5652. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
  5653. CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I",
  5654. "holdoff packet counter index");
  5655. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
  5656. CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I",
  5657. "rx queue size");
  5658. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
  5659. CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I",
  5660. "tx queue size");
  5661. }
  5662. static void
  5663. cxgbe_sysctls(struct port_info *pi)
  5664. {
  5665. struct sysctl_ctx_list *ctx;
  5666. struct sysctl_oid *oid;
  5667. struct sysctl_oid_list *children, *children2;
  5668. struct adapter *sc = pi->adapter;
  5669. int i;
  5670. char name[16];
  5671. static char *tc_flags = {"\20\1USER\2SYNC\3ASYNC\4ERR"};
  5672. ctx = device_get_sysctl_ctx(pi->dev);
  5673. /*
  5674. * dev.cxgbe.X.
  5675. */
  5676. oid = device_get_sysctl_tree(pi->dev);
  5677. children = SYSCTL_CHILDREN(oid);
  5678. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING |
  5679. CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down");
  5680. if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
  5681. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
  5682. CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I",
  5683. "PHY temperature (in Celsius)");
  5684. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
  5685. CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I",
  5686. "PHY firmware version");
  5687. }
  5688. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
  5689. CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_pause_settings, "A",
  5690. "PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)");
  5691. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fec",
  5692. CTLTYPE_STRING | CTLFLAG_RW, pi, 0, sysctl_fec, "A",
  5693. "FECs to use (bit 0 = RS, 1 = FC, 2 = none, 5 = auto, 6 = module)");
  5694. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "module_fec",
  5695. CTLTYPE_STRING, pi, 0, sysctl_module_fec, "A",
  5696. "FEC recommended by the cable/transceiver");
  5697. SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "autoneg",
  5698. CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_autoneg, "I",
  5699. "autonegotiation (-1 = not supported)");
  5700. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "pcaps", CTLFLAG_RD,
  5701. &pi->link_cfg.pcaps, 0, "port capabilities");
  5702. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "acaps", CTLFLAG_RD,
  5703. &pi->link_cfg.acaps, 0, "advertised capabilities");
  5704. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lpacaps", CTLFLAG_RD,
  5705. &pi->link_cfg.lpacaps, 0, "link partner advertised capabilities");
  5706. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL,
  5707. port_top_speed(pi), "max speed (in Gbps)");
  5708. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "mps_bg_map", CTLFLAG_RD, NULL,
  5709. pi->mps_bg_map, "MPS buffer group map");
  5710. SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_e_chan_map", CTLFLAG_RD,
  5711. NULL, pi->rx_e_chan_map, "TP rx e-channel map");
  5712. if (sc->flags & IS_VF)
  5713. return;
  5714. /*
  5715. * dev.(cxgbe|cxl).X.tc.
  5716. */
  5717. oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", CTLFLAG_RD, NULL,
  5718. "Tx scheduler traffic classes (cl_rl)");
  5719. children2 = SYSCTL_CHILDREN(oid);
  5720. SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "pktsize",
  5721. CTLFLAG_RW, &pi->sched_params->pktsize, 0,
  5722. "pktsize for per-flow cl-rl (0 means up to the driver )");
  5723. SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "burstsize",
  5724. CTLFLAG_RW, &pi->sched_params->burstsize, 0,
  5725. "burstsize for per-flow cl-rl (0 means up to the driver)");
  5726. for (i = 0; i < sc->chip_params->nsched_cls; i++) {
  5727. struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i];
  5728. snprintf(name, sizeof(name), "%d", i);
  5729. children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx,
  5730. SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD, NULL,
  5731. "traffic class"));
  5732. SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "flags",
  5733. CTLTYPE_STRING | CTLFLAG_RD, tc_flags, (uintptr_t)&tc->flags,
  5734. sysctl_bitfield_8b, "A", "flags");
  5735. SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount",
  5736. CTLFLAG_RD, &tc->refcount, 0, "references to this class");
  5737. SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params",
  5738. CTLTYPE_STRING | CTLFLAG_RD, sc, (pi->port_id << 16) | i,
  5739. sysctl_tc_params, "A", "traffic class parameters");
  5740. }
  5741. /*
  5742. * dev.cxgbe.X.stats.
  5743. */
  5744. oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
  5745. NULL, "port statistics");
  5746. children = SYSCTL_CHILDREN(oid);
  5747. SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
  5748. &pi->tx_parse_error, 0,
  5749. "# of tx packets with invalid length or # of segments");
  5750. #define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
  5751. SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
  5752. CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \
  5753. sysctl_handle_t4_reg64, "QU", desc)
  5754. SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
  5755. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
  5756. SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
  5757. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
  5758. SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
  5759. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
  5760. SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
  5761. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
  5762. SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
  5763. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
  5764. SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
  5765. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
  5766. SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
  5767. "# of tx frames in this range",
  5768. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
  5769. SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
  5770. "# of tx frames in this range",
  5771. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
  5772. SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
  5773. "# of tx frames in this range",
  5774. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
  5775. SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
  5776. "# of tx frames in this range",
  5777. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
  5778. SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
  5779. "# of tx frames in this range",
  5780. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
  5781. SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
  5782. "# of tx frames in this range",
  5783. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
  5784. SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
  5785. "# of tx frames in this range",
  5786. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
  5787. SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
  5788. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
  5789. SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
  5790. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
  5791. SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
  5792. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
  5793. SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
  5794. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
  5795. SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
  5796. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
  5797. SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
  5798. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
  5799. SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
  5800. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
  5801. SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
  5802. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
  5803. SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
  5804. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
  5805. SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
  5806. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
  5807. SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
  5808. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
  5809. SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
  5810. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
  5811. SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
  5812. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
  5813. SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
  5814. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
  5815. SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
  5816. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
  5817. SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
  5818. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
  5819. SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
  5820. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
  5821. SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
  5822. "# of frames received with bad FCS",
  5823. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
  5824. SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
  5825. "# of frames received with length error",
  5826. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
  5827. SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
  5828. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
  5829. SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
  5830. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
  5831. SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
  5832. "# of rx frames in this range",
  5833. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
  5834. SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
  5835. "# of rx frames in this range",
  5836. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
  5837. SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
  5838. "# of rx frames in this range",
  5839. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
  5840. SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
  5841. "# of rx frames in this range",
  5842. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
  5843. SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
  5844. "# of rx frames in this range",
  5845. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
  5846. SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
  5847. "# of rx frames in this range",
  5848. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
  5849. SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
  5850. "# of rx frames in this range",
  5851. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
  5852. SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
  5853. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
  5854. SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
  5855. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
  5856. SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
  5857. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
  5858. SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
  5859. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
  5860. SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
  5861. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
  5862. SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
  5863. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
  5864. SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
  5865. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
  5866. SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
  5867. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
  5868. SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
  5869. PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
  5870. #undef SYSCTL_ADD_T4_REG64
  5871. #define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
  5872. SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
  5873. &pi->stats.name, desc)
  5874. /* We get these from port_stats and they may be stale by up to 1s */
  5875. SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
  5876. "# drops due to buffer-group 0 overflows");
  5877. SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
  5878. "# drops due to buffer-group 1 overflows");
  5879. SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
  5880. "# drops due to buffer-group 2 overflows");
  5881. SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
  5882. "# drops due to buffer-group 3 overflows");
  5883. SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
  5884. "# of buffer-group 0 truncated packets");
  5885. SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
  5886. "# of buffer-group 1 truncated packets");
  5887. SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
  5888. "# of buffer-group 2 truncated packets");
  5889. SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
  5890. "# of buffer-group 3 truncated packets");
  5891. #undef SYSCTL_ADD_T4_PORTSTAT
  5892. SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_toe_tls_records",
  5893. CTLFLAG_RD, &pi->tx_toe_tls_records,
  5894. "# of TLS records transmitted");
  5895. SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "tx_toe_tls_octets",
  5896. CTLFLAG_RD, &pi->tx_toe_tls_octets,
  5897. "# of payload octets in transmitted TLS records");
  5898. SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_toe_tls_records",
  5899. CTLFLAG_RD, &pi->rx_toe_tls_records,
  5900. "# of TLS records received");
  5901. SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, "rx_toe_tls_octets",
  5902. CTLFLAG_RD, &pi->rx_toe_tls_octets,
  5903. "# of payload octets in received TLS records");
  5904. }
  5905. static int
  5906. sysctl_int_array(SYSCTL_HANDLER_ARGS)
  5907. {
  5908. int rc, *i, space = 0;
  5909. struct sbuf sb;
  5910. sbuf_new_for_sysctl(&sb, NULL, 64, req);
  5911. for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
  5912. if (space)
  5913. sbuf_printf(&sb, " ");
  5914. sbuf_printf(&sb, "%d", *i);
  5915. space = 1;
  5916. }
  5917. rc = sbuf_finish(&sb);
  5918. sbuf_delete(&sb);
  5919. return (rc);
  5920. }
  5921. static int
  5922. sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS)
  5923. {
  5924. int rc;
  5925. struct sbuf *sb;
  5926. rc = sysctl_wire_old_buffer(req, 0);
  5927. if (rc != 0)
  5928. return(rc);
  5929. sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
  5930. if (sb == NULL)
  5931. return (ENOMEM);
  5932. sbuf_printf(sb, "%b", *(uint8_t *)(uintptr_t)arg2, (char *)arg1);
  5933. rc = sbuf_finish(sb);
  5934. sbuf_delete(sb);
  5935. return (rc);
  5936. }
  5937. static int
  5938. sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS)
  5939. {
  5940. int rc;
  5941. struct sbuf *sb;
  5942. rc = sysctl_wire_old_buffer(req, 0);
  5943. if (rc != 0)
  5944. return(rc);
  5945. sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
  5946. if (sb == NULL)
  5947. return (ENOMEM);
  5948. sbuf_printf(sb, "%b", *(uint16_t *)(uintptr_t)arg2, (char *)arg1);
  5949. rc = sbuf_finish(sb);
  5950. sbuf_delete(sb);
  5951. return (rc);
  5952. }
  5953. static int
  5954. sysctl_btphy(SYSCTL_HANDLER_ARGS)
  5955. {
  5956. struct port_info *pi = arg1;
  5957. int op = arg2;
  5958. struct adapter *sc = pi->adapter;
  5959. u_int v;
  5960. int rc;
  5961. rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
  5962. if (rc)
  5963. return (rc);
  5964. /* XXX: magic numbers */
  5965. rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820,
  5966. &v);
  5967. end_synchronized_op(sc, 0);
  5968. if (rc)
  5969. return (rc);
  5970. if (op == 0)
  5971. v /= 256;
  5972. rc = sysctl_handle_int(oidp, &v, 0, req);
  5973. return (rc);
  5974. }
  5975. static int
  5976. sysctl_noflowq(SYSCTL_HANDLER_ARGS)
  5977. {
  5978. struct vi_info *vi = arg1;
  5979. int rc, val;
  5980. val = vi->rsrv_noflowq;
  5981. rc = sysctl_handle_int(oidp, &val, 0, req);
  5982. if (rc != 0 || req->newptr == NULL)
  5983. return (rc);
  5984. if ((val >= 1) && (vi->ntxq > 1))
  5985. vi->rsrv_noflowq = 1;
  5986. else
  5987. vi->rsrv_noflowq = 0;
  5988. return (rc);
  5989. }
  5990. static int
  5991. sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
  5992. {
  5993. struct vi_info *vi = arg1;
  5994. struct adapter *sc = vi->pi->adapter;
  5995. int idx, rc, i;
  5996. struct sge_rxq *rxq;
  5997. uint8_t v;
  5998. idx = vi->tmr_idx;
  5999. rc = sysctl_handle_int(oidp, &idx, 0, req);
  6000. if (rc != 0 || req->newptr == NULL)
  6001. return (rc);
  6002. if (idx < 0 || idx >= SGE_NTIMERS)
  6003. return (EINVAL);
  6004. rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
  6005. "t4tmr");
  6006. if (rc)
  6007. return (rc);
  6008. v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
  6009. for_each_rxq(vi, i, rxq) {
  6010. #ifdef atomic_store_rel_8
  6011. atomic_store_rel_8(&rxq->iq.intr_params, v);
  6012. #else
  6013. rxq->iq.intr_params = v;
  6014. #endif
  6015. }
  6016. vi->tmr_idx = idx;
  6017. end_synchronized_op(sc, LOCK_HELD);
  6018. return (0);
  6019. }
  6020. static int
  6021. sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
  6022. {
  6023. struct vi_info *vi = arg1;
  6024. struct adapter *sc = vi->pi->adapter;
  6025. int idx, rc;
  6026. idx = vi->pktc_idx;
  6027. rc = sysctl_handle_int(oidp, &idx, 0, req);
  6028. if (rc != 0 || req->newptr == NULL)
  6029. return (rc);
  6030. if (idx < -1 || idx >= SGE_NCOUNTERS)
  6031. return (EINVAL);
  6032. rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
  6033. "t4pktc");
  6034. if (rc)
  6035. return (rc);
  6036. if (vi->flags & VI_INIT_DONE)
  6037. rc = EBUSY; /* cannot be changed once the queues are created */
  6038. else
  6039. vi->pktc_idx = idx;
  6040. end_synchronized_op(sc, LOCK_HELD);
  6041. return (rc);
  6042. }
  6043. static int
  6044. sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
  6045. {
  6046. struct vi_info *vi = arg1;
  6047. struct adapter *sc = vi->pi->adapter;
  6048. int qsize, rc;
  6049. qsize = vi->qsize_rxq;
  6050. rc = sysctl_handle_int(oidp, &qsize, 0, req);
  6051. if (rc != 0 || req->newptr == NULL)
  6052. return (rc);
  6053. if (qsize < 128 || (qsize & 7))
  6054. return (EINVAL);
  6055. rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
  6056. "t4rxqs");
  6057. if (rc)
  6058. return (rc);
  6059. if (vi->flags & VI_INIT_DONE)
  6060. rc = EBUSY; /* cannot be changed once the queues are created */
  6061. else
  6062. vi->qsize_rxq = qsize;
  6063. end_synchronized_op(sc, LOCK_HELD);
  6064. return (rc);
  6065. }
  6066. static int
  6067. sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
  6068. {
  6069. struct vi_info *vi = arg1;
  6070. struct adapter *sc = vi->pi->adapter;
  6071. int qsize, rc;
  6072. qsize = vi->qsize_txq;
  6073. rc = sysctl_handle_int(oidp, &qsize, 0, req);
  6074. if (rc != 0 || req->newptr == NULL)
  6075. return (rc);
  6076. if (qsize < 128 || qsize > 65536)
  6077. return (EINVAL);
  6078. rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
  6079. "t4txqs");
  6080. if (rc)
  6081. return (rc);
  6082. if (vi->flags & VI_INIT_DONE)
  6083. rc = EBUSY; /* cannot be changed once the queues are created */
  6084. else
  6085. vi->qsize_txq = qsize;
  6086. end_synchronized_op(sc, LOCK_HELD);
  6087. return (rc);
  6088. }
  6089. static int
  6090. sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
  6091. {
  6092. struct port_info *pi = arg1;
  6093. struct adapter *sc = pi->adapter;
  6094. struct link_config *lc = &pi->link_cfg;
  6095. int rc;
  6096. if (req->newptr == NULL) {
  6097. struct sbuf *sb;
  6098. static char *bits = "\20\1RX\2TX\3AUTO";
  6099. rc = sysctl_wire_old_buffer(req, 0);
  6100. if (rc != 0)
  6101. return(rc);
  6102. sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
  6103. if (sb == NULL)
  6104. return (ENOMEM);
  6105. if (lc->link_ok) {
  6106. sbuf_printf(sb, "%b", (lc->fc & (PAUSE_TX | PAUSE_RX)) |
  6107. (lc->requested_fc & PAUSE_AUTONEG), bits);
  6108. } else {
  6109. sbuf_printf(sb, "%b", lc->requested_fc & (PAUSE_TX |
  6110. PAUSE_RX | PAUSE_AUTONEG), bits);
  6111. }
  6112. rc = sbuf_finish(sb);
  6113. sbuf_delete(sb);
  6114. } else {
  6115. char s[2];
  6116. int n;
  6117. s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX |
  6118. PAUSE_AUTONEG));
  6119. s[1] = 0;
  6120. rc = sysctl_handle_string(oidp, s, sizeof(s), req);
  6121. if (rc != 0)
  6122. return(rc);
  6123. if (s[1] != 0)
  6124. return (EINVAL);
  6125. if (s[0] < '0' || s[0] > '9')
  6126. return (EINVAL); /* not a number */
  6127. n = s[0] - '0';
  6128. if (n & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG))
  6129. return (EINVAL); /* some other bit is set too */
  6130. rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
  6131. "t4PAUSE");
  6132. if (rc)
  6133. return (rc);
  6134. PORT_LOCK(pi);
  6135. lc->requested_fc = n;
  6136. fixup_link_config(pi);
  6137. if (pi->up_vis > 0)
  6138. rc = apply_link_config(pi);
  6139. set_current_media(pi);
  6140. PORT_UNLOCK(pi);
  6141. end_synchronized_op(sc, 0);
  6142. }
  6143. return (rc);
  6144. }
  6145. static int
  6146. sysctl_fec(SYSCTL_HANDLER_ARGS)
  6147. {
  6148. struct port_info *pi = arg1;
  6149. struct adapter *sc = pi->adapter;
  6150. struct link_config *lc = &pi->link_cfg;
  6151. int rc;
  6152. int8_t old;
  6153. if (req->newptr == NULL) {
  6154. struct sbuf *sb;
  6155. static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2"
  6156. "\5RSVD3\6auto\7module";
  6157. rc = sysctl_wire_old_buffer(req, 0);
  6158. if (rc != 0)
  6159. return(rc);
  6160. sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
  6161. if (sb == NULL)
  6162. return (ENOMEM);
  6163. /*
  6164. * Display the requested_fec when the link is down -- the actual
  6165. * FEC makes sense only when the link is up.
  6166. */
  6167. if (lc->link_ok) {
  6168. sbuf_printf(sb, "%b", (lc->fec & M_FW_PORT_CAP32_FEC) |
  6169. (lc->requested_fec & (FEC_AUTO | FEC_MODULE)),
  6170. bits);
  6171. } else {
  6172. sbuf_printf(sb, "%b", lc->requested_fec, bits);
  6173. }
  6174. rc = sbuf_finish(sb);
  6175. sbuf_delete(sb);
  6176. } else {
  6177. char s[8];
  6178. int n;
  6179. snprintf(s, sizeof(s), "%d",
  6180. lc->requested_fec == FEC_AUTO ? -1 :
  6181. lc->requested_fec & (M_FW_PORT_CAP32_FEC | FEC_MODULE));
  6182. rc = sysctl_handle_string(oidp, s, sizeof(s), req);
  6183. if (rc != 0)
  6184. return(rc);
  6185. n = strtol(&s[0], NULL, 0);
  6186. if (n < 0 || n & FEC_AUTO)
  6187. n = FEC_AUTO;
  6188. else if (n & ~(M_FW_PORT_CAP32_FEC | FEC_MODULE))
  6189. return (EINVAL);/* some other bit is set too */
  6190. rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
  6191. "t4fec");
  6192. if (rc)
  6193. return (rc);
  6194. PORT_LOCK(pi);
  6195. old = lc->requested_fec;
  6196. if (n == FEC_AUTO)
  6197. lc->requested_fec = FEC_AUTO;
  6198. else if (n == 0 || n == FEC_NONE)
  6199. lc->requested_fec = FEC_NONE;
  6200. else {
  6201. if ((lc->pcaps |
  6202. V_FW_PORT_CAP32_FEC(n & M_FW_PORT_CAP32_FEC)) !=
  6203. lc->pcaps) {
  6204. rc = ENOTSUP;
  6205. goto done;
  6206. }
  6207. lc->requested_fec = n & (M_FW_PORT_CAP32_FEC |
  6208. FEC_MODULE);
  6209. }
  6210. fixup_link_config(pi);
  6211. if (pi->up_vis > 0) {
  6212. rc = apply_link_config(pi);
  6213. if (rc != 0) {
  6214. lc->requested_fec = old;
  6215. if (rc == FW_EPROTO)
  6216. rc = ENOTSUP;
  6217. }
  6218. }
  6219. done:
  6220. PORT_UNLOCK(pi);
  6221. end_synchronized_op(sc, 0);
  6222. }
  6223. return (rc);
  6224. }
  6225. static int
  6226. sysctl_module_fec(SYSCTL_HANDLER_ARGS)
  6227. {
  6228. struct port_info *pi = arg1;
  6229. struct adapter *sc = pi->adapter;
  6230. struct link_config *lc = &pi->link_cfg;
  6231. int rc;
  6232. int8_t fec;
  6233. struct sbuf *sb;
  6234. static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2\5RSVD3";
  6235. rc = sysctl_wire_old_buffer(req, 0);
  6236. if (rc != 0)
  6237. return (rc);
  6238. sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
  6239. if (sb == NULL)
  6240. return (ENOMEM);
  6241. if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mfec") != 0)
  6242. return (EBUSY);
  6243. PORT_LOCK(pi);
  6244. if (pi->up_vis == 0) {
  6245. /*
  6246. * If all the interfaces are administratively down the firmware
  6247. * does not report transceiver changes. Refresh port info here.
  6248. * This is the only reason we have a synchronized op in this
  6249. * function. Just PORT_LOCK would have been enough otherwise.
  6250. */
  6251. t4_update_port_info(pi);
  6252. }
  6253. fec = lc->fec_hint;
  6254. if (pi->mod_type == FW_PORT_MOD_TYPE_NONE ||
  6255. !fec_supported(lc->pcaps)) {
  6256. sbuf_printf(sb, "n/a");
  6257. } else {
  6258. if (fec == 0)
  6259. fec = FEC_NONE;
  6260. sbuf_printf(sb, "%b", fec & M_FW_PORT_CAP32_FEC, bits);
  6261. }
  6262. rc = sbuf_finish(sb);
  6263. sbuf_delete(sb);
  6264. PORT_UNLOCK(pi);
  6265. end_synchronized_op(sc, 0);
  6266. return (rc);
  6267. }
  6268. static int
  6269. sysctl_autoneg(SYSCTL_HANDLER_ARGS)
  6270. {
  6271. struct port_info *pi = arg1;
  6272. struct adapter *sc = pi->adapter;
  6273. struct link_config *lc = &pi->link_cfg;
  6274. int rc, val;
  6275. if (lc->pcaps & FW_PORT_CAP32_ANEG)
  6276. val = lc->requested_aneg == AUTONEG_DISABLE ? 0 : 1;
  6277. else
  6278. val = -1;
  6279. rc = sysctl_handle_int(oidp, &val, 0, req);
  6280. if (rc != 0 || req->newptr == NULL)
  6281. return (rc);
  6282. if (val == 0)
  6283. val = AUTONEG_DISABLE;
  6284. else if (val == 1)
  6285. val = AUTONEG_ENABLE;
  6286. else
  6287. val = AUTONEG_AUTO;
  6288. rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
  6289. "t4aneg");
  6290. if (rc)
  6291. return (rc);
  6292. PORT_LOCK(pi);
  6293. if (val == AUTONEG_ENABLE && !(lc->pcaps & FW_PORT_CAP32_ANEG)) {
  6294. rc = ENOTSUP;
  6295. goto done;
  6296. }
  6297. lc->requested_aneg = val;
  6298. fixup_link_config(pi);
  6299. if (pi->up_vis > 0)
  6300. rc = apply_link_config(pi);
  6301. set_current_media(pi);
  6302. done:
  6303. PORT_UNLOCK(pi);
  6304. end_synchronized_op(sc, 0);
  6305. return (rc);
  6306. }
  6307. static int
  6308. sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
  6309. {
  6310. struct adapter *sc = arg1;
  6311. int reg = arg2;
  6312. uint64_t val;
  6313. val = t4_read_reg64(sc, reg);
  6314. return (sysctl_handle_64(oidp, &val, 0, req));
  6315. }
  6316. static int
  6317. sysctl_temperature(SYSCTL_HANDLER_ARGS)
  6318. {
  6319. struct adapter *sc = arg1;
  6320. int rc, t;
  6321. uint32_t param, val;
  6322. rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
  6323. if (rc)
  6324. return (rc);
  6325. param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
  6326. V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
  6327. V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
  6328. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
  6329. end_synchronized_op(sc, 0);
  6330. if (rc)
  6331. return (rc);
  6332. /* unknown is returned as 0 but we display -1 in that case */
  6333. t = val == 0 ? -1 : val;
  6334. rc = sysctl_handle_int(oidp, &t, 0, req);
  6335. return (rc);
  6336. }
  6337. static int
  6338. sysctl_vdd(SYSCTL_HANDLER_ARGS)
  6339. {
  6340. struct adapter *sc = arg1;
  6341. int rc;
  6342. uint32_t param, val;
  6343. if (sc->params.core_vdd == 0) {
  6344. rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
  6345. "t4vdd");
  6346. if (rc)
  6347. return (rc);
  6348. param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
  6349. V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
  6350. V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD);
  6351. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
  6352. end_synchronized_op(sc, 0);
  6353. if (rc)
  6354. return (rc);
  6355. sc->params.core_vdd = val;
  6356. }
  6357. return (sysctl_handle_int(oidp, &sc->params.core_vdd, 0, req));
  6358. }
  6359. static int
  6360. sysctl_reset_sensor(SYSCTL_HANDLER_ARGS)
  6361. {
  6362. struct adapter *sc = arg1;
  6363. int rc, v;
  6364. uint32_t param, val;
  6365. v = sc->sensor_resets;
  6366. rc = sysctl_handle_int(oidp, &v, 0, req);
  6367. if (rc != 0 || req->newptr == NULL || v <= 0)
  6368. return (rc);
  6369. if (sc->params.fw_vers < FW_VERSION32(1, 24, 7, 0) ||
  6370. chip_id(sc) < CHELSIO_T5)
  6371. return (ENOTSUP);
  6372. rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4srst");
  6373. if (rc)
  6374. return (rc);
  6375. param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
  6376. V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
  6377. V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_RESET_TMP_SENSOR));
  6378. val = 1;
  6379. rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
  6380. end_synchronized_op(sc, 0);
  6381. if (rc == 0)
  6382. sc->sensor_resets++;
  6383. return (rc);
  6384. }
  6385. static int
  6386. sysctl_loadavg(SYSCTL_HANDLER_ARGS)
  6387. {
  6388. struct adapter *sc = arg1;
  6389. struct sbuf *sb;
  6390. int rc;
  6391. uint32_t param, val;
  6392. rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4lavg");
  6393. if (rc)
  6394. return (rc);
  6395. param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
  6396. V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_LOAD);
  6397. rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
  6398. end_synchronized_op(sc, 0);
  6399. if (rc)
  6400. return (rc);
  6401. rc = sysctl_wire_old_buffer(req, 0);
  6402. if (rc != 0)
  6403. return (rc);
  6404. sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
  6405. if (sb == NULL)
  6406. return (ENOMEM);
  6407. if (val == 0xffffffff) {
  6408. /* Only debug and custom firmwares report load averages. */
  6409. sbuf_printf(sb, "not available");
  6410. } else {
  6411. sbuf_printf(sb, "%d %d %d", val & 0xff, (val >> 8) & 0xff,
  6412. (val >> 16) & 0xff);
  6413. }
  6414. rc = sbuf_finish(sb);
  6415. sbuf_delete(sb);
  6416. return (rc);
  6417. }
  6418. static int
  6419. sysctl_cctrl(SYSCTL_HANDLER_ARGS)
  6420. {
  6421. struct adapter *sc = arg1;
  6422. struct sbuf *sb;
  6423. int rc, i;
  6424. uint16_t incr[NMTUS][NCCTRL_WIN];
  6425. static const char *dec_fac[] = {
  6426. "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
  6427. "0.9375"
  6428. };
  6429. rc = sysctl_wire_old_buffer(req, 0);
  6430. if (rc != 0)
  6431. return (rc);
  6432. sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
  6433. if (sb == NULL)
  6434. return (ENOMEM);
  6435. t4_read_cong_tbl(sc, incr);
  6436. for (i = 0; i < NCCTRL_WIN; ++i) {
  6437. sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
  6438. incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
  6439. incr[5][i], incr[6][i], incr[7][i]);
  6440. sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
  6441. incr[8][i], incr[9][i], incr[10][i], incr[11][i],
  6442. incr[12][i], incr[13][i], incr[14][i], incr[15][i],
  6443. sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
  6444. }
  6445. rc = sbuf_finish(sb);
  6446. sbuf_delete(sb);
  6447. return (rc);
  6448. }
  6449. static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
  6450. "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */
  6451. "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
  6452. "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */
  6453. };
  6454. static int
  6455. sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
  6456. {
  6457. struct adapter *sc = arg1;
  6458. struct sbuf *sb;
  6459. int rc, i, n, qid = arg2;
  6460. uint32_t *buf, *p;
  6461. char *qtype;
  6462. u_int cim_num_obq = sc->chip_params->cim_num_obq;
  6463. KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
  6464. ("%s: bad qid %d\n", __func__, qid));
  6465. if (qid < CIM_NUM_IBQ) {
  6466. /* inbound queue */
  6467. qtype = "IBQ";
  6468. n = 4 * CIM_IBQ_SIZE;
  6469. buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
  6470. rc = t4_read_cim_ibq(sc, qid, buf, n);
  6471. } else {
  6472. /* outbound queue */
  6473. qtype = "OBQ";
  6474. qid -= CIM_NUM_IBQ;
  6475. n = 4 * cim_num_obq * CIM_OBQ_SIZE;
  6476. buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
  6477. rc = t4_read_cim_obq(sc, qid, buf, n);
  6478. }
  6479. if (rc < 0) {
  6480. rc = -rc;
  6481. goto done;
  6482. }
  6483. n = rc * sizeof(uint32_t); /* rc has # of words actually read */
  6484. rc = sysctl_wire_old_buffer(req, 0);
  6485. if (rc != 0)
  6486. goto done;
  6487. sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
  6488. if (sb == NULL) {
  6489. rc = ENOMEM;
  6490. goto done;
  6491. }
  6492. sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
  6493. for (i = 0, p = buf; i < n; i += 16, p += 4)
  6494. sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
  6495. p[2], p[3]);
  6496. rc = sbuf_finish(sb);
  6497. sbuf_delete(sb);
  6498. done:
  6499. free(buf, M_CXGBE);
  6500. return (rc);
  6501. }
  6502. static void
  6503. sbuf_cim_la4(struct adapter *sc, struct sbuf *sb, uint32_t *buf, uint32_t cfg)
  6504. {
  6505. uint32_t *p;
  6506. sbuf_printf(sb, "Status Data PC%s",
  6507. cfg & F_UPDBGLACAPTPCONLY ? "" :
  6508. " LS0Stat LS0Addr LS0Data");
  6509. for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) {
  6510. if (cfg & F_UPDBGLACAPTPCONLY) {
  6511. sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff,
  6512. p[6], p[7]);
  6513. sbuf_printf(sb, "\n %02x %02x%06x %02x%06x",
  6514. (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
  6515. p[4] & 0xff, p[5] >> 8);
  6516. sbuf_printf(sb, "\n %02x %x%07x %x%07x",
  6517. (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
  6518. p[1] & 0xf, p[2] >> 4);
  6519. } else {
  6520. sbuf_printf(sb,
  6521. "\n %02x %x%07x %x%07x %08x %08x "
  6522. "%08x%08x%08x%08x",
  6523. (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
  6524. p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
  6525. p[6], p[7]);
  6526. }
  6527. }
  6528. }
  6529. static void
  6530. sbuf_cim_la6(struct adapter *sc, struct sbuf *sb, uint32_t *buf, uint32_t cfg)
  6531. {
  6532. uint32_t *p;
  6533. sbuf_printf(sb, "Status Inst Data PC%s",
  6534. cfg & F_UPDBGLACAPTPCONLY ? "" :
  6535. " LS0Stat LS0Addr LS0Data LS1Stat LS1Addr LS1Data");
  6536. for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) {
  6537. if (cfg & F_UPDBGLACAPTPCONLY) {
  6538. sbuf_printf(sb, "\n %02x %08x %08x %08x",
  6539. p[3] & 0xff, p[2], p[1], p[0]);
  6540. sbuf_printf(sb, "\n %02x %02x%06x %02x%06x %02x%06x",
  6541. (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8,
  6542. p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8);
  6543. sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x",
  6544. (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16,
  6545. p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff,
  6546. p[6] >> 16);
  6547. } else {
  6548. sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x "
  6549. "%08x %08x %08x %08x %08x %08x",
  6550. (p[9] >> 16) & 0xff,
  6551. p[9] & 0xffff, p[8] >> 16,
  6552. p[8] & 0xffff, p[7] >> 16,
  6553. p[7] & 0xffff, p[6] >> 16,
  6554. p[2], p[1], p[0], p[5], p[4], p[3]);
  6555. }
  6556. }
  6557. }
  6558. static int
  6559. sbuf_cim_la(struct adapter *sc, struct sbuf *sb, int flags)
  6560. {
  6561. uint32_t cfg, *buf;
  6562. int rc;
  6563. rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
  6564. if (rc != 0)
  6565. return (rc);
  6566. MPASS(flags == M_WAITOK || flags == M_NOWAIT);
  6567. buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
  6568. M_ZERO | flags);
  6569. if (buf == NULL)
  6570. return (ENOMEM);
  6571. rc = -t4_cim_read_la(sc, buf, NULL);
  6572. if (rc != 0)
  6573. goto done;
  6574. if (chip_id(sc) < CHELSIO_T6)
  6575. sbuf_cim_la4(sc, sb, buf, cfg);
  6576. else
  6577. sbuf_cim_la6(sc, sb, buf, cfg);
  6578. done:
  6579. free(buf, M_CXGBE);
  6580. return (rc);
  6581. }
  6582. static int
  6583. sysctl_cim_la(SYSCTL_HANDLER_ARGS)
  6584. {
  6585. struct adapter *sc = arg1;
  6586. struct sbuf *sb;
  6587. int rc;
  6588. rc = sysctl_wire_old_buffer(req, 0);
  6589. if (rc != 0)
  6590. return (rc);
  6591. sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
  6592. if (sb == NULL)
  6593. return (ENOMEM);
  6594. rc = sbuf_cim_la(sc, sb, M_WAITOK);
  6595. if (rc == 0)
  6596. rc = sbuf_finish(sb);
  6597. sbuf_delete(sb);
  6598. return (rc);
  6599. }
  6600. bool
  6601. t4_os_dump_cimla(struct adapter *sc, int arg, bool verbose)
  6602. {
  6603. struct sbuf sb;
  6604. int rc;
  6605. if (sbuf_new(&sb, NULL, 4096, SBUF_AUTOEXTEND) != &sb)
  6606. return (false);
  6607. rc = sbuf_cim_la(sc, &sb, M_NOWAIT);
  6608. if (rc == 0) {
  6609. rc = sbuf_finish(&sb);
  6610. if (rc == 0) {
  6611. log(LOG_DEBUG, "%s: CIM LA dump follows.\n%s",
  6612. device_get_nameunit(sc->dev), sbuf_data(&sb));
  6613. }
  6614. }
  6615. sbuf_delete(&sb);
  6616. return (false);
  6617. }
  6618. static int
  6619. sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
  6620. {
  6621. struct adapter *sc = arg1;
  6622. u_int i;
  6623. struct sbuf *sb;
  6624. uint32_t *buf, *p;
  6625. int rc;
  6626. rc = sysctl_wire_old_buffer(req, 0);
  6627. if (rc != 0)
  6628. return (rc);
  6629. sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
  6630. if (sb == NULL)
  6631. return (ENOMEM);
  6632. buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
  6633. M_ZERO | M_WAITOK);
  6634. t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
  6635. p = buf;
  6636. for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
  6637. sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
  6638. p[1], p[0]);
  6639. }
  6640. sbuf_printf(sb, "\n\