HardenedBSD src tree https://hardenedbsd.org/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

3661 lines
90 KiB

  1. /**************************************************************************
  2. SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  3. Copyright (c) 2007-2009, Chelsio Inc.
  4. All rights reserved.
  5. Redistribution and use in source and binary forms, with or without
  6. modification, are permitted provided that the following conditions are met:
  7. 1. Redistributions of source code must retain the above copyright notice,
  8. this list of conditions and the following disclaimer.
  9. 2. Neither the name of the Chelsio Corporation nor the names of its
  10. contributors may be used to endorse or promote products derived from
  11. this software without specific prior written permission.
  12. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  13. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  14. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  15. ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  16. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  17. CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  18. SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  19. INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  20. CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  21. ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  22. POSSIBILITY OF SUCH DAMAGE.
  23. ***************************************************************************/
  24. #include <sys/cdefs.h>
  25. __FBSDID("$FreeBSD$");
  26. #include "opt_inet.h"
  27. #include <sys/param.h>
  28. #include <sys/systm.h>
  29. #include <sys/kernel.h>
  30. #include <sys/bus.h>
  31. #include <sys/module.h>
  32. #include <sys/pciio.h>
  33. #include <sys/conf.h>
  34. #include <machine/bus.h>
  35. #include <machine/resource.h>
  36. #include <sys/ktr.h>
  37. #include <sys/rman.h>
  38. #include <sys/ioccom.h>
  39. #include <sys/mbuf.h>
  40. #include <sys/linker.h>
  41. #include <sys/firmware.h>
  42. #include <sys/socket.h>
  43. #include <sys/sockio.h>
  44. #include <sys/smp.h>
  45. #include <sys/sysctl.h>
  46. #include <sys/syslog.h>
  47. #include <sys/queue.h>
  48. #include <sys/taskqueue.h>
  49. #include <sys/proc.h>
  50. #include <net/bpf.h>
  51. #include <net/ethernet.h>
  52. #include <net/if.h>
  53. #include <net/if_var.h>
  54. #include <net/if_arp.h>
  55. #include <net/if_dl.h>
  56. #include <net/if_media.h>
  57. #include <net/if_types.h>
  58. #include <net/if_vlan_var.h>
  59. #include <netinet/in_systm.h>
  60. #include <netinet/in.h>
  61. #include <netinet/if_ether.h>
  62. #include <netinet/ip.h>
  63. #include <netinet/ip.h>
  64. #include <netinet/tcp.h>
  65. #include <netinet/udp.h>
  66. #include <netinet/netdump/netdump.h>
  67. #include <dev/pci/pcireg.h>
  68. #include <dev/pci/pcivar.h>
  69. #include <dev/pci/pci_private.h>
  70. #include <cxgb_include.h>
  71. #ifdef PRIV_SUPPORTED
  72. #include <sys/priv.h>
  73. #endif
  74. static int cxgb_setup_interrupts(adapter_t *);
  75. static void cxgb_teardown_interrupts(adapter_t *);
  76. static void cxgb_init(void *);
  77. static int cxgb_init_locked(struct port_info *);
  78. static int cxgb_uninit_locked(struct port_info *);
  79. static int cxgb_uninit_synchronized(struct port_info *);
  80. static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
  81. static int cxgb_media_change(struct ifnet *);
  82. static int cxgb_ifm_type(int);
  83. static void cxgb_build_medialist(struct port_info *);
  84. static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
  85. static uint64_t cxgb_get_counter(struct ifnet *, ift_counter);
  86. static int setup_sge_qsets(adapter_t *);
  87. static void cxgb_async_intr(void *);
  88. static void cxgb_tick_handler(void *, int);
  89. static void cxgb_tick(void *);
  90. static void link_check_callout(void *);
  91. static void check_link_status(void *, int);
  92. static void setup_rss(adapter_t *sc);
  93. static int alloc_filters(struct adapter *);
  94. static int setup_hw_filters(struct adapter *);
  95. static int set_filter(struct adapter *, int, const struct filter_info *);
  96. static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
  97. unsigned int, u64, u64);
  98. static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
  99. unsigned int, u64, u64);
  100. #ifdef TCP_OFFLOAD
  101. static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
  102. #endif
  103. /* Attachment glue for the PCI controller end of the device. Each port of
  104. * the device is attached separately, as defined later.
  105. */
  106. static int cxgb_controller_probe(device_t);
  107. static int cxgb_controller_attach(device_t);
  108. static int cxgb_controller_detach(device_t);
  109. static void cxgb_free(struct adapter *);
  110. static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
  111. unsigned int end);
  112. static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
  113. static int cxgb_get_regs_len(void);
  114. static void touch_bars(device_t dev);
  115. static void cxgb_update_mac_settings(struct port_info *p);
  116. #ifdef TCP_OFFLOAD
  117. static int toe_capability(struct port_info *, int);
  118. #endif
  119. /* Table for probing the cards. The desc field isn't actually used */
  120. struct cxgb_ident {
  121. uint16_t vendor;
  122. uint16_t device;
  123. int index;
  124. char *desc;
  125. } cxgb_identifiers[] = {
  126. {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
  127. {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
  128. {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
  129. {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
  130. {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
  131. {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
  132. {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
  133. {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
  134. {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
  135. {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
  136. {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
  137. {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
  138. {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
  139. {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
  140. {0, 0, 0, NULL}
  141. };
  142. static device_method_t cxgb_controller_methods[] = {
  143. DEVMETHOD(device_probe, cxgb_controller_probe),
  144. DEVMETHOD(device_attach, cxgb_controller_attach),
  145. DEVMETHOD(device_detach, cxgb_controller_detach),
  146. DEVMETHOD_END
  147. };
  148. static driver_t cxgb_controller_driver = {
  149. "cxgbc",
  150. cxgb_controller_methods,
  151. sizeof(struct adapter)
  152. };
  153. static int cxgbc_mod_event(module_t, int, void *);
  154. static devclass_t cxgb_controller_devclass;
  155. DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
  156. cxgbc_mod_event, 0);
  157. MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers,
  158. nitems(cxgb_identifiers) - 1);
  159. MODULE_VERSION(cxgbc, 1);
  160. MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
  161. /*
  162. * Attachment glue for the ports. Attachment is done directly to the
  163. * controller device.
  164. */
  165. static int cxgb_port_probe(device_t);
  166. static int cxgb_port_attach(device_t);
  167. static int cxgb_port_detach(device_t);
  168. static device_method_t cxgb_port_methods[] = {
  169. DEVMETHOD(device_probe, cxgb_port_probe),
  170. DEVMETHOD(device_attach, cxgb_port_attach),
  171. DEVMETHOD(device_detach, cxgb_port_detach),
  172. { 0, 0 }
  173. };
  174. static driver_t cxgb_port_driver = {
  175. "cxgb",
  176. cxgb_port_methods,
  177. 0
  178. };
  179. static d_ioctl_t cxgb_extension_ioctl;
  180. static d_open_t cxgb_extension_open;
  181. static d_close_t cxgb_extension_close;
  182. static struct cdevsw cxgb_cdevsw = {
  183. .d_version = D_VERSION,
  184. .d_flags = 0,
  185. .d_open = cxgb_extension_open,
  186. .d_close = cxgb_extension_close,
  187. .d_ioctl = cxgb_extension_ioctl,
  188. .d_name = "cxgb",
  189. };
  190. static devclass_t cxgb_port_devclass;
  191. DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
  192. MODULE_VERSION(cxgb, 1);
  193. NETDUMP_DEFINE(cxgb);
  194. static struct mtx t3_list_lock;
  195. static SLIST_HEAD(, adapter) t3_list;
  196. #ifdef TCP_OFFLOAD
  197. static struct mtx t3_uld_list_lock;
  198. static SLIST_HEAD(, uld_info) t3_uld_list;
  199. #endif
  200. /*
  201. * The driver uses the best interrupt scheme available on a platform in the
  202. * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
  203. * of these schemes the driver may consider as follows:
  204. *
  205. * msi = 2: choose from among all three options
  206. * msi = 1 : only consider MSI and pin interrupts
  207. * msi = 0: force pin interrupts
  208. */
  209. static int msi_allowed = 2;
  210. SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
  211. SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
  212. "MSI-X, MSI, INTx selector");
  213. /*
  214. * The driver uses an auto-queue algorithm by default.
  215. * To disable it and force a single queue-set per port, use multiq = 0
  216. */
  217. static int multiq = 1;
  218. SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
  219. "use min(ncpus/ports, 8) queue-sets per port");
  220. /*
  221. * By default the driver will not update the firmware unless
  222. * it was compiled against a newer version
  223. *
  224. */
  225. static int force_fw_update = 0;
  226. SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
  227. "update firmware even if up to date");
  228. int cxgb_use_16k_clusters = -1;
  229. SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
  230. &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
  231. static int nfilters = -1;
  232. SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
  233. &nfilters, 0, "max number of entries in the filter table");
  234. enum {
  235. MAX_TXQ_ENTRIES = 16384,
  236. MAX_CTRL_TXQ_ENTRIES = 1024,
  237. MAX_RSPQ_ENTRIES = 16384,
  238. MAX_RX_BUFFERS = 16384,
  239. MAX_RX_JUMBO_BUFFERS = 16384,
  240. MIN_TXQ_ENTRIES = 4,
  241. MIN_CTRL_TXQ_ENTRIES = 4,
  242. MIN_RSPQ_ENTRIES = 32,
  243. MIN_FL_ENTRIES = 32,
  244. MIN_FL_JUMBO_ENTRIES = 32
  245. };
  246. struct filter_info {
  247. u32 sip;
  248. u32 sip_mask;
  249. u32 dip;
  250. u16 sport;
  251. u16 dport;
  252. u32 vlan:12;
  253. u32 vlan_prio:3;
  254. u32 mac_hit:1;
  255. u32 mac_idx:4;
  256. u32 mac_vld:1;
  257. u32 pkt_type:2;
  258. u32 report_filter_id:1;
  259. u32 pass:1;
  260. u32 rss:1;
  261. u32 qset:3;
  262. u32 locked:1;
  263. u32 valid:1;
  264. };
  265. enum { FILTER_NO_VLAN_PRI = 7 };
  266. #define EEPROM_MAGIC 0x38E2F10C
  267. #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  268. static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
  269. static __inline char
  270. t3rev2char(struct adapter *adapter)
  271. {
  272. char rev = 'z';
  273. switch(adapter->params.rev) {
  274. case T3_REV_A:
  275. rev = 'a';
  276. break;
  277. case T3_REV_B:
  278. case T3_REV_B2:
  279. rev = 'b';
  280. break;
  281. case T3_REV_C:
  282. rev = 'c';
  283. break;
  284. }
  285. return rev;
  286. }
  287. static struct cxgb_ident *
  288. cxgb_get_ident(device_t dev)
  289. {
  290. struct cxgb_ident *id;
  291. for (id = cxgb_identifiers; id->desc != NULL; id++) {
  292. if ((id->vendor == pci_get_vendor(dev)) &&
  293. (id->device == pci_get_device(dev))) {
  294. return (id);
  295. }
  296. }
  297. return (NULL);
  298. }
  299. static const struct adapter_info *
  300. cxgb_get_adapter_info(device_t dev)
  301. {
  302. struct cxgb_ident *id;
  303. const struct adapter_info *ai;
  304. id = cxgb_get_ident(dev);
  305. if (id == NULL)
  306. return (NULL);
  307. ai = t3_get_adapter_info(id->index);
  308. return (ai);
  309. }
  310. static int
  311. cxgb_controller_probe(device_t dev)
  312. {
  313. const struct adapter_info *ai;
  314. char *ports, buf[80];
  315. int nports;
  316. ai = cxgb_get_adapter_info(dev);
  317. if (ai == NULL)
  318. return (ENXIO);
  319. nports = ai->nports0 + ai->nports1;
  320. if (nports == 1)
  321. ports = "port";
  322. else
  323. ports = "ports";
  324. snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
  325. device_set_desc_copy(dev, buf);
  326. return (BUS_PROBE_DEFAULT);
  327. }
  328. #define FW_FNAME "cxgb_t3fw"
  329. #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
  330. #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
  331. static int
  332. upgrade_fw(adapter_t *sc)
  333. {
  334. const struct firmware *fw;
  335. int status;
  336. u32 vers;
  337. if ((fw = firmware_get(FW_FNAME)) == NULL) {
  338. device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
  339. return (ENOENT);
  340. } else
  341. device_printf(sc->dev, "installing firmware on card\n");
  342. status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
  343. if (status != 0) {
  344. device_printf(sc->dev, "failed to install firmware: %d\n",
  345. status);
  346. } else {
  347. t3_get_fw_version(sc, &vers);
  348. snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
  349. G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
  350. G_FW_VERSION_MICRO(vers));
  351. }
  352. firmware_put(fw, FIRMWARE_UNLOAD);
  353. return (status);
  354. }
  355. /*
  356. * The cxgb_controller_attach function is responsible for the initial
  357. * bringup of the device. Its responsibilities include:
  358. *
  359. * 1. Determine if the device supports MSI or MSI-X.
  360. * 2. Allocate bus resources so that we can access the Base Address Register
  361. * 3. Create and initialize mutexes for the controller and its control
  362. * logic such as SGE and MDIO.
  363. * 4. Call hardware specific setup routine for the adapter as a whole.
  364. * 5. Allocate the BAR for doing MSI-X.
  365. * 6. Setup the line interrupt iff MSI-X is not supported.
  366. * 7. Create the driver's taskq.
  367. * 8. Start one task queue service thread.
  368. * 9. Check if the firmware and SRAM are up-to-date. They will be
  369. * auto-updated later (before FULL_INIT_DONE), if required.
  370. * 10. Create a child device for each MAC (port)
  371. * 11. Initialize T3 private state.
  372. * 12. Trigger the LED
  373. * 13. Setup offload iff supported.
  374. * 14. Reset/restart the tick callout.
  375. * 15. Attach sysctls
  376. *
  377. * NOTE: Any modification or deviation from this list MUST be reflected in
  378. * the above comment. Failure to do so will result in problems on various
  379. * error conditions including link flapping.
  380. */
  381. static int
  382. cxgb_controller_attach(device_t dev)
  383. {
  384. device_t child;
  385. const struct adapter_info *ai;
  386. struct adapter *sc;
  387. int i, error = 0;
  388. uint32_t vers;
  389. int port_qsets = 1;
  390. int msi_needed, reg;
  391. char buf[80];
  392. sc = device_get_softc(dev);
  393. sc->dev = dev;
  394. sc->msi_count = 0;
  395. ai = cxgb_get_adapter_info(dev);
  396. snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
  397. device_get_unit(dev));
  398. ADAPTER_LOCK_INIT(sc, sc->lockbuf);
  399. snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
  400. device_get_unit(dev));
  401. snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
  402. device_get_unit(dev));
  403. snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
  404. device_get_unit(dev));
  405. MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
  406. MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
  407. MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
  408. mtx_lock(&t3_list_lock);
  409. SLIST_INSERT_HEAD(&t3_list, sc, link);
  410. mtx_unlock(&t3_list_lock);
  411. /* find the PCIe link width and set max read request to 4KB*/
  412. if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
  413. uint16_t lnk;
  414. lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
  415. sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
  416. if (sc->link_width < 8 &&
  417. (ai->caps & SUPPORTED_10000baseT_Full)) {
  418. device_printf(sc->dev,
  419. "PCIe x%d Link, expect reduced performance\n",
  420. sc->link_width);
  421. }
  422. pci_set_max_read_req(dev, 4096);
  423. }
  424. touch_bars(dev);
  425. pci_enable_busmaster(dev);
  426. /*
  427. * Allocate the registers and make them available to the driver.
  428. * The registers that we care about for NIC mode are in BAR 0
  429. */
  430. sc->regs_rid = PCIR_BAR(0);
  431. if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  432. &sc->regs_rid, RF_ACTIVE)) == NULL) {
  433. device_printf(dev, "Cannot allocate BAR region 0\n");
  434. error = ENXIO;
  435. goto out;
  436. }
  437. sc->bt = rman_get_bustag(sc->regs_res);
  438. sc->bh = rman_get_bushandle(sc->regs_res);
  439. sc->mmio_len = rman_get_size(sc->regs_res);
  440. for (i = 0; i < MAX_NPORTS; i++)
  441. sc->port[i].adapter = sc;
  442. if (t3_prep_adapter(sc, ai, 1) < 0) {
  443. printf("prep adapter failed\n");
  444. error = ENODEV;
  445. goto out;
  446. }
  447. sc->udbs_rid = PCIR_BAR(2);
  448. sc->udbs_res = NULL;
  449. if (is_offload(sc) &&
  450. ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  451. &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
  452. device_printf(dev, "Cannot allocate BAR region 1\n");
  453. error = ENXIO;
  454. goto out;
  455. }
  456. /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
  457. * enough messages for the queue sets. If that fails, try falling
  458. * back to MSI. If that fails, then try falling back to the legacy
  459. * interrupt pin model.
  460. */
  461. sc->msix_regs_rid = 0x20;
  462. if ((msi_allowed >= 2) &&
  463. (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
  464. &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
  465. if (multiq)
  466. port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
  467. msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
  468. if (pci_msix_count(dev) == 0 ||
  469. (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
  470. sc->msi_count != msi_needed) {
  471. device_printf(dev, "alloc msix failed - "
  472. "msi_count=%d, msi_needed=%d, err=%d; "
  473. "will try MSI\n", sc->msi_count,
  474. msi_needed, error);
  475. sc->msi_count = 0;
  476. port_qsets = 1;
  477. pci_release_msi(dev);
  478. bus_release_resource(dev, SYS_RES_MEMORY,
  479. sc->msix_regs_rid, sc->msix_regs_res);
  480. sc->msix_regs_res = NULL;
  481. } else {
  482. sc->flags |= USING_MSIX;
  483. sc->cxgb_intr = cxgb_async_intr;
  484. device_printf(dev,
  485. "using MSI-X interrupts (%u vectors)\n",
  486. sc->msi_count);
  487. }
  488. }
  489. if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
  490. sc->msi_count = 1;
  491. if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
  492. device_printf(dev, "alloc msi failed - "
  493. "err=%d; will try INTx\n", error);
  494. sc->msi_count = 0;
  495. port_qsets = 1;
  496. pci_release_msi(dev);
  497. } else {
  498. sc->flags |= USING_MSI;
  499. sc->cxgb_intr = t3_intr_msi;
  500. device_printf(dev, "using MSI interrupts\n");
  501. }
  502. }
  503. if (sc->msi_count == 0) {
  504. device_printf(dev, "using line interrupts\n");
  505. sc->cxgb_intr = t3b_intr;
  506. }
  507. /* Create a private taskqueue thread for handling driver events */
  508. sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
  509. taskqueue_thread_enqueue, &sc->tq);
  510. if (sc->tq == NULL) {
  511. device_printf(dev, "failed to allocate controller task queue\n");
  512. goto out;
  513. }
  514. taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
  515. device_get_nameunit(dev));
  516. TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
  517. /* Create a periodic callout for checking adapter status */
  518. callout_init(&sc->cxgb_tick_ch, 1);
  519. if (t3_check_fw_version(sc) < 0 || force_fw_update) {
  520. /*
  521. * Warn user that a firmware update will be attempted in init.
  522. */
  523. device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
  524. FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
  525. sc->flags &= ~FW_UPTODATE;
  526. } else {
  527. sc->flags |= FW_UPTODATE;
  528. }
  529. if (t3_check_tpsram_version(sc) < 0) {
  530. /*
  531. * Warn user that a firmware update will be attempted in init.
  532. */
  533. device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
  534. t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
  535. sc->flags &= ~TPS_UPTODATE;
  536. } else {
  537. sc->flags |= TPS_UPTODATE;
  538. }
  539. /*
  540. * Create a child device for each MAC. The ethernet attachment
  541. * will be done in these children.
  542. */
  543. for (i = 0; i < (sc)->params.nports; i++) {
  544. struct port_info *pi;
  545. if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
  546. device_printf(dev, "failed to add child port\n");
  547. error = EINVAL;
  548. goto out;
  549. }
  550. pi = &sc->port[i];
  551. pi->adapter = sc;
  552. pi->nqsets = port_qsets;
  553. pi->first_qset = i*port_qsets;
  554. pi->port_id = i;
  555. pi->tx_chan = i >= ai->nports0;
  556. pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
  557. sc->rxpkt_map[pi->txpkt_intf] = i;
  558. sc->port[i].tx_chan = i >= ai->nports0;
  559. sc->portdev[i] = child;
  560. device_set_softc(child, pi);
  561. }
  562. if ((error = bus_generic_attach(dev)) != 0)
  563. goto out;
  564. /* initialize sge private state */
  565. t3_sge_init_adapter(sc);
  566. t3_led_ready(sc);
  567. error = t3_get_fw_version(sc, &vers);
  568. if (error)
  569. goto out;
  570. snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
  571. G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
  572. G_FW_VERSION_MICRO(vers));
  573. snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
  574. ai->desc, is_offload(sc) ? "R" : "",
  575. sc->params.vpd.ec, sc->params.vpd.sn);
  576. device_set_desc_copy(dev, buf);
  577. snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
  578. sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
  579. sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
  580. device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
  581. callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
  582. t3_add_attach_sysctls(sc);
  583. #ifdef TCP_OFFLOAD
  584. for (i = 0; i < NUM_CPL_HANDLERS; i++)
  585. sc->cpl_handler[i] = cpl_not_handled;
  586. #endif
  587. t3_intr_clear(sc);
  588. error = cxgb_setup_interrupts(sc);
  589. out:
  590. if (error)
  591. cxgb_free(sc);
  592. return (error);
  593. }
  594. /*
  595. * The cxgb_controller_detach routine is called with the device is
  596. * unloaded from the system.
  597. */
  598. static int
  599. cxgb_controller_detach(device_t dev)
  600. {
  601. struct adapter *sc;
  602. sc = device_get_softc(dev);
  603. cxgb_free(sc);
  604. return (0);
  605. }
  606. /*
  607. * The cxgb_free() is called by the cxgb_controller_detach() routine
  608. * to tear down the structures that were built up in
  609. * cxgb_controller_attach(), and should be the final piece of work
  610. * done when fully unloading the driver.
  611. *
  612. *
  613. * 1. Shutting down the threads started by the cxgb_controller_attach()
  614. * routine.
  615. * 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
  616. * 3. Detaching all of the port devices created during the
  617. * cxgb_controller_attach() routine.
  618. * 4. Removing the device children created via cxgb_controller_attach().
  619. * 5. Releasing PCI resources associated with the device.
  620. * 6. Turning off the offload support, iff it was turned on.
  621. * 7. Destroying the mutexes created in cxgb_controller_attach().
  622. *
  623. */
  624. static void
  625. cxgb_free(struct adapter *sc)
  626. {
  627. int i, nqsets = 0;
  628. ADAPTER_LOCK(sc);
  629. sc->flags |= CXGB_SHUTDOWN;
  630. ADAPTER_UNLOCK(sc);
  631. /*
  632. * Make sure all child devices are gone.
  633. */
  634. bus_generic_detach(sc->dev);
  635. for (i = 0; i < (sc)->params.nports; i++) {
  636. if (sc->portdev[i] &&
  637. device_delete_child(sc->dev, sc->portdev[i]) != 0)
  638. device_printf(sc->dev, "failed to delete child port\n");
  639. nqsets += sc->port[i].nqsets;
  640. }
  641. /*
  642. * At this point, it is as if cxgb_port_detach has run on all ports, and
  643. * cxgb_down has run on the adapter. All interrupts have been silenced,
  644. * all open devices have been closed.
  645. */
  646. KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
  647. __func__, sc->open_device_map));
  648. for (i = 0; i < sc->params.nports; i++) {
  649. KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
  650. __func__, i));
  651. }
  652. /*
  653. * Finish off the adapter's callouts.
  654. */
  655. callout_drain(&sc->cxgb_tick_ch);
  656. callout_drain(&sc->sge_timer_ch);
  657. /*
  658. * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
  659. * sysctls are cleaned up by the kernel linker.
  660. */
  661. if (sc->flags & FULL_INIT_DONE) {
  662. t3_free_sge_resources(sc, nqsets);
  663. sc->flags &= ~FULL_INIT_DONE;
  664. }
  665. /*
  666. * Release all interrupt resources.
  667. */
  668. cxgb_teardown_interrupts(sc);
  669. if (sc->flags & (USING_MSI | USING_MSIX)) {
  670. device_printf(sc->dev, "releasing msi message(s)\n");
  671. pci_release_msi(sc->dev);
  672. } else {
  673. device_printf(sc->dev, "no msi message to release\n");
  674. }
  675. if (sc->msix_regs_res != NULL) {
  676. bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
  677. sc->msix_regs_res);
  678. }
  679. /*
  680. * Free the adapter's taskqueue.
  681. */
  682. if (sc->tq != NULL) {
  683. taskqueue_free(sc->tq);
  684. sc->tq = NULL;
  685. }
  686. free(sc->filters, M_DEVBUF);
  687. t3_sge_free(sc);
  688. if (sc->udbs_res != NULL)
  689. bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
  690. sc->udbs_res);
  691. if (sc->regs_res != NULL)
  692. bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
  693. sc->regs_res);
  694. MTX_DESTROY(&sc->mdio_lock);
  695. MTX_DESTROY(&sc->sge.reg_lock);
  696. MTX_DESTROY(&sc->elmer_lock);
  697. mtx_lock(&t3_list_lock);
  698. SLIST_REMOVE(&t3_list, sc, adapter, link);
  699. mtx_unlock(&t3_list_lock);
  700. ADAPTER_LOCK_DEINIT(sc);
  701. }
  702. /**
  703. * setup_sge_qsets - configure SGE Tx/Rx/response queues
  704. * @sc: the controller softc
  705. *
  706. * Determines how many sets of SGE queues to use and initializes them.
  707. * We support multiple queue sets per port if we have MSI-X, otherwise
  708. * just one queue set per port.
  709. */
  710. static int
  711. setup_sge_qsets(adapter_t *sc)
  712. {
  713. int i, j, err, irq_idx = 0, qset_idx = 0;
  714. u_int ntxq = SGE_TXQ_PER_SET;
  715. if ((err = t3_sge_alloc(sc)) != 0) {
  716. device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
  717. return (err);
  718. }
  719. if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
  720. irq_idx = -1;
  721. for (i = 0; i < (sc)->params.nports; i++) {
  722. struct port_info *pi = &sc->port[i];
  723. for (j = 0; j < pi->nqsets; j++, qset_idx++) {
  724. err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
  725. (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
  726. &sc->params.sge.qset[qset_idx], ntxq, pi);
  727. if (err) {
  728. t3_free_sge_resources(sc, qset_idx);
  729. device_printf(sc->dev,
  730. "t3_sge_alloc_qset failed with %d\n", err);
  731. return (err);
  732. }
  733. }
  734. }
  735. return (0);
  736. }
  737. static void
  738. cxgb_teardown_interrupts(adapter_t *sc)
  739. {
  740. int i;
  741. for (i = 0; i < SGE_QSETS; i++) {
  742. if (sc->msix_intr_tag[i] == NULL) {
  743. /* Should have been setup fully or not at all */
  744. KASSERT(sc->msix_irq_res[i] == NULL &&
  745. sc->msix_irq_rid[i] == 0,
  746. ("%s: half-done interrupt (%d).", __func__, i));
  747. continue;
  748. }
  749. bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
  750. sc->msix_intr_tag[i]);
  751. bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
  752. sc->msix_irq_res[i]);
  753. sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
  754. sc->msix_irq_rid[i] = 0;
  755. }
  756. if (sc->intr_tag) {
  757. KASSERT(sc->irq_res != NULL,
  758. ("%s: half-done interrupt.", __func__));
  759. bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
  760. bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
  761. sc->irq_res);
  762. sc->irq_res = sc->intr_tag = NULL;
  763. sc->irq_rid = 0;
  764. }
  765. }
  766. static int
  767. cxgb_setup_interrupts(adapter_t *sc)
  768. {
  769. struct resource *res;
  770. void *tag;
  771. int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
  772. sc->irq_rid = intr_flag ? 1 : 0;
  773. sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
  774. RF_SHAREABLE | RF_ACTIVE);
  775. if (sc->irq_res == NULL) {
  776. device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
  777. intr_flag, sc->irq_rid);
  778. err = EINVAL;
  779. sc->irq_rid = 0;
  780. } else {
  781. err = bus_setup_intr(sc->dev, sc->irq_res,
  782. INTR_MPSAFE | INTR_TYPE_NET, NULL,
  783. sc->cxgb_intr, sc, &sc->intr_tag);
  784. if (err) {
  785. device_printf(sc->dev,
  786. "Cannot set up interrupt (%x, %u, %d)\n",
  787. intr_flag, sc->irq_rid, err);
  788. bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
  789. sc->irq_res);
  790. sc->irq_res = sc->intr_tag = NULL;
  791. sc->irq_rid = 0;
  792. }
  793. }
  794. /* That's all for INTx or MSI */
  795. if (!(intr_flag & USING_MSIX) || err)
  796. return (err);
  797. bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
  798. for (i = 0; i < sc->msi_count - 1; i++) {
  799. rid = i + 2;
  800. res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
  801. RF_SHAREABLE | RF_ACTIVE);
  802. if (res == NULL) {
  803. device_printf(sc->dev, "Cannot allocate interrupt "
  804. "for message %d\n", rid);
  805. err = EINVAL;
  806. break;
  807. }
  808. err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
  809. NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
  810. if (err) {
  811. device_printf(sc->dev, "Cannot set up interrupt "
  812. "for message %d (%d)\n", rid, err);
  813. bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
  814. break;
  815. }
  816. sc->msix_irq_rid[i] = rid;
  817. sc->msix_irq_res[i] = res;
  818. sc->msix_intr_tag[i] = tag;
  819. bus_describe_intr(sc->dev, res, tag, "qs%d", i);
  820. }
  821. if (err)
  822. cxgb_teardown_interrupts(sc);
  823. return (err);
  824. }
  825. static int
  826. cxgb_port_probe(device_t dev)
  827. {
  828. struct port_info *p;
  829. char buf[80];
  830. const char *desc;
  831. p = device_get_softc(dev);
  832. desc = p->phy.desc;
  833. snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
  834. device_set_desc_copy(dev, buf);
  835. return (0);
  836. }
  837. static int
  838. cxgb_makedev(struct port_info *pi)
  839. {
  840. pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
  841. UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
  842. if (pi->port_cdev == NULL)
  843. return (ENOMEM);
  844. pi->port_cdev->si_drv1 = (void *)pi;
  845. return (0);
  846. }
  847. #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
  848. IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
  849. IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
  850. #define CXGB_CAP_ENABLE CXGB_CAP
  851. static int
  852. cxgb_port_attach(device_t dev)
  853. {
  854. struct port_info *p;
  855. struct ifnet *ifp;
  856. int err;
  857. struct adapter *sc;
  858. p = device_get_softc(dev);
  859. sc = p->adapter;
  860. snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
  861. device_get_unit(device_get_parent(dev)), p->port_id);
  862. PORT_LOCK_INIT(p, p->lockbuf);
  863. callout_init(&p->link_check_ch, 1);
  864. TASK_INIT(&p->link_check_task, 0, check_link_status, p);
  865. /* Allocate an ifnet object and set it up */
  866. ifp = p->ifp = if_alloc(IFT_ETHER);
  867. if (ifp == NULL) {
  868. device_printf(dev, "Cannot allocate ifnet\n");
  869. return (ENOMEM);
  870. }
  871. if_initname(ifp, device_get_name(dev), device_get_unit(dev));
  872. ifp->if_init = cxgb_init;
  873. ifp->if_softc = p;
  874. ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
  875. ifp->if_ioctl = cxgb_ioctl;
  876. ifp->if_transmit = cxgb_transmit;
  877. ifp->if_qflush = cxgb_qflush;
  878. ifp->if_get_counter = cxgb_get_counter;
  879. ifp->if_capabilities = CXGB_CAP;
  880. #ifdef TCP_OFFLOAD
  881. if (is_offload(sc))
  882. ifp->if_capabilities |= IFCAP_TOE4;
  883. #endif
  884. ifp->if_capenable = CXGB_CAP_ENABLE;
  885. ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
  886. CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
  887. /*
  888. * Disable TSO on 4-port - it isn't supported by the firmware.
  889. */
  890. if (sc->params.nports > 2) {
  891. ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
  892. ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
  893. ifp->if_hwassist &= ~CSUM_TSO;
  894. }
  895. ether_ifattach(ifp, p->hw_addr);
  896. /* Attach driver netdump methods. */
  897. NETDUMP_SET(ifp, cxgb);
  898. #ifdef DEFAULT_JUMBO
  899. if (sc->params.nports <= 2)
  900. ifp->if_mtu = ETHERMTU_JUMBO;
  901. #endif
  902. if ((err = cxgb_makedev(p)) != 0) {
  903. printf("makedev failed %d\n", err);
  904. return (err);
  905. }
  906. /* Create a list of media supported by this port */
  907. ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
  908. cxgb_media_status);
  909. cxgb_build_medialist(p);
  910. t3_sge_init_port(p);
  911. return (err);
  912. }
  913. /*
  914. * cxgb_port_detach() is called via the device_detach methods when
  915. * cxgb_free() calls the bus_generic_detach. It is responsible for
  916. * removing the device from the view of the kernel, i.e. from all
  917. * interfaces lists etc. This routine is only called when the driver is
  918. * being unloaded, not when the link goes down.
  919. */
  920. static int
  921. cxgb_port_detach(device_t dev)
  922. {
  923. struct port_info *p;
  924. struct adapter *sc;
  925. int i;
  926. p = device_get_softc(dev);
  927. sc = p->adapter;
  928. /* Tell cxgb_ioctl and if_init that the port is going away */
  929. ADAPTER_LOCK(sc);
  930. SET_DOOMED(p);
  931. wakeup(&sc->flags);
  932. while (IS_BUSY(sc))
  933. mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
  934. SET_BUSY(sc);
  935. ADAPTER_UNLOCK(sc);
  936. if (p->port_cdev != NULL)
  937. destroy_dev(p->port_cdev);
  938. cxgb_uninit_synchronized(p);
  939. ether_ifdetach(p->ifp);
  940. for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
  941. struct sge_qset *qs = &sc->sge.qs[i];
  942. struct sge_txq *txq = &qs->txq[TXQ_ETH];
  943. callout_drain(&txq->txq_watchdog);
  944. callout_drain(&txq->txq_timer);
  945. }
  946. PORT_LOCK_DEINIT(p);
  947. if_free(p->ifp);
  948. p->ifp = NULL;
  949. ADAPTER_LOCK(sc);
  950. CLR_BUSY(sc);
  951. wakeup_one(&sc->flags);
  952. ADAPTER_UNLOCK(sc);
  953. return (0);
  954. }
  955. void
  956. t3_fatal_err(struct adapter *sc)
  957. {
  958. u_int fw_status[4];
  959. if (sc->flags & FULL_INIT_DONE) {
  960. t3_sge_stop(sc);
  961. t3_write_reg(sc, A_XGM_TX_CTRL, 0);
  962. t3_write_reg(sc, A_XGM_RX_CTRL, 0);
  963. t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
  964. t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
  965. t3_intr_disable(sc);
  966. }
  967. device_printf(sc->dev,"encountered fatal error, operation suspended\n");
  968. if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
  969. device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
  970. fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
  971. }
  972. int
  973. t3_os_find_pci_capability(adapter_t *sc, int cap)
  974. {
  975. device_t dev;
  976. struct pci_devinfo *dinfo;
  977. pcicfgregs *cfg;
  978. uint32_t status;
  979. uint8_t ptr;
  980. dev = sc->dev;
  981. dinfo = device_get_ivars(dev);
  982. cfg = &dinfo->cfg;
  983. status = pci_read_config(dev, PCIR_STATUS, 2);
  984. if (!(status & PCIM_STATUS_CAPPRESENT))
  985. return (0);
  986. switch (cfg->hdrtype & PCIM_HDRTYPE) {
  987. case 0:
  988. case 1:
  989. ptr = PCIR_CAP_PTR;
  990. break;
  991. case 2:
  992. ptr = PCIR_CAP_PTR_2;
  993. break;
  994. default:
  995. return (0);
  996. break;
  997. }
  998. ptr = pci_read_config(dev, ptr, 1);
  999. while (ptr != 0) {
  1000. if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
  1001. return (ptr);
  1002. ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
  1003. }
  1004. return (0);
  1005. }
  1006. int
  1007. t3_os_pci_save_state(struct adapter *sc)
  1008. {
  1009. device_t dev;
  1010. struct pci_devinfo *dinfo;
  1011. dev = sc->dev;
  1012. dinfo = device_get_ivars(dev);
  1013. pci_cfg_save(dev, dinfo, 0);
  1014. return (0);
  1015. }
  1016. int
  1017. t3_os_pci_restore_state(struct adapter *sc)
  1018. {
  1019. device_t dev;
  1020. struct pci_devinfo *dinfo;
  1021. dev = sc->dev;
  1022. dinfo = device_get_ivars(dev);
  1023. pci_cfg_restore(dev, dinfo);
  1024. return (0);
  1025. }
  1026. /**
  1027. * t3_os_link_changed - handle link status changes
  1028. * @sc: the adapter associated with the link change
  1029. * @port_id: the port index whose link status has changed
  1030. * @link_status: the new status of the link
  1031. * @speed: the new speed setting
  1032. * @duplex: the new duplex setting
  1033. * @fc: the new flow-control setting
  1034. *
  1035. * This is the OS-dependent handler for link status changes. The OS
  1036. * neutral handler takes care of most of the processing for these events,
  1037. * then calls this handler for any OS-specific processing.
  1038. */
  1039. void
  1040. t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
  1041. int duplex, int fc, int mac_was_reset)
  1042. {
  1043. struct port_info *pi = &adapter->port[port_id];
  1044. struct ifnet *ifp = pi->ifp;
  1045. /* no race with detach, so ifp should always be good */
  1046. KASSERT(ifp, ("%s: if detached.", __func__));
  1047. /* Reapply mac settings if they were lost due to a reset */
  1048. if (mac_was_reset) {
  1049. PORT_LOCK(pi);
  1050. cxgb_update_mac_settings(pi);
  1051. PORT_UNLOCK(pi);
  1052. }
  1053. if (link_status) {
  1054. ifp->if_baudrate = IF_Mbps(speed);
  1055. if_link_state_change(ifp, LINK_STATE_UP);
  1056. } else
  1057. if_link_state_change(ifp, LINK_STATE_DOWN);
  1058. }
  1059. /**
  1060. * t3_os_phymod_changed - handle PHY module changes
  1061. * @phy: the PHY reporting the module change
  1062. * @mod_type: new module type
  1063. *
  1064. * This is the OS-dependent handler for PHY module changes. It is
  1065. * invoked when a PHY module is removed or inserted for any OS-specific
  1066. * processing.
  1067. */
  1068. void t3_os_phymod_changed(struct adapter *adap, int port_id)
  1069. {
  1070. static const char *mod_str[] = {
  1071. NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
  1072. };
  1073. struct port_info *pi = &adap->port[port_id];
  1074. int mod = pi->phy.modtype;
  1075. if (mod != pi->media.ifm_cur->ifm_data)
  1076. cxgb_build_medialist(pi);
  1077. if (mod == phy_modtype_none)
  1078. if_printf(pi->ifp, "PHY module unplugged\n");
  1079. else {
  1080. KASSERT(mod < ARRAY_SIZE(mod_str),
  1081. ("invalid PHY module type %d", mod));
  1082. if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
  1083. }
  1084. }
  1085. void
  1086. t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
  1087. {
  1088. /*
  1089. * The ifnet might not be allocated before this gets called,
  1090. * as this is called early on in attach by t3_prep_adapter
  1091. * save the address off in the port structure
  1092. */
  1093. if (cxgb_debug)
  1094. printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
  1095. bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
  1096. }
  1097. /*
  1098. * Programs the XGMAC based on the settings in the ifnet. These settings
  1099. * include MTU, MAC address, mcast addresses, etc.
  1100. */
  1101. static void
  1102. cxgb_update_mac_settings(struct port_info *p)
  1103. {
  1104. struct ifnet *ifp = p->ifp;
  1105. struct t3_rx_mode rm;
  1106. struct cmac *mac = &p->mac;
  1107. int mtu, hwtagging;
  1108. PORT_LOCK_ASSERT_OWNED(p);
  1109. bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
  1110. mtu = ifp->if_mtu;
  1111. if (ifp->if_capenable & IFCAP_VLAN_MTU)
  1112. mtu += ETHER_VLAN_ENCAP_LEN;
  1113. hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
  1114. t3_mac_set_mtu(mac, mtu);
  1115. t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
  1116. t3_mac_set_address(mac, 0, p->hw_addr);
  1117. t3_init_rx_mode(&rm, p);
  1118. t3_mac_set_rx_mode(mac, &rm);
  1119. }
  1120. static int
  1121. await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
  1122. unsigned long n)
  1123. {
  1124. int attempts = 5;
  1125. while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
  1126. if (!--attempts)
  1127. return (ETIMEDOUT);
  1128. t3_os_sleep(10);
  1129. }
  1130. return 0;
  1131. }
  1132. static int
  1133. init_tp_parity(struct adapter *adap)
  1134. {
  1135. int i;
  1136. struct mbuf *m;
  1137. struct cpl_set_tcb_field *greq;
  1138. unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
  1139. t3_tp_set_offload_mode(adap, 1);
  1140. for (i = 0; i < 16; i++) {
  1141. struct cpl_smt_write_req *req;
  1142. m = m_gethdr(M_WAITOK, MT_DATA);
  1143. req = mtod(m, struct cpl_smt_write_req *);
  1144. m->m_len = m->m_pkthdr.len = sizeof(*req);
  1145. memset(req, 0, sizeof(*req));
  1146. req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
  1147. OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
  1148. req->iff = i;
  1149. t3_mgmt_tx(adap, m);
  1150. }
  1151. for (i = 0; i < 2048; i++) {
  1152. struct cpl_l2t_write_req *req;
  1153. m = m_gethdr(M_WAITOK, MT_DATA);
  1154. req = mtod(m, struct cpl_l2t_write_req *);
  1155. m->m_len = m->m_pkthdr.len = sizeof(*req);
  1156. memset(req, 0, sizeof(*req));
  1157. req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
  1158. OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
  1159. req->params = htonl(V_L2T_W_IDX(i));
  1160. t3_mgmt_tx(adap, m);
  1161. }
  1162. for (i = 0; i < 2048; i++) {
  1163. struct cpl_rte_write_req *req;
  1164. m = m_gethdr(M_WAITOK, MT_DATA);
  1165. req = mtod(m, struct cpl_rte_write_req *);
  1166. m->m_len = m->m_pkthdr.len = sizeof(*req);
  1167. memset(req, 0, sizeof(*req));
  1168. req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
  1169. OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
  1170. req->l2t_idx = htonl(V_L2T_W_IDX(i));
  1171. t3_mgmt_tx(adap, m);
  1172. }
  1173. m = m_gethdr(M_WAITOK, MT_DATA);
  1174. greq = mtod(m, struct cpl_set_tcb_field *);
  1175. m->m_len = m->m_pkthdr.len = sizeof(*greq);
  1176. memset(greq, 0, sizeof(*greq));
  1177. greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
  1178. OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
  1179. greq->mask = htobe64(1);
  1180. t3_mgmt_tx(adap, m);
  1181. i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
  1182. t3_tp_set_offload_mode(adap, 0);
  1183. return (i);
  1184. }
  1185. /**
  1186. * setup_rss - configure Receive Side Steering (per-queue connection demux)
  1187. * @adap: the adapter
  1188. *
  1189. * Sets up RSS to distribute packets to multiple receive queues. We
  1190. * configure the RSS CPU lookup table to distribute to the number of HW
  1191. * receive queues, and the response queue lookup table to narrow that
  1192. * down to the response queues actually configured for each port.
  1193. * We always configure the RSS mapping for two ports since the mapping
  1194. * table has plenty of entries.
  1195. */
  1196. static void
  1197. setup_rss(adapter_t *adap)
  1198. {
  1199. int i;
  1200. u_int nq[2];
  1201. uint8_t cpus[SGE_QSETS + 1];
  1202. uint16_t rspq_map[RSS_TABLE_SIZE];
  1203. for (i = 0; i < SGE_QSETS; ++i)
  1204. cpus[i] = i;
  1205. cpus[SGE_QSETS] = 0xff;
  1206. nq[0] = nq[1] = 0;
  1207. for_each_port(adap, i) {
  1208. const struct port_info *pi = adap2pinfo(adap, i);
  1209. nq[pi->tx_chan] += pi->nqsets;
  1210. }
  1211. for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
  1212. rspq_map[i] = nq[0] ? i % nq[0] : 0;
  1213. rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
  1214. }
  1215. /* Calculate the reverse RSS map table */
  1216. for (i = 0; i < SGE_QSETS; ++i)
  1217. adap->rrss_map[i] = 0xff;
  1218. for (i = 0; i < RSS_TABLE_SIZE; ++i)
  1219. if (adap->rrss_map[rspq_map[i]] == 0xff)
  1220. adap->rrss_map[rspq_map[i]] = i;
  1221. t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
  1222. F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
  1223. F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
  1224. cpus, rspq_map);
  1225. }
  1226. static void
  1227. send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
  1228. int hi, int port)
  1229. {
  1230. struct mbuf *m;
  1231. struct mngt_pktsched_wr *req;
  1232. m = m_gethdr(M_NOWAIT, MT_DATA);
  1233. if (m) {
  1234. req = mtod(m, struct mngt_pktsched_wr *);
  1235. req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
  1236. req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
  1237. req->sched = sched;
  1238. req->idx = qidx;
  1239. req->min = lo;
  1240. req->max = hi;
  1241. req->binding = port;
  1242. m->m_len = m->m_pkthdr.len = sizeof(*req);
  1243. t3_mgmt_tx(adap, m);
  1244. }
  1245. }
  1246. static void
  1247. bind_qsets(adapter_t *sc)
  1248. {
  1249. int i, j;
  1250. for (i = 0; i < (sc)->params.nports; ++i) {
  1251. const struct port_info *pi = adap2pinfo(sc, i);
  1252. for (j = 0; j < pi->nqsets; ++j) {
  1253. send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
  1254. -1, pi->tx_chan);
  1255. }
  1256. }
  1257. }
  1258. static void
  1259. update_tpeeprom(struct adapter *adap)
  1260. {
  1261. const struct firmware *tpeeprom;
  1262. uint32_t version;
  1263. unsigned int major, minor;
  1264. int ret, len;
  1265. char rev, name[32];
  1266. t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
  1267. major = G_TP_VERSION_MAJOR(version);
  1268. minor = G_TP_VERSION_MINOR(version);
  1269. if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
  1270. return;
  1271. rev = t3rev2char(adap);
  1272. snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
  1273. tpeeprom = firmware_get(name);
  1274. if (tpeeprom == NULL) {
  1275. device_printf(adap->dev,
  1276. "could not load TP EEPROM: unable to load %s\n",
  1277. name);
  1278. return;
  1279. }
  1280. len = tpeeprom->datasize - 4;
  1281. ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
  1282. if (ret)
  1283. goto release_tpeeprom;
  1284. if (len != TP_SRAM_LEN) {
  1285. device_printf(adap->dev,
  1286. "%s length is wrong len=%d expected=%d\n", name,
  1287. len, TP_SRAM_LEN);
  1288. return;
  1289. }
  1290. ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
  1291. TP_SRAM_OFFSET);
  1292. if (!ret) {
  1293. device_printf(adap->dev,
  1294. "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
  1295. TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
  1296. } else
  1297. device_printf(adap->dev,
  1298. "Protocol SRAM image update in EEPROM failed\n");
  1299. release_tpeeprom:
  1300. firmware_put(tpeeprom, FIRMWARE_UNLOAD);
  1301. return;
  1302. }
  1303. static int
  1304. update_tpsram(struct adapter *adap)
  1305. {
  1306. const struct firmware *tpsram;
  1307. int ret;
  1308. char rev, name[32];
  1309. rev = t3rev2char(adap);
  1310. snprintf(name, sizeof(name), TPSRAM_NAME, rev);
  1311. update_tpeeprom(adap);
  1312. tpsram = firmware_get(name);
  1313. if (tpsram == NULL){
  1314. device_printf(adap->dev, "could not load TP SRAM\n");
  1315. return (EINVAL);
  1316. } else
  1317. device_printf(adap->dev, "updating TP SRAM\n");
  1318. ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
  1319. if (ret)
  1320. goto release_tpsram;
  1321. ret = t3_set_proto_sram(adap, tpsram->data);
  1322. if (ret)
  1323. device_printf(adap->dev, "loading protocol SRAM failed\n");
  1324. release_tpsram:
  1325. firmware_put(tpsram, FIRMWARE_UNLOAD);
  1326. return ret;
  1327. }
  1328. /**
  1329. * cxgb_up - enable the adapter
  1330. * @adap: adapter being enabled
  1331. *
  1332. * Called when the first port is enabled, this function performs the
  1333. * actions necessary to make an adapter operational, such as completing
  1334. * the initialization of HW modules, and enabling interrupts.
  1335. */
  1336. static int
  1337. cxgb_up(struct adapter *sc)
  1338. {
  1339. int err = 0;
  1340. unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
  1341. KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
  1342. __func__, sc->open_device_map));
  1343. if ((sc->flags & FULL_INIT_DONE) == 0) {
  1344. ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
  1345. if ((sc->flags & FW_UPTODATE) == 0)
  1346. if ((err = upgrade_fw(sc)))
  1347. goto out;
  1348. if ((sc->flags & TPS_UPTODATE) == 0)
  1349. if ((err = update_tpsram(sc)))
  1350. goto out;
  1351. if (is_offload(sc) && nfilters != 0) {
  1352. sc->params.mc5.nservers = 0;
  1353. if (nfilters < 0)
  1354. sc->params.mc5.nfilters = mxf;
  1355. else
  1356. sc->params.mc5.nfilters = min(nfilters, mxf);
  1357. }
  1358. err = t3_init_hw(sc, 0);
  1359. if (err)
  1360. goto out;
  1361. t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
  1362. t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
  1363. err = setup_sge_qsets(sc);
  1364. if (err)
  1365. goto out;
  1366. alloc_filters(sc);
  1367. setup_rss(sc);
  1368. t3_add_configured_sysctls(sc);
  1369. sc->flags |= FULL_INIT_DONE;
  1370. }
  1371. t3_intr_clear(sc);
  1372. t3_sge_start(sc);
  1373. t3_intr_enable(sc);
  1374. if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
  1375. is_offload(sc) && init_tp_parity(sc) == 0)
  1376. sc->flags |= TP_PARITY_INIT;
  1377. if (sc->flags & TP_PARITY_INIT) {
  1378. t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
  1379. t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
  1380. }
  1381. if (!(sc->flags & QUEUES_BOUND)) {
  1382. bind_qsets(sc);
  1383. setup_hw_filters(sc);
  1384. sc->flags |= QUEUES_BOUND;
  1385. }
  1386. t3_sge_reset_adapter(sc);
  1387. out:
  1388. return (err);
  1389. }
  1390. /*
  1391. * Called when the last open device is closed. Does NOT undo all of cxgb_up's
  1392. * work. Specifically, the resources grabbed under FULL_INIT_DONE are released
  1393. * during controller_detach, not here.
  1394. */
  1395. static void
  1396. cxgb_down(struct adapter *sc)
  1397. {
  1398. t3_sge_stop(sc);
  1399. t3_intr_disable(sc);
  1400. }
  1401. /*
  1402. * if_init for cxgb ports.
  1403. */
  1404. static void
  1405. cxgb_init(void *arg)
  1406. {
  1407. struct port_info *p = arg;
  1408. struct adapter *sc = p->adapter;
  1409. ADAPTER_LOCK(sc);
  1410. cxgb_init_locked(p); /* releases adapter lock */
  1411. ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
  1412. }
  1413. static int
  1414. cxgb_init_locked(struct port_info *p)
  1415. {
  1416. struct adapter *sc = p->adapter;
  1417. struct ifnet *ifp = p->ifp;
  1418. struct cmac *mac = &p->mac;
  1419. int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
  1420. ADAPTER_LOCK_ASSERT_OWNED(sc);
  1421. while (!IS_DOOMED(p) && IS_BUSY(sc)) {
  1422. gave_up_lock = 1;
  1423. if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
  1424. rc = EINTR;
  1425. goto done;
  1426. }
  1427. }
  1428. if (IS_DOOMED(p)) {
  1429. rc = ENXIO;
  1430. goto done;
  1431. }
  1432. KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
  1433. /*
  1434. * The code that runs during one-time adapter initialization can sleep
  1435. * so it's important not to hold any locks across it.
  1436. */
  1437. may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
  1438. if (may_sleep) {
  1439. SET_BUSY(sc);
  1440. gave_up_lock = 1;
  1441. ADAPTER_UNLOCK(sc);
  1442. }
  1443. if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
  1444. goto done;
  1445. PORT_LOCK(p);
  1446. if (isset(&sc->open_device_map, p->port_id) &&
  1447. (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
  1448. PORT_UNLOCK(p);
  1449. goto done;
  1450. }
  1451. t3_port_intr_enable(sc, p->port_id);
  1452. if (!mac->multiport)
  1453. t3_mac_init(mac);
  1454. cxgb_update_mac_settings(p);
  1455. t3_link_start(&p->phy, mac, &p->link_config);
  1456. t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
  1457. ifp->if_drv_flags |= IFF_DRV_RUNNING;
  1458. ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
  1459. PORT_UNLOCK(p);
  1460. for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
  1461. struct sge_qset *qs = &sc->sge.qs[i];
  1462. struct sge_txq *txq = &qs->txq[TXQ_ETH];
  1463. callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
  1464. txq->txq_watchdog.c_cpu);
  1465. }
  1466. /* all ok */
  1467. setbit(&sc->open_device_map, p->port_id);
  1468. callout_reset(&p->link_check_ch,
  1469. p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4,
  1470. link_check_callout, p);
  1471. done:
  1472. if (may_sleep) {
  1473. ADAPTER_LOCK(sc);
  1474. KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
  1475. CLR_BUSY(sc);
  1476. }
  1477. if (gave_up_lock)
  1478. wakeup_one(&sc->flags);
  1479. ADAPTER_UNLOCK(sc);
  1480. return (rc);
  1481. }
  1482. static int
  1483. cxgb_uninit_locked(struct port_info *p)
  1484. {
  1485. struct adapter *sc = p->adapter;
  1486. int rc;
  1487. ADAPTER_LOCK_ASSERT_OWNED(sc);
  1488. while (!IS_DOOMED(p) && IS_BUSY(sc)) {
  1489. if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
  1490. rc = EINTR;
  1491. goto done;
  1492. }
  1493. }
  1494. if (IS_DOOMED(p)) {
  1495. rc = ENXIO;
  1496. goto done;
  1497. }
  1498. KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
  1499. SET_BUSY(sc);
  1500. ADAPTER_UNLOCK(sc);
  1501. rc = cxgb_uninit_synchronized(p);
  1502. ADAPTER_LOCK(sc);
  1503. KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
  1504. CLR_BUSY(sc);
  1505. wakeup_one(&sc->flags);
  1506. done:
  1507. ADAPTER_UNLOCK(sc);
  1508. return (rc);
  1509. }
  1510. /*
  1511. * Called on "ifconfig down", and from port_detach
  1512. */
  1513. static int
  1514. cxgb_uninit_synchronized(struct port_info *pi)
  1515. {
  1516. struct adapter *sc = pi->adapter;
  1517. struct ifnet *ifp = pi->ifp;
  1518. /*
  1519. * taskqueue_drain may cause a deadlock if the adapter lock is held.
  1520. */
  1521. ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
  1522. /*
  1523. * Clear this port's bit from the open device map, and then drain all
  1524. * the tasks that can access/manipulate this port's port_info or ifp.
  1525. * We disable this port's interrupts here and so the slow/ext
  1526. * interrupt tasks won't be enqueued. The tick task will continue to
  1527. * be enqueued every second but the runs after this drain will not see
  1528. * this port in the open device map.
  1529. *
  1530. * A well behaved task must take open_device_map into account and ignore
  1531. * ports that are not open.
  1532. */
  1533. clrbit(&sc->open_device_map, pi->port_id);
  1534. t3_port_intr_disable(sc, pi->port_id);
  1535. taskqueue_drain(sc->tq, &sc->slow_intr_task);
  1536. taskqueue_drain(sc->tq, &sc->tick_task);
  1537. callout_drain(&pi->link_check_ch);
  1538. taskqueue_drain(sc->tq, &pi->link_check_task);
  1539. PORT_LOCK(pi);
  1540. ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
  1541. /* disable pause frames */
  1542. t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
  1543. /* Reset RX FIFO HWM */
  1544. t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
  1545. V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
  1546. DELAY(100 * 1000);
  1547. /* Wait for TXFIFO empty */
  1548. t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
  1549. F_TXFIFO_EMPTY, 1, 20, 5);
  1550. DELAY(100 * 1000);
  1551. t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
  1552. pi->phy.ops->power_down(&pi->phy, 1);
  1553. PORT_UNLOCK(pi);
  1554. pi->link_config.link_ok = 0;
  1555. t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
  1556. if (sc->open_device_map == 0)
  1557. cxgb_down(pi->adapter);
  1558. return (0);
  1559. }
  1560. /*
  1561. * Mark lro enabled or disabled in all qsets for this port
  1562. */
  1563. static int
  1564. cxgb_set_lro(struct port_info *p, int enabled)
  1565. {
  1566. int i;
  1567. struct adapter *adp = p->adapter;
  1568. struct sge_qset *q;
  1569. for (i = 0; i < p->nqsets; i++) {
  1570. q = &adp->sge.qs[p->first_qset + i];
  1571. q->lro.enabled = (enabled != 0);
  1572. }
  1573. return (0);
  1574. }
  1575. static int
  1576. cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
  1577. {
  1578. struct port_info *p = ifp->if_softc;
  1579. struct adapter *sc = p->adapter;
  1580. struct ifreq *ifr = (struct ifreq *)data;
  1581. int flags, error = 0, mtu;
  1582. uint32_t mask;
  1583. switch (command) {
  1584. case SIOCSIFMTU:
  1585. ADAPTER_LOCK(sc);
  1586. error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
  1587. if (error) {
  1588. fail:
  1589. ADAPTER_UNLOCK(sc);
  1590. return (error);
  1591. }
  1592. mtu = ifr->ifr_mtu;
  1593. if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
  1594. error = EINVAL;
  1595. } else {
  1596. ifp->if_mtu = mtu;
  1597. PORT_LOCK(p);
  1598. cxgb_update_mac_settings(p);
  1599. PORT_UNLOCK(p);
  1600. }
  1601. ADAPTER_UNLOCK(sc);
  1602. break;
  1603. case SIOCSIFFLAGS:
  1604. ADAPTER_LOCK(sc);
  1605. if (IS_DOOMED(p)) {
  1606. error = ENXIO;
  1607. goto fail;
  1608. }
  1609. if (ifp->if_flags & IFF_UP) {
  1610. if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  1611. flags = p->if_flags;
  1612. if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
  1613. ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
  1614. if (IS_BUSY(sc)) {
  1615. error = EBUSY;
  1616. goto fail;
  1617. }
  1618. PORT_LOCK(p);
  1619. cxgb_update_mac_settings(p);
  1620. PORT_UNLOCK(p);
  1621. }
  1622. ADAPTER_UNLOCK(sc);
  1623. } else
  1624. error = cxgb_init_locked(p);
  1625. p->if_flags = ifp->if_flags;
  1626. } else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
  1627. error = cxgb_uninit_locked(p);
  1628. else
  1629. ADAPTER_UNLOCK(sc);
  1630. ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
  1631. break;
  1632. case SIOCADDMULTI:
  1633. case SIOCDELMULTI:
  1634. ADAPTER_LOCK(sc);
  1635. error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
  1636. if (error)
  1637. goto fail;
  1638. if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  1639. PORT_LOCK(p);
  1640. cxgb_update_mac_settings(p);
  1641. PORT_UNLOCK(p);
  1642. }
  1643. ADAPTER_UNLOCK(sc);
  1644. break;
  1645. case SIOCSIFCAP:
  1646. ADAPTER_LOCK(sc);
  1647. error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
  1648. if (error)
  1649. goto fail;
  1650. mask = ifr->ifr_reqcap ^ ifp->if_capenable;
  1651. if (mask & IFCAP_TXCSUM) {
  1652. ifp->if_capenable ^= IFCAP_TXCSUM;
  1653. ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
  1654. if (IFCAP_TSO4 & ifp->if_capenable &&
  1655. !(IFCAP_TXCSUM & ifp->if_capenable)) {
  1656. mask &= ~IFCAP_TSO4;
  1657. ifp->if_capenable &= ~IFCAP_TSO4;
  1658. if_printf(ifp,
  1659. "tso4 disabled due to -txcsum.\n");
  1660. }
  1661. }
  1662. if (mask & IFCAP_TXCSUM_IPV6) {
  1663. ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
  1664. ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
  1665. if (IFCAP_TSO6 & ifp->if_capenable &&
  1666. !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
  1667. mask &= ~IFCAP_TSO6;
  1668. ifp->if_capenable &= ~IFCAP_TSO6;
  1669. if_printf(ifp,
  1670. "tso6 disabled due to -txcsum6.\n");
  1671. }
  1672. }
  1673. if (mask & IFCAP_RXCSUM)
  1674. ifp->if_capenable ^= IFCAP_RXCSUM;
  1675. if (mask & IFCAP_RXCSUM_IPV6)
  1676. ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
  1677. /*
  1678. * Note that we leave CSUM_TSO alone (it is always set). The
  1679. * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
  1680. * sending a TSO request our way, so it's sufficient to toggle
  1681. * IFCAP_TSOx only.
  1682. */
  1683. if (mask & IFCAP_TSO4) {
  1684. if (!(IFCAP_TSO4 & ifp->if_capenable) &&
  1685. !(IFCAP_TXCSUM & ifp->if_capenable)) {
  1686. if_printf(ifp, "enable txcsum first.\n");
  1687. error = EAGAIN;
  1688. goto fail;
  1689. }
  1690. ifp->if_capenable ^= IFCAP_TSO4;
  1691. }
  1692. if (mask & IFCAP_TSO6) {
  1693. if (!(IFCAP_TSO6 & ifp->if_capenable) &&
  1694. !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
  1695. if_printf(ifp, "enable txcsum6 first.\n");
  1696. error = EAGAIN;
  1697. goto fail;
  1698. }
  1699. ifp->if_capenable ^= IFCAP_TSO6;
  1700. }
  1701. if (mask & IFCAP_LRO) {
  1702. ifp->if_capenable ^= IFCAP_LRO;
  1703. /* Safe to do this even if cxgb_up not called yet */
  1704. cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
  1705. }
  1706. #ifdef TCP_OFFLOAD
  1707. if (mask & IFCAP_TOE4) {
  1708. int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
  1709. error = toe_capability(p, enable);
  1710. if (error == 0)
  1711. ifp->if_capenable ^= mask;
  1712. }
  1713. #endif
  1714. if (mask & IFCAP_VLAN_HWTAGGING) {
  1715. ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
  1716. if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  1717. PORT_LOCK(p);
  1718. cxgb_update_mac_settings(p);
  1719. PORT_UNLOCK(p);
  1720. }
  1721. }
  1722. if (mask & IFCAP_VLAN_MTU) {
  1723. ifp->if_capenable ^= IFCAP_VLAN_MTU;
  1724. if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
  1725. PORT_LOCK(p);
  1726. cxgb_update_mac_settings(p);
  1727. PORT_UNLOCK(p);
  1728. }
  1729. }
  1730. if (mask & IFCAP_VLAN_HWTSO)
  1731. ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
  1732. if (mask & IFCAP_VLAN_HWCSUM)
  1733. ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
  1734. #ifdef VLAN_CAPABILITIES
  1735. VLAN_CAPABILITIES(ifp);
  1736. #endif
  1737. ADAPTER_UNLOCK(sc);
  1738. break;
  1739. case SIOCSIFMEDIA:
  1740. case SIOCGIFMEDIA:
  1741. error = ifmedia_ioctl(ifp, ifr, &p->media, command);
  1742. break;
  1743. default:
  1744. error = ether_ioctl(ifp, command, data);
  1745. }
  1746. return (error);
  1747. }
  1748. static int
  1749. cxgb_media_change(struct ifnet *ifp)
  1750. {
  1751. return (EOPNOTSUPP);
  1752. }
  1753. /*
  1754. * Translates phy->modtype to the correct Ethernet media subtype.
  1755. */
  1756. static int
  1757. cxgb_ifm_type(int mod)
  1758. {
  1759. switch (mod) {
  1760. case phy_modtype_sr:
  1761. return (IFM_10G_SR);
  1762. case phy_modtype_lr:
  1763. return (IFM_10G_LR);
  1764. case phy_modtype_lrm:
  1765. return (IFM_10G_LRM);
  1766. case phy_modtype_twinax:
  1767. return (IFM_10G_TWINAX);
  1768. case phy_modtype_twinax_long:
  1769. return (IFM_10G_TWINAX_LONG);
  1770. case phy_modtype_none:
  1771. return (IFM_NONE);
  1772. case phy_modtype_unknown:
  1773. return (IFM_UNKNOWN);
  1774. }
  1775. KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
  1776. return (IFM_UNKNOWN);
  1777. }
  1778. /*
  1779. * Rebuilds the ifmedia list for this port, and sets the current media.
  1780. */
  1781. static void
  1782. cxgb_build_medialist(struct port_info *p)
  1783. {
  1784. struct cphy *phy = &p->phy;
  1785. struct ifmedia *media = &p->media;
  1786. int mod = phy->modtype;
  1787. int m = IFM_ETHER | IFM_FDX;
  1788. PORT_LOCK(p);
  1789. ifmedia_removeall(media);
  1790. if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
  1791. /* Copper (RJ45) */
  1792. if (phy->caps & SUPPORTED_10000baseT_Full)
  1793. ifmedia_add(media, m | IFM_10G_T, mod, NULL);
  1794. if (phy->caps & SUPPORTED_1000baseT_Full)
  1795. ifmedia_add(media, m | IFM_1000_T, mod, NULL);
  1796. if (phy->caps & SUPPORTED_100baseT_Full)
  1797. ifmedia_add(media, m | IFM_100_TX, mod, NULL);
  1798. if (phy->caps & SUPPORTED_10baseT_Full)
  1799. ifmedia_add(media, m | IFM_10_T, mod, NULL);
  1800. ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
  1801. ifmedia_set(media, IFM_ETHER | IFM_AUTO);
  1802. } else if (phy->caps & SUPPORTED_TP) {
  1803. /* Copper (CX4) */
  1804. KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
  1805. ("%s: unexpected cap 0x%x", __func__, phy->caps));
  1806. ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
  1807. ifmedia_set(media, m | IFM_10G_CX4);
  1808. } else if (phy->caps & SUPPORTED_FIBRE &&
  1809. phy->caps & SUPPORTED_10000baseT_Full) {
  1810. /* 10G optical (but includes SFP+ twinax) */
  1811. m |= cxgb_ifm_type(mod);
  1812. if (IFM_SUBTYPE(m) == IFM_NONE)
  1813. m &= ~IFM_FDX;
  1814. ifmedia_add(media, m, mod, NULL);
  1815. ifmedia_set(media, m);
  1816. } else if (phy->caps & SUPPORTED_FIBRE &&
  1817. phy->caps & SUPPORTED_1000baseT_Full) {
  1818. /* 1G optical */
  1819. /* XXX: Lie and claim to be SX, could actually be any 1G-X */
  1820. ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
  1821. ifmedia_set(media, m | IFM_1000_SX);
  1822. } else {
  1823. KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
  1824. phy->caps));
  1825. }
  1826. PORT_UNLOCK(p);
  1827. }
  1828. static void
  1829. cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
  1830. {
  1831. struct port_info *p = ifp->if_softc;
  1832. struct ifmedia_entry *cur = p->media.ifm_cur;
  1833. int speed = p->link_config.speed;
  1834. if (cur->ifm_data != p->phy.modtype) {
  1835. cxgb_build_medialist(p);
  1836. cur = p->media.ifm_cur;
  1837. }
  1838. ifmr->ifm_status = IFM_AVALID;
  1839. if (!p->link_config.link_ok)
  1840. return;
  1841. ifmr->ifm_status |= IFM_ACTIVE;
  1842. /*
  1843. * active and current will differ iff current media is autoselect. That
  1844. * can happen only for copper RJ45.
  1845. */
  1846. if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
  1847. return;
  1848. KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
  1849. ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
  1850. ifmr->ifm_active = IFM_ETHER | IFM_FDX;
  1851. if (speed == SPEED_10000)
  1852. ifmr->ifm_active |= IFM_10G_T;
  1853. else if (speed == SPEED_1000)
  1854. ifmr->ifm_active |= IFM_1000_T;
  1855. else if (speed == SPEED_100)
  1856. ifmr->ifm_active |= IFM_100_TX;
  1857. else if (speed == SPEED_10)
  1858. ifmr->ifm_active |= IFM_10_T;
  1859. else
  1860. KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
  1861. speed));
  1862. }
  1863. static uint64_t
  1864. cxgb_get_counter(struct ifnet *ifp, ift_counter c)
  1865. {
  1866. struct port_info *pi = ifp->if_softc;
  1867. struct adapter *sc = pi->adapter;
  1868. struct cmac *mac = &pi->mac;
  1869. struct mac_stats *mstats = &mac->stats;
  1870. cxgb_refresh_stats(pi);
  1871. switch (c) {
  1872. case IFCOUNTER_IPACKETS:
  1873. return (mstats->rx_frames);
  1874. case IFCOUNTER_IERRORS:
  1875. return (mstats->rx_jabber + mstats->rx_data_errs +
  1876. mstats->rx_sequence_errs + mstats->rx_runt +
  1877. mstats->rx_too_long + mstats->rx_mac_internal_errs +
  1878. mstats->rx_short + mstats->rx_fcs_errs);
  1879. case IFCOUNTER_OPACKETS:
  1880. return (mstats->tx_frames);
  1881. case IFCOUNTER_OERRORS:
  1882. return (mstats->tx_excess_collisions + mstats->tx_underrun +
  1883. mstats->tx_len_errs + mstats->tx_mac_internal_errs +
  1884. mstats->tx_excess_deferral + mstats->tx_fcs_errs);
  1885. case IFCOUNTER_COLLISIONS:
  1886. return (mstats->tx_total_collisions);
  1887. case IFCOUNTER_IBYTES:
  1888. return (mstats->rx_octets);
  1889. case IFCOUNTER_OBYTES:
  1890. return (mstats->tx_octets);
  1891. case IFCOUNTER_IMCASTS:
  1892. return (mstats->rx_mcast_frames);
  1893. case IFCOUNTER_OMCASTS:
  1894. return (mstats->tx_mcast_frames);
  1895. case IFCOUNTER_IQDROPS:
  1896. return (mstats->rx_cong_drops);
  1897. case IFCOUNTER_OQDROPS: {
  1898. int i;
  1899. uint64_t drops;
  1900. drops = 0;
  1901. if (sc->flags & FULL_INIT_DONE) {
  1902. for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
  1903. drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
  1904. }
  1905. return (drops);
  1906. }
  1907. default:
  1908. return (if_get_counter_default(ifp, c));
  1909. }
  1910. }
  1911. static void
  1912. cxgb_async_intr(void *data)
  1913. {
  1914. adapter_t *sc = data;
  1915. t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
  1916. (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
  1917. taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
  1918. }
  1919. static void
  1920. link_check_callout(void *arg)
  1921. {
  1922. struct port_info *pi = arg;
  1923. struct adapter *sc = pi->adapter;
  1924. if (!isset(&sc->open_device_map, pi->port_id))
  1925. return;
  1926. taskqueue_enqueue(sc->tq, &pi->link_check_task);
  1927. }
  1928. static void
  1929. check_link_status(void *arg, int pending)
  1930. {
  1931. struct port_info *pi = arg;
  1932. struct adapter *sc = pi->adapter;
  1933. if (!isset(&sc->open_device_map, pi->port_id))
  1934. return;
  1935. t3_link_changed(sc, pi->port_id);
  1936. if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
  1937. pi->link_config.link_ok == 0)
  1938. callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
  1939. }
  1940. void
  1941. t3_os_link_intr(struct port_info *pi)
  1942. {
  1943. /*
  1944. * Schedule a link check in the near future. If the link is flapping
  1945. * rapidly we'll keep resetting the callout and delaying the check until
  1946. * things stabilize a bit.
  1947. */
  1948. callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
  1949. }
  1950. static void
  1951. check_t3b2_mac(struct adapter *sc)
  1952. {
  1953. int i;
  1954. if (sc->flags & CXGB_SHUTDOWN)
  1955. return;
  1956. for_each_port(sc, i) {
  1957. struct port_info *p = &sc->port[i];
  1958. int status;
  1959. #ifdef INVARIANTS
  1960. struct ifnet *ifp = p->ifp;
  1961. #endif
  1962. if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
  1963. !p->link_config.link_ok)
  1964. continue;
  1965. KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
  1966. ("%s: state mismatch (drv_flags %x, device_map %x)",
  1967. __func__, ifp->if_drv_flags, sc->open_device_map));
  1968. PORT_LOCK(p);
  1969. status = t3b2_mac_watchdog_task(&p->mac);
  1970. if (status == 1)
  1971. p->mac.stats.num_toggled++;
  1972. else if (status == 2) {
  1973. struct cmac *mac = &p->mac;
  1974. cxgb_update_mac_settings(p);
  1975. t3_link_start(&p->phy, mac, &p->link_config);
  1976. t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
  1977. t3_port_intr_enable(sc, p->port_id);
  1978. p->mac.stats.num_resets++;
  1979. }
  1980. PORT_UNLOCK(p);
  1981. }
  1982. }
  1983. static void
  1984. cxgb_tick(void *arg)
  1985. {
  1986. adapter_t *sc = (adapter_t *)arg;
  1987. if (sc->flags & CXGB_SHUTDOWN)
  1988. return;
  1989. taskqueue_enqueue(sc->tq, &sc->tick_task);
  1990. callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
  1991. }
  1992. void
  1993. cxgb_refresh_stats(struct port_info *pi)
  1994. {
  1995. struct timeval tv;
  1996. const struct timeval interval = {0, 250000}; /* 250ms */
  1997. getmicrotime(&tv);
  1998. timevalsub(&tv, &interval);
  1999. if (timevalcmp(&tv, &pi->last_refreshed, <))
  2000. return;
  2001. PORT_LOCK(pi);
  2002. t3_mac_update_stats(&pi->mac);
  2003. PORT_UNLOCK(pi);
  2004. getmicrotime(&pi->last_refreshed);
  2005. }
  2006. static void
  2007. cxgb_tick_handler(void *arg, int count)
  2008. {
  2009. adapter_t *sc = (adapter_t *)arg;
  2010. const struct adapter_params *p = &sc->params;
  2011. int i;
  2012. uint32_t cause, reset;
  2013. if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
  2014. return;
  2015. if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
  2016. check_t3b2_mac(sc);
  2017. cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
  2018. if (cause) {
  2019. struct sge_qset *qs = &sc->sge.qs[0];
  2020. uint32_t mask, v;
  2021. v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
  2022. mask = 1;
  2023. for (i = 0; i < SGE_QSETS; i++) {
  2024. if (v & mask)
  2025. qs[i].rspq.starved++;
  2026. mask <<= 1;
  2027. }
  2028. mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
  2029. for (i = 0; i < SGE_QSETS * 2; i++) {
  2030. if (v & mask) {
  2031. qs[i / 2].fl[i % 2].empty++;
  2032. }
  2033. mask <<= 1;
  2034. }
  2035. /* clear */
  2036. t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
  2037. t3_write_reg(sc, A_SG_INT_CAUSE, cause);
  2038. }
  2039. for (i = 0; i < sc->params.nports; i++) {
  2040. struct port_info *pi = &sc->port[i];
  2041. struct cmac *mac = &pi->mac;
  2042. if (!isset(&sc->open_device_map, pi->port_id))
  2043. continue;
  2044. cxgb_refresh_stats(pi);
  2045. if (mac->multiport)
  2046. continue;
  2047. /* Count rx fifo overflows, once per second */
  2048. cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
  2049. reset = 0;
  2050. if (cause & F_RXFIFO_OVERFLOW) {
  2051. mac->stats.rx_fifo_ovfl++;
  2052. reset |= F_RXFIFO_OVERFLOW;
  2053. }
  2054. t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
  2055. }
  2056. }
  2057. static void
  2058. touch_bars(device_t dev)
  2059. {
  2060. /*
  2061. * Don't enable yet
  2062. */
  2063. #if !defined(__LP64__) && 0
  2064. u32 v;
  2065. pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
  2066. pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
  2067. pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
  2068. pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
  2069. pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
  2070. pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
  2071. #endif
  2072. }
  2073. static int
  2074. set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
  2075. {
  2076. uint8_t *buf;
  2077. int err = 0;
  2078. u32 aligned_offset, aligned_len, *p;
  2079. struct adapter *adapter = pi->adapter;
  2080. aligned_offset = offset & ~3;
  2081. aligned_len = (len + (offset & 3) + 3) & ~3;
  2082. if (aligned_offset != offset || aligned_len != len) {
  2083. buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
  2084. if (!buf)
  2085. return (ENOMEM);
  2086. err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
  2087. if (!err && aligned_len > 4)
  2088. err = t3_seeprom_read(adapter,
  2089. aligned_offset + aligned_len - 4,
  2090. (u32 *)&buf[aligned_len - 4]);
  2091. if (err)
  2092. goto out;
  2093. memcpy(buf + (offset & 3), data, len);
  2094. } else
  2095. buf = (uint8_t *)(uintptr_t)data;
  2096. err = t3_seeprom_wp(adapter, 0);
  2097. if (err)
  2098. goto out;
  2099. for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
  2100. err = t3_seeprom_write(adapter, aligned_offset, *p);
  2101. aligned_offset += 4;
  2102. }
  2103. if (!err)
  2104. err = t3_seeprom_wp(adapter, 1);
  2105. out:
  2106. if (buf != data)
  2107. free(buf, M_DEVBUF);
  2108. return err;
  2109. }
  2110. static int
  2111. in_range(int val, int lo, int hi)
  2112. {
  2113. return val < 0 || (val <= hi && val >= lo);
  2114. }
  2115. static int
  2116. cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
  2117. {
  2118. return (0);
  2119. }
  2120. static int
  2121. cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
  2122. {
  2123. return (0);
  2124. }
  2125. static int
  2126. cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
  2127. int fflag, struct thread *td)
  2128. {
  2129. int mmd, error = 0;
  2130. struct port_info *pi = dev->si_drv1;
  2131. adapter_t *sc = pi->adapter;
  2132. #ifdef PRIV_SUPPORTED
  2133. if (priv_check(td, PRIV_DRIVER)) {
  2134. if (cxgb_debug)
  2135. printf("user does not have access to privileged ioctls\n");
  2136. return (EPERM);
  2137. }
  2138. #else
  2139. if (suser(td)) {
  2140. if (cxgb_debug)
  2141. printf("user does not have access to privileged ioctls\n");
  2142. return (EPERM);
  2143. }
  2144. #endif
  2145. switch (cmd) {
  2146. case CHELSIO_GET_MIIREG: {
  2147. uint32_t val;
  2148. struct cphy *phy = &pi->phy;
  2149. struct ch_mii_data *mid = (struct ch_mii_data *)data;
  2150. if (!phy->mdio_read)
  2151. return (EOPNOTSUPP);
  2152. if (is_10G(sc)) {
  2153. mmd = mid->phy_id >> 8;
  2154. if (!mmd)
  2155. mmd = MDIO_DEV_PCS;
  2156. else if (mmd > MDIO_DEV_VEND2)
  2157. return (EINVAL);
  2158. error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
  2159. mid->reg_num, &val);
  2160. } else
  2161. error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
  2162. mid->reg_num & 0x1f, &val);
  2163. if (error == 0)
  2164. mid->val_out = val;
  2165. break;
  2166. }
  2167. case CHELSIO_SET_MIIREG: {
  2168. struct cphy *phy = &pi->phy;
  2169. struct ch_mii_data *mid = (struct ch_mii_data *)data;
  2170. if (!phy->mdio_write)
  2171. return (EOPNOTSUPP);
  2172. if (is_10G(sc)) {
  2173. mmd = mid->phy_id >> 8;
  2174. if (!mmd)
  2175. mmd = MDIO_DEV_PCS;
  2176. else if (mmd > MDIO_DEV_VEND2)
  2177. return (EINVAL);
  2178. error = phy->mdio_write(sc, mid->phy_id & 0x1f,
  2179. mmd, mid->reg_num, mid->val_in);
  2180. } else
  2181. error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
  2182. mid->reg_num & 0x1f,
  2183. mid->val_in);
  2184. break;
  2185. }
  2186. case CHELSIO_SETREG: {
  2187. struct ch_reg *edata = (struct ch_reg *)data;
  2188. if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
  2189. return (EFAULT);
  2190. t3_write_reg(sc, edata->addr, edata->val);
  2191. break;
  2192. }
  2193. case CHELSIO_GETREG: {
  2194. struct ch_reg *edata = (struct ch_reg *)data;
  2195. if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
  2196. return (EFAULT);
  2197. edata->val = t3_read_reg(sc, edata->addr);
  2198. break;
  2199. }
  2200. case CHELSIO_GET_SGE_CONTEXT: {
  2201. struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
  2202. mtx_lock_spin(&sc->sge.reg_lock);
  2203. switch (ecntxt->cntxt_type) {
  2204. case CNTXT_TYPE_EGRESS:
  2205. error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
  2206. ecntxt->data);
  2207. break;
  2208. case CNTXT_TYPE_FL:
  2209. error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
  2210. ecntxt->data);
  2211. break;
  2212. case CNTXT_TYPE_RSP:
  2213. error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
  2214. ecntxt->data);
  2215. break;
  2216. case CNTXT_TYPE_CQ:
  2217. error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
  2218. ecntxt->data);
  2219. break;
  2220. default:
  2221. error = EINVAL;
  2222. break;
  2223. }
  2224. mtx_unlock_spin(&sc->sge.reg_lock);
  2225. break;
  2226. }
  2227. case CHELSIO_GET_SGE_DESC: {
  2228. struct ch_desc *edesc = (struct ch_desc *)data;
  2229. int ret;
  2230. if (edesc->queue_num >= SGE_QSETS * 6)
  2231. return (EINVAL);
  2232. ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
  2233. edesc->queue_num % 6, edesc->idx, edesc->data);
  2234. if (ret < 0)
  2235. return (EINVAL);
  2236. edesc->size = ret;
  2237. break;
  2238. }
  2239. case CHELSIO_GET_QSET_PARAMS: {
  2240. struct qset_params *q;
  2241. struct ch_qset_params *t = (struct ch_qset_params *)data;
  2242. int q1 = pi->first_qset;
  2243. int nqsets = pi->nqsets;
  2244. int i;
  2245. if (t->qset_idx >= nqsets)
  2246. return EINVAL;
  2247. i = q1 + t->qset_idx;
  2248. q = &sc->params.sge.qset[i];
  2249. t->rspq_size = q->rspq_size;
  2250. t->txq_size[0] = q->txq_size[0];
  2251. t->txq_size[1] = q->txq_size[1];
  2252. t->txq_size[2] = q->txq_size[2];
  2253. t->fl_size[0] = q->fl_size;
  2254. t->fl_size[1] = q->jumbo_size;
  2255. t->polling = q->polling;
  2256. t->lro = q->lro;
  2257. t->intr_lat = q->coalesce_usecs;
  2258. t->cong_thres = q->cong_thres;
  2259. t->qnum = i;
  2260. if ((sc->flags & FULL_INIT_DONE) == 0)
  2261. t->vector = 0;
  2262. else if (sc->flags & USING_MSIX)
  2263. t->vector = rman_get_start(sc->msix_irq_res[i]);
  2264. else
  2265. t->vector = rman_get_start(sc->irq_res);
  2266. break;
  2267. }
  2268. case CHELSIO_GET_QSET_NUM: {
  2269. struct ch_reg *edata = (struct ch_reg *)data;
  2270. edata->val = pi->nqsets;
  2271. break;
  2272. }
  2273. case CHELSIO_LOAD_FW: {
  2274. uint8_t *fw_data;
  2275. uint32_t vers;
  2276. struct ch_mem_range *t = (struct ch_mem_range *)data;
  2277. /*
  2278. * You're allowed to load a firmware only before FULL_INIT_DONE
  2279. *
  2280. * FW_UPTODATE is also set so the rest of the initialization
  2281. * will not overwrite what was loaded here. This gives you the
  2282. * flexibility to load any firmware (and maybe shoot yourself in
  2283. * the foot).
  2284. */
  2285. ADAPTER_LOCK(sc);
  2286. if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
  2287. ADAPTER_UNLOCK(sc);
  2288. return (EBUSY);
  2289. }
  2290. fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
  2291. if (!fw_data)
  2292. error = ENOMEM;
  2293. else
  2294. error = copyin(t->buf, fw_data, t->len);
  2295. if (!error)
  2296. error = -t3_load_fw(sc, fw_data, t->len);
  2297. if (t3_get_fw_version(sc, &vers) == 0) {
  2298. snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
  2299. "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
  2300. G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
  2301. }
  2302. if (!error)
  2303. sc->flags |= FW_UPTODATE;
  2304. free(fw_data, M_DEVBUF);
  2305. ADAPTER_UNLOCK(sc);
  2306. break;
  2307. }
  2308. case CHELSIO_LOAD_BOOT: {
  2309. uint8_t *boot_data;
  2310. struct ch_mem_range *t = (struct ch_mem_range *)data;
  2311. boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
  2312. if (!boot_data)
  2313. return ENOMEM;
  2314. error = copyin(t->buf, boot_data, t->len);
  2315. if (!error)
  2316. error = -t3_load_boot(sc, boot_data, t->len);
  2317. free(boot_data, M_DEVBUF);
  2318. break;
  2319. }
  2320. case CHELSIO_GET_PM: {
  2321. struct ch_pm *m = (struct ch_pm *)data;
  2322. struct tp_params *p = &sc->params.tp;
  2323. if (!is_offload(sc))
  2324. return (EOPNOTSUPP);
  2325. m->tx_pg_sz = p->tx_pg_size;
  2326. m->tx_num_pg = p->tx_num_pgs;
  2327. m->rx_pg_sz = p->rx_pg_size;
  2328. m->rx_num_pg = p->rx_num_pgs;
  2329. m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
  2330. break;
  2331. }
  2332. case CHELSIO_SET_PM: {
  2333. struct ch_pm *m = (struct ch_pm *)data;
  2334. struct tp_params *p = &sc->params.tp;
  2335. if (!is_offload(sc))
  2336. return (EOPNOTSUPP);
  2337. if (sc->flags & FULL_INIT_DONE)
  2338. return (EBUSY);
  2339. if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
  2340. !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
  2341. return (EINVAL); /* not power of 2 */
  2342. if (!(m->rx_pg_sz & 0x14000))
  2343. return (EINVAL); /* not 16KB or 64KB */
  2344. if (!(m->tx_pg_sz & 0x1554000))
  2345. return (EINVAL);
  2346. if (m->tx_num_pg == -1)
  2347. m->tx_num_pg = p->tx_num_pgs;
  2348. if (m->rx_num_pg == -1)
  2349. m->rx_num_pg = p->rx_num_pgs;
  2350. if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
  2351. return (EINVAL);
  2352. if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
  2353. m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
  2354. return (EINVAL);
  2355. p->rx_pg_size = m->rx_pg_sz;
  2356. p->tx_pg_size = m->tx_pg_sz;
  2357. p->rx_num_pgs = m->rx_num_pg;
  2358. p->tx_num_pgs = m->tx_num_pg;
  2359. break;
  2360. }
  2361. case CHELSIO_SETMTUTAB: {
  2362. struct ch_mtus *m = (struct ch_mtus *)data;
  2363. int i;
  2364. if (!is_offload(sc))
  2365. return (EOPNOTSUPP);
  2366. if (offload_running(sc))
  2367. return (EBUSY);
  2368. if (m->nmtus != NMTUS)
  2369. return (EINVAL);
  2370. if (m->mtus[0] < 81) /* accommodate SACK */
  2371. return (EINVAL);
  2372. /*
  2373. * MTUs must be in ascending order
  2374. */
  2375. for (i = 1; i < NMTUS; ++i)
  2376. if (m->mtus[i] < m->mtus[i - 1])
  2377. return (EINVAL);
  2378. memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
  2379. break;
  2380. }
  2381. case CHELSIO_GETMTUTAB: {
  2382. struct ch_mtus *m = (struct ch_mtus *)data;
  2383. if (!is_offload(sc))
  2384. return (EOPNOTSUPP);
  2385. memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
  2386. m->nmtus = NMTUS;
  2387. break;
  2388. }
  2389. case CHELSIO_GET_MEM: {
  2390. struct ch_mem_range *t = (struct ch_mem_range *)data;
  2391. struct mc7 *mem;
  2392. uint8_t *useraddr;
  2393. u64 buf[32];
  2394. /*
  2395. * Use these to avoid modifying len/addr in the return
  2396. * struct
  2397. */
  2398. uint32_t len = t->len, addr = t->addr;
  2399. if (!is_offload(sc))
  2400. return (EOPNOTSUPP);
  2401. if (!(sc->flags & FULL_INIT_DONE))
  2402. return (EIO); /* need the memory controllers */
  2403. if ((addr & 0x7) || (len & 0x7))
  2404. return (EINVAL);
  2405. if (t->mem_id == MEM_CM)
  2406. mem = &sc->cm;
  2407. else if (t->mem_id == MEM_PMRX)
  2408. mem = &sc->pmrx;
  2409. else if (t->mem_id == MEM_PMTX)
  2410. mem = &sc->pmtx;
  2411. else
  2412. return (EINVAL);
  2413. /*
  2414. * Version scheme:
  2415. * bits 0..9: chip version
  2416. * bits 10..15: chip revision
  2417. */
  2418. t->version = 3 | (sc->params.rev << 10);
  2419. /*
  2420. * Read 256 bytes at a time as len can be large and we don't
  2421. * want to use huge intermediate buffers.
  2422. */
  2423. useraddr = (uint8_t *)t->buf;
  2424. while (len) {
  2425. unsigned int chunk = min(len, sizeof(buf));
  2426. error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
  2427. if (error)
  2428. return (-error);
  2429. if (copyout(buf, useraddr, chunk))
  2430. return (EFAULT);
  2431. useraddr += chunk;
  2432. addr += chunk;
  2433. len -= chunk;
  2434. }
  2435. break;
  2436. }
  2437. case CHELSIO_READ_TCAM_WORD: {
  2438. struct ch_tcam_word *t = (struct ch_tcam_word *)data;
  2439. if (!is_offload(sc))
  2440. return (EOPNOTSUPP);
  2441. if (!(sc->flags & FULL_INIT_DONE))
  2442. return (EIO); /* need MC5 */
  2443. return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
  2444. break;
  2445. }
  2446. case CHELSIO_SET_TRACE_FILTER: {
  2447. struct ch_trace *t = (struct ch_trace *)data;
  2448. const struct trace_params *tp;
  2449. tp = (const struct trace_params *)&t->sip;
  2450. if (t->config_tx)
  2451. t3_config_trace_filter(sc, tp, 0, t->invert_match,
  2452. t->trace_tx);
  2453. if (t->config_rx)
  2454. t3_config_trace_filter(sc, tp, 1, t->invert_match,
  2455. t->trace_rx);
  2456. break;
  2457. }
  2458. case CHELSIO_SET_PKTSCHED: {
  2459. struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
  2460. if (sc->open_device_map == 0)
  2461. return (EAGAIN);
  2462. send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
  2463. p->binding);
  2464. break;
  2465. }
  2466. case CHELSIO_IFCONF_GETREGS: {
  2467. struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
  2468. int reglen = cxgb_get_regs_len();
  2469. uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
  2470. if (buf == NULL) {
  2471. return (ENOMEM);
  2472. }
  2473. if (regs->len > reglen)
  2474. regs->len = reglen;
  2475. else if (regs->len < reglen)
  2476. error = ENOBUFS;
  2477. if (!error) {
  2478. cxgb_get_regs(sc, regs, buf);
  2479. error = copyout(buf, regs->data, reglen);
  2480. }
  2481. free(buf, M_DEVBUF);
  2482. break;
  2483. }
  2484. case CHELSIO_SET_HW_SCHED: {
  2485. struct ch_hw_sched *t = (struct ch_hw_sched *)data;
  2486. unsigned int ticks_per_usec = core_ticks_per_usec(sc);
  2487. if ((sc->flags & FULL_INIT_DONE) == 0)
  2488. return (EAGAIN); /* need TP to be initialized */
  2489. if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
  2490. !in_range(t->channel, 0, 1) ||
  2491. !in_range(t->kbps, 0, 10000000) ||
  2492. !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
  2493. !in_range(t->flow_ipg, 0,
  2494. dack_ticks_to_usec(sc, 0x7ff)))
  2495. return (EINVAL);
  2496. if (t->kbps >= 0) {
  2497. error = t3_config_sched(sc, t->kbps, t->sched);
  2498. if (error < 0)
  2499. return (-error);
  2500. }
  2501. if (t->class_ipg >= 0)
  2502. t3_set_sched_ipg(sc, t->sched, t->class_ipg);
  2503. if (t->flow_ipg >= 0) {
  2504. t->flow_ipg *= 1000; /* us -> ns */
  2505. t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
  2506. }
  2507. if (t->mode >= 0) {
  2508. int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
  2509. t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
  2510. bit, t->mode ? bit : 0);
  2511. }
  2512. if (t->channel >= 0)
  2513. t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
  2514. 1 << t->sched, t->channel << t->sched);
  2515. break;
  2516. }
  2517. case CHELSIO_GET_EEPROM: {
  2518. int i;
  2519. struct ch_eeprom *e = (struct ch_eeprom *)data;
  2520. uint8_t *buf;
  2521. if (e->offset & 3 || e->offset >= EEPROMSIZE ||
  2522. e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) {
  2523. return (EINVAL);
  2524. }
  2525. buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
  2526. if (buf == NULL) {
  2527. return (ENOMEM);
  2528. }
  2529. e->magic = EEPROM_MAGIC;
  2530. for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
  2531. error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
  2532. if (!error)
  2533. error = copyout(buf + e->offset, e->data, e->len);
  2534. free(buf, M_DEVBUF);
  2535. break;
  2536. }
  2537. case CHELSIO_CLEAR_STATS: {
  2538. if (!(sc->flags & FULL_INIT_DONE))
  2539. return EAGAIN;
  2540. PORT_LOCK(pi);
  2541. t3_mac_update_stats(&pi->mac);
  2542. memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
  2543. PORT_UNLOCK(pi);
  2544. break;
  2545. }
  2546. case CHELSIO_GET_UP_LA: {
  2547. struct ch_up_la *la = (struct ch_up_la *)data;
  2548. uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
  2549. if (buf == NULL) {
  2550. return (ENOMEM);
  2551. }
  2552. if (la->bufsize < LA_BUFSIZE)
  2553. error = ENOBUFS;
  2554. if (!error)
  2555. error = -t3_get_up_la(sc, &la->stopped, &la->idx,
  2556. &la->bufsize, buf);
  2557. if (!error)
  2558. error = copyout(buf, la->data, la->bufsize);
  2559. free(buf, M_DEVBUF);
  2560. break;
  2561. }
  2562. case CHELSIO_GET_UP_IOQS: {
  2563. struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
  2564. uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
  2565. uint32_t *v;
  2566. if (buf == NULL) {
  2567. return (ENOMEM);
  2568. }
  2569. if (ioqs->bufsize < IOQS_BUFSIZE)
  2570. error = ENOBUFS;
  2571. if (!error)
  2572. error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
  2573. if (!error) {
  2574. v = (uint32_t *)buf;
  2575. ioqs->ioq_rx_enable = *v++;
  2576. ioqs->ioq_tx_enable = *v++;
  2577. ioqs->ioq_rx_status = *v++;
  2578. ioqs->ioq_tx_status = *v++;
  2579. error = copyout(v, ioqs->data, ioqs->bufsize);
  2580. }
  2581. free(buf, M_DEVBUF);
  2582. break;
  2583. }
  2584. case CHELSIO_SET_FILTER: {
  2585. struct ch_filter *f = (struct ch_filter *)data;
  2586. struct filter_info *p;
  2587. unsigned int nfilters = sc->params.mc5.nfilters;
  2588. if (!is_offload(sc))
  2589. return (EOPNOTSUPP); /* No TCAM */
  2590. if (!(sc->flags & FULL_INIT_DONE))
  2591. return (EAGAIN); /* mc5 not setup yet */
  2592. if (nfilters == 0)
  2593. return (EBUSY); /* TOE will use TCAM */
  2594. /* sanity checks */
  2595. if (f->filter_id >= nfilters ||
  2596. (f->val.dip && f->mask.dip != 0xffffffff) ||
  2597. (f->val.sport && f->mask.sport != 0xffff) ||
  2598. (f->val.dport && f->mask.dport != 0xffff) ||
  2599. (f->val.vlan && f->mask.vlan != 0xfff) ||
  2600. (f->val.vlan_prio &&
  2601. f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
  2602. (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
  2603. f->qset >= SGE_QSETS ||
  2604. sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
  2605. return (EINVAL);
  2606. /* Was allocated with M_WAITOK */
  2607. KASSERT(sc->filters, ("filter table NULL\n"));
  2608. p = &sc->filters[f->filter_id];
  2609. if (p->locked)
  2610. return (EPERM);
  2611. bzero(p, sizeof(*p));
  2612. p->sip = f->val.sip;
  2613. p->sip_mask = f->mask.sip;
  2614. p->dip = f->val.dip;
  2615. p->sport = f->val.sport;
  2616. p->dport = f->val.dport;
  2617. p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
  2618. p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
  2619. FILTER_NO_VLAN_PRI;
  2620. p->mac_hit = f->mac_hit;
  2621. p->mac_vld = f->mac_addr_idx != 0xffff;
  2622. p->mac_idx = f->mac_addr_idx;
  2623. p->pkt_type = f->proto;
  2624. p->report_filter_id = f->want_filter_id;
  2625. p->pass = f->pass;
  2626. p->rss = f->rss;
  2627. p->qset = f->qset;
  2628. error = set_filter(sc, f->filter_id, p);
  2629. if (error == 0)
  2630. p->valid = 1;
  2631. break;
  2632. }
  2633. case CHELSIO_DEL_FILTER: {
  2634. struct ch_filter *f = (struct ch_filter *)data;
  2635. struct filter_info *p;
  2636. unsigned int nfilters = sc->params.mc5.nfilters;
  2637. if (!is_offload(sc))
  2638. return (EOPNOTSUPP);
  2639. if (!(sc->flags & FULL_INIT_DONE))
  2640. return (EAGAIN);
  2641. if (nfilters == 0 || sc->filters == NULL)
  2642. return (EINVAL);
  2643. if (f->filter_id >= nfilters)
  2644. return (EINVAL);
  2645. p = &sc->filters[f->filter_id];
  2646. if (p->locked)
  2647. return (EPERM);
  2648. if (!p->valid)
  2649. return (EFAULT); /* Read "Bad address" as "Bad index" */
  2650. bzero(p, sizeof(*p));
  2651. p->sip = p->sip_mask = 0xffffffff;
  2652. p->vlan = 0xfff;
  2653. p->vlan_prio = FILTER_NO_VLAN_PRI;
  2654. p->pkt_type = 1;
  2655. error = set_filter(sc, f->filter_id, p);
  2656. break;
  2657. }
  2658. case CHELSIO_GET_FILTER: {
  2659. struct ch_filter *f = (struct ch_filter *)data;
  2660. struct filter_info *p;
  2661. unsigned int i, nfilters = sc->params.mc5.nfilters;
  2662. if (!is_offload(sc))
  2663. return (EOPNOTSUPP);
  2664. if (!(sc->flags & FULL_INIT_DONE))
  2665. return (EAGAIN);
  2666. if (nfilters == 0 || sc->filters == NULL)
  2667. return (EINVAL);
  2668. i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
  2669. for (; i < nfilters; i++) {
  2670. p = &sc->filters[i];
  2671. if (!p->valid)
  2672. continue;
  2673. bzero(f, sizeof(*f));
  2674. f->filter_id = i;
  2675. f->val.sip = p->sip;
  2676. f->mask.sip = p->sip_mask;
  2677. f->val.dip = p->dip;
  2678. f->mask.dip = p->dip ? 0xffffffff : 0;
  2679. f->val.sport = p->sport;
  2680. f->mask.sport = p->sport ? 0xffff : 0;
  2681. f->val.dport = p->dport;
  2682. f->mask.dport = p->dport ? 0xffff : 0;
  2683. f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
  2684. f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
  2685. f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
  2686. 0 : p->vlan_prio;
  2687. f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
  2688. 0 : FILTER_NO_VLAN_PRI;
  2689. f->mac_hit = p->mac_hit;
  2690. f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
  2691. f->proto = p->pkt_type;
  2692. f->want_filter_id = p->report_filter_id;
  2693. f->pass = p->pass;
  2694. f->rss = p->rss;
  2695. f->qset = p->qset;
  2696. break;
  2697. }
  2698. if (i == nfilters)
  2699. f->filter_id = 0xffffffff;
  2700. break;
  2701. }
  2702. default:
  2703. return (EOPNOTSUPP);
  2704. break;
  2705. }
  2706. return (error);
  2707. }
  2708. static __inline void
  2709. reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
  2710. unsigned int end)
  2711. {
  2712. uint32_t *p = (uint32_t *)(buf + start);
  2713. for ( ; start <= end; start += sizeof(uint32_t))
  2714. *p++ = t3_read_reg(ap, start);
  2715. }
  2716. #define T3_REGMAP_SIZE (3 * 1024)
  2717. static int
  2718. cxgb_get_regs_len(void)
  2719. {
  2720. return T3_REGMAP_SIZE;
  2721. }
  2722. static void
  2723. cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
  2724. {
  2725. /*
  2726. * Version scheme:
  2727. * bits 0..9: chip version
  2728. * bits 10..15: chip revision
  2729. * bit 31: set for PCIe cards
  2730. */
  2731. regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
  2732. /*
  2733. * We skip the MAC statistics registers because they are clear-on-read.
  2734. * Also reading multi-register stats would need to synchronize with the
  2735. * periodic mac stats accumulation. Hard to justify the complexity.
  2736. */
  2737. memset(buf, 0, cxgb_get_regs_len());
  2738. reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
  2739. reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
  2740. reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
  2741. reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
  2742. reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
  2743. reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
  2744. XGM_REG(A_XGM_SERDES_STAT3, 1));
  2745. reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
  2746. XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
  2747. }
  2748. static int
  2749. alloc_filters(struct adapter *sc)
  2750. {
  2751. struct filter_info *p;
  2752. unsigned int nfilters = sc->params.mc5.nfilters;
  2753. if (nfilters == 0)
  2754. return (0);
  2755. p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
  2756. sc->filters = p;
  2757. p = &sc->filters[nfilters - 1];
  2758. p->vlan = 0xfff;
  2759. p->vlan_prio = FILTER_NO_VLAN_PRI;
  2760. p->pass = p->rss = p->valid = p->locked = 1;
  2761. return (0);
  2762. }
  2763. static int
  2764. setup_hw_filters(struct adapter *sc)
  2765. {
  2766. int i, rc;
  2767. unsigned int nfilters = sc->params.mc5.nfilters;
  2768. if (!sc->filters)
  2769. return (0);
  2770. t3_enable_filters(sc);
  2771. for (i = rc = 0; i < nfilters && !rc; i++) {
  2772. if (sc->filters[i].locked)
  2773. rc = set_filter(sc, i, &sc->filters[i]);
  2774. }
  2775. return (rc);
  2776. }
  2777. static int
  2778. set_filter(struct adapter *sc, int id, const struct filter_info *f)
  2779. {
  2780. int len;
  2781. struct mbuf *m;
  2782. struct ulp_txpkt *txpkt;
  2783. struct work_request_hdr *wr;
  2784. struct cpl_pass_open_req *oreq;
  2785. struct cpl_set_tcb_field *sreq;
  2786. len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
  2787. KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
  2788. id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
  2789. sc->params.mc5.nfilters;
  2790. m = m_gethdr(M_WAITOK, MT_DATA);
  2791. m->m_len = m->m_pkthdr.len = len;
  2792. bzero(mtod(m, char *), len);
  2793. wr = mtod(m, struct work_request_hdr *);
  2794. wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
  2795. oreq = (struct cpl_pass_open_req *)(wr + 1);
  2796. txpkt = (struct ulp_txpkt *)oreq;
  2797. txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
  2798. txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
  2799. OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
  2800. oreq->local_port = htons(f->dport);
  2801. oreq->peer_port = htons(f->sport);
  2802. oreq->local_ip = htonl(f->dip);
  2803. oreq->peer_ip = htonl(f->sip);
  2804. oreq->peer_netmask = htonl(f->sip_mask);
  2805. oreq->opt0h = 0;
  2806. oreq->opt0l = htonl(F_NO_OFFLOAD);
  2807. oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
  2808. V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
  2809. V_VLAN_PRI(f->vlan_prio >> 1) |
  2810. V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
  2811. V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
  2812. V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
  2813. sreq = (struct cpl_set_tcb_field *)(oreq + 1);
  2814. set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
  2815. (f->report_filter_id << 15) | (1 << 23) |
  2816. ((u64)f->pass << 35) | ((u64)!f->rss << 36));
  2817. set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
  2818. t3_mgmt_tx(sc, m);
  2819. if (f->pass && !f->rss) {
  2820. len = sizeof(*sreq);
  2821. m = m_gethdr(M_WAITOK, MT_DATA);
  2822. m->m_len = m->m_pkthdr.len = len;
  2823. bzero(mtod(m, char *), len);
  2824. sreq = mtod(m, struct cpl_set_tcb_field *);
  2825. sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
  2826. mk_set_tcb_field(sreq, id, 25, 0x3f80000,
  2827. (u64)sc->rrss_map[f->qset] << 19);
  2828. t3_mgmt_tx(sc, m);
  2829. }
  2830. return 0;
  2831. }
  2832. static inline void
  2833. mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
  2834. unsigned int word, u64 mask, u64 val)
  2835. {
  2836. OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
  2837. req->reply = V_NO_REPLY(1);
  2838. req->cpu_idx = 0;
  2839. req->word = htons(word);
  2840. req->mask = htobe64(mask);
  2841. req->val = htobe64(val);
  2842. }
  2843. static inline void
  2844. set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
  2845. unsigned int word, u64 mask, u64 val)
  2846. {
  2847. struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
  2848. txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
  2849. txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
  2850. mk_set_tcb_field(req, tid, word, mask, val);
  2851. }
  2852. void
  2853. t3_iterate(void (*func)(struct adapter *, void *), void *arg)
  2854. {
  2855. struct adapter *sc;
  2856. mtx_lock(&t3_list_lock);
  2857. SLIST_FOREACH(sc, &t3_list, link) {
  2858. /*
  2859. * func should not make any assumptions about what state sc is
  2860. * in - the only guarantee is that sc->sc_lock is a valid lock.
  2861. */
  2862. func(sc, arg);
  2863. }
  2864. mtx_unlock(&t3_list_lock);
  2865. }
  2866. #ifdef TCP_OFFLOAD
  2867. static int
  2868. toe_capability(struct port_info *pi, int enable)
  2869. {
  2870. int rc;
  2871. struct adapter *sc = pi->adapter;
  2872. ADAPTER_LOCK_ASSERT_OWNED(sc);
  2873. if (!is_offload(sc))
  2874. return (ENODEV);
  2875. if (enable) {
  2876. if (!(sc->flags & FULL_INIT_DONE)) {
  2877. log(LOG_WARNING,
  2878. "You must enable a cxgb interface first\n");
  2879. return (EAGAIN);
  2880. }
  2881. if (isset(&sc->offload_map, pi->port_id))
  2882. return (0);
  2883. if (!(sc->flags & TOM_INIT_DONE)) {
  2884. rc = t3_activate_uld(sc, ULD_TOM);
  2885. if (rc == EAGAIN) {
  2886. log(LOG_WARNING,
  2887. "You must kldload t3_tom.ko before trying "
  2888. "to enable TOE on a cxgb interface.\n");
  2889. }
  2890. if (rc != 0)
  2891. return (rc);
  2892. KASSERT(sc->tom_softc != NULL,
  2893. ("%s: TOM activated but softc NULL", __func__));
  2894. KASSERT(sc->flags & TOM_INIT_DONE,
  2895. ("%s: TOM activated but flag not set", __func__));
  2896. }
  2897. setbit(&sc->offload_map, pi->port_id);
  2898. /*
  2899. * XXX: Temporary code to allow iWARP to be enabled when TOE is
  2900. * enabled on any port. Need to figure out how to enable,
  2901. * disable, load, and unload iWARP cleanly.
  2902. */
  2903. if (!isset(&sc->offload_map, MAX_NPORTS) &&
  2904. t3_activate_uld(sc, ULD_IWARP) == 0)
  2905. setbit(&sc->offload_map, MAX_NPORTS);
  2906. } else {
  2907. if (!isset(&sc->offload_map, pi->port_id))
  2908. return (0);
  2909. KASSERT(sc->flags & TOM_INIT_DONE,
  2910. ("%s: TOM never initialized?", __func__));
  2911. clrbit(&sc->offload_map, pi->port_id);
  2912. }
  2913. return (0);
  2914. }
  2915. /*
  2916. * Add an upper layer driver to the global list.
  2917. */
  2918. int
  2919. t3_register_uld(struct uld_info *ui)
  2920. {
  2921. int rc = 0;
  2922. struct uld_info *u;
  2923. mtx_lock(&t3_uld_list_lock);
  2924. SLIST_FOREACH(u, &t3_uld_list, link) {
  2925. if (u->uld_id == ui->uld_id) {
  2926. rc = EEXIST;
  2927. goto done;
  2928. }
  2929. }
  2930. SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
  2931. ui->refcount = 0;
  2932. done:
  2933. mtx_unlock(&t3_uld_list_lock);
  2934. return (rc);
  2935. }
  2936. int
  2937. t3_unregister_uld(struct uld_info *ui)
  2938. {
  2939. int rc = EINVAL;
  2940. struct uld_info *u;
  2941. mtx_lock(&t3_uld_list_lock);
  2942. SLIST_FOREACH(u, &t3_uld_list, link) {
  2943. if (u == ui) {
  2944. if (ui->refcount > 0) {
  2945. rc = EBUSY;
  2946. goto done;
  2947. }
  2948. SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
  2949. rc = 0;
  2950. goto done;
  2951. }
  2952. }
  2953. done:
  2954. mtx_unlock(&t3_uld_list_lock);
  2955. return (rc);
  2956. }
  2957. int
  2958. t3_activate_uld(struct adapter *sc, int id)
  2959. {
  2960. int rc = EAGAIN;
  2961. struct uld_info *ui;
  2962. mtx_lock(&t3_uld_list_lock);
  2963. SLIST_FOREACH(ui, &t3_uld_list, link) {
  2964. if (ui->uld_id == id) {
  2965. rc = ui->activate(sc);
  2966. if (rc == 0)
  2967. ui->refcount++;
  2968. goto done;
  2969. }
  2970. }
  2971. done:
  2972. mtx_unlock(&t3_uld_list_lock);
  2973. return (rc);
  2974. }
  2975. int
  2976. t3_deactivate_uld(struct adapter *sc, int id)
  2977. {
  2978. int rc = EINVAL;
  2979. struct uld_info *ui;
  2980. mtx_lock(&t3_uld_list_lock);
  2981. SLIST_FOREACH(ui, &t3_uld_list, link) {
  2982. if (ui->uld_id == id) {
  2983. rc = ui->deactivate(sc);
  2984. if (rc == 0)
  2985. ui->refcount--;
  2986. goto done;
  2987. }
  2988. }
  2989. done:
  2990. mtx_unlock(&t3_uld_list_lock);
  2991. return (rc);
  2992. }
  2993. static int
  2994. cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
  2995. struct mbuf *m)
  2996. {
  2997. m_freem(m);
  2998. return (EDOOFUS);
  2999. }
  3000. int
  3001. t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
  3002. {
  3003. uintptr_t *loc, new;
  3004. if (opcode >= NUM_CPL_HANDLERS)
  3005. return (EINVAL);
  3006. new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
  3007. loc = (uintptr_t *) &sc->cpl_handler[opcode];
  3008. atomic_store_rel_ptr(loc, new);
  3009. return (0);
  3010. }
  3011. #endif
  3012. static int
  3013. cxgbc_mod_event(module_t mod, int cmd, void *arg)
  3014. {
  3015. int rc = 0;
  3016. switch (cmd) {
  3017. case MOD_LOAD:
  3018. mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
  3019. SLIST_INIT(&t3_list);
  3020. #ifdef TCP_OFFLOAD
  3021. mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
  3022. SLIST_INIT(&t3_uld_list);
  3023. #endif
  3024. break;
  3025. case MOD_UNLOAD:
  3026. #ifdef TCP_OFFLOAD
  3027. mtx_lock(&t3_uld_list_lock);
  3028. if (!SLIST_EMPTY(&t3_uld_list)) {
  3029. rc = EBUSY;
  3030. mtx_unlock(&t3_uld_list_lock);
  3031. break;
  3032. }
  3033. mtx_unlock(&t3_uld_list_lock);
  3034. mtx_destroy(&t3_uld_list_lock);
  3035. #endif
  3036. mtx_lock(&t3_list_lock);
  3037. if (!SLIST_EMPTY(&t3_list)) {
  3038. rc = EBUSY;
  3039. mtx_unlock(&t3_list_lock);
  3040. break;
  3041. }
  3042. mtx_unlock(&t3_list_lock);
  3043. mtx_destroy(&t3_list_lock);
  3044. break;
  3045. }
  3046. return (rc);
  3047. }
  3048. #ifdef NETDUMP
  3049. static void
  3050. cxgb_netdump_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize)
  3051. {
  3052. struct port_info *pi;
  3053. adapter_t *adap;
  3054. pi = if_getsoftc(ifp);
  3055. adap = pi->adapter;
  3056. ADAPTER_LOCK(adap);
  3057. *nrxr = SGE_QSETS;
  3058. *ncl = adap->sge.qs[0].fl[1].size;
  3059. *clsize = adap->sge.qs[0].fl[1].buf_size;
  3060. ADAPTER_UNLOCK(adap);
  3061. }
  3062. static void
  3063. cxgb_netdump_event(struct ifnet *ifp, enum netdump_ev event)
  3064. {
  3065. struct port_info *pi;
  3066. struct sge_qset *qs;
  3067. int i;
  3068. pi = if_getsoftc(ifp);
  3069. if (event == NETDUMP_START)
  3070. for (i = 0; i < SGE_QSETS; i++) {
  3071. qs = &pi->adapter->sge.qs[i];
  3072. /* Need to reinit after netdump_mbuf_dump(). */
  3073. qs->fl[0].zone = zone_pack;
  3074. qs->fl[1].zone = zone_clust;
  3075. qs->lro.enabled = 0;
  3076. }
  3077. }
  3078. static int
  3079. cxgb_netdump_transmit(struct ifnet *ifp, struct mbuf *m)
  3080. {
  3081. struct port_info *pi;
  3082. struct sge_qset *qs;
  3083. pi = if_getsoftc(ifp);
  3084. if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
  3085. IFF_DRV_RUNNING)
  3086. return (ENOENT);
  3087. qs = &pi->adapter->sge.qs[pi->first_qset];
  3088. return (cxgb_netdump_encap(qs, &m));
  3089. }
  3090. static int
  3091. cxgb_netdump_poll(struct ifnet *ifp, int count)
  3092. {
  3093. struct port_info *pi;
  3094. adapter_t *adap;
  3095. int i;
  3096. pi = if_getsoftc(ifp);
  3097. if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
  3098. return (ENOENT);
  3099. adap = pi->adapter;
  3100. for (i = 0; i < SGE_QSETS; i++)
  3101. (void)cxgb_netdump_poll_rx(adap, &adap->sge.qs[i]);
  3102. (void)cxgb_netdump_poll_tx(&adap->sge.qs[pi->first_qset]);
  3103. return (0);
  3104. }
  3105. #endif /* NETDUMP */