HardenedBSD src tree https://hardenedbsd.org/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

3048 lines
74 KiB

  1. /*-
  2. * SPDX-License-Identifier: BSD-3-Clause
  3. *
  4. * Copyright (C) 2013 Emulex
  5. * All rights reserved.
  6. *
  7. * Redistribution and use in source and binary forms, with or without
  8. * modification, are permitted provided that the following conditions are met:
  9. *
  10. * 1. Redistributions of source code must retain the above copyright notice,
  11. * this list of conditions and the following disclaimer.
  12. *
  13. * 2. Redistributions in binary form must reproduce the above copyright
  14. * notice, this list of conditions and the following disclaimer in the
  15. * documentation and/or other materials provided with the distribution.
  16. *
  17. * 3. Neither the name of the Emulex Corporation nor the names of its
  18. * contributors may be used to endorse or promote products derived from
  19. * this software without specific prior written permission.
  20. *
  21. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  22. * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23. * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  25. * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26. * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27. * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28. * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29. * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  31. * POSSIBILITY OF SUCH DAMAGE.
  32. *
  33. * Contact Information:
  34. * freebsd-drivers@emulex.com
  35. *
  36. * Emulex
  37. * 3333 Susan Street
  38. * Costa Mesa, CA 92626
  39. */
  40. /* $FreeBSD$ */
  41. #include "opt_inet6.h"
  42. #include "opt_inet.h"
  43. #include "oce_if.h"
  44. #include "oce_user.h"
  45. #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO)
  46. /* UE Status Low CSR */
  47. static char *ue_status_low_desc[] = {
  48. "CEV",
  49. "CTX",
  50. "DBUF",
  51. "ERX",
  52. "Host",
  53. "MPU",
  54. "NDMA",
  55. "PTC ",
  56. "RDMA ",
  57. "RXF ",
  58. "RXIPS ",
  59. "RXULP0 ",
  60. "RXULP1 ",
  61. "RXULP2 ",
  62. "TIM ",
  63. "TPOST ",
  64. "TPRE ",
  65. "TXIPS ",
  66. "TXULP0 ",
  67. "TXULP1 ",
  68. "UC ",
  69. "WDMA ",
  70. "TXULP2 ",
  71. "HOST1 ",
  72. "P0_OB_LINK ",
  73. "P1_OB_LINK ",
  74. "HOST_GPIO ",
  75. "MBOX ",
  76. "AXGMAC0",
  77. "AXGMAC1",
  78. "JTAG",
  79. "MPU_INTPEND"
  80. };
  81. /* UE Status High CSR */
  82. static char *ue_status_hi_desc[] = {
  83. "LPCMEMHOST",
  84. "MGMT_MAC",
  85. "PCS0ONLINE",
  86. "MPU_IRAM",
  87. "PCS1ONLINE",
  88. "PCTL0",
  89. "PCTL1",
  90. "PMEM",
  91. "RR",
  92. "TXPB",
  93. "RXPP",
  94. "XAUI",
  95. "TXP",
  96. "ARM",
  97. "IPC",
  98. "HOST2",
  99. "HOST3",
  100. "HOST4",
  101. "HOST5",
  102. "HOST6",
  103. "HOST7",
  104. "HOST8",
  105. "HOST9",
  106. "NETC",
  107. "Unknown",
  108. "Unknown",
  109. "Unknown",
  110. "Unknown",
  111. "Unknown",
  112. "Unknown",
  113. "Unknown",
  114. "Unknown"
  115. };
  116. struct oce_common_cqe_info{
  117. uint8_t vtp:1;
  118. uint8_t l4_cksum_pass:1;
  119. uint8_t ip_cksum_pass:1;
  120. uint8_t ipv6_frame:1;
  121. uint8_t qnq:1;
  122. uint8_t rsvd:3;
  123. uint8_t num_frags;
  124. uint16_t pkt_size;
  125. uint16_t vtag;
  126. };
  127. /* Driver entry points prototypes */
  128. static int oce_probe(device_t dev);
  129. static int oce_attach(device_t dev);
  130. static int oce_detach(device_t dev);
  131. static int oce_shutdown(device_t dev);
  132. static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data);
  133. static void oce_init(void *xsc);
  134. static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m);
  135. static void oce_multiq_flush(struct ifnet *ifp);
  136. /* Driver interrupt routines protypes */
  137. static void oce_intr(void *arg, int pending);
  138. static int oce_setup_intr(POCE_SOFTC sc);
  139. static int oce_fast_isr(void *arg);
  140. static int oce_alloc_intr(POCE_SOFTC sc, int vector,
  141. void (*isr) (void *arg, int pending));
  142. /* Media callbacks prototypes */
  143. static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req);
  144. static int oce_media_change(struct ifnet *ifp);
  145. /* Transmit routines prototypes */
  146. static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
  147. static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
  148. static void oce_process_tx_completion(struct oce_wq *wq);
  149. static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m,
  150. struct oce_wq *wq);
  151. /* Receive routines prototypes */
  152. static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
  153. static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
  154. static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
  155. static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq);
  156. static uint16_t oce_rq_handler_lro(void *arg);
  157. static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2);
  158. static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2);
  159. static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m);
  160. /* Helper function prototypes in this file */
  161. static int oce_attach_ifp(POCE_SOFTC sc);
  162. static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
  163. static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag);
  164. static int oce_vid_config(POCE_SOFTC sc);
  165. static void oce_mac_addr_set(POCE_SOFTC sc);
  166. static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data);
  167. static void oce_local_timer(void *arg);
  168. static void oce_if_deactivate(POCE_SOFTC sc);
  169. static void oce_if_activate(POCE_SOFTC sc);
  170. static void setup_max_queues_want(POCE_SOFTC sc);
  171. static void update_queues_got(POCE_SOFTC sc);
  172. static void process_link_state(POCE_SOFTC sc,
  173. struct oce_async_cqe_link_state *acqe);
  174. static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m);
  175. static void oce_get_config(POCE_SOFTC sc);
  176. static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete);
  177. static void oce_read_env_variables(POCE_SOFTC sc);
  178. /* IP specific */
  179. #if defined(INET6) || defined(INET)
  180. static int oce_init_lro(POCE_SOFTC sc);
  181. static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
  182. #endif
  183. static device_method_t oce_dispatch[] = {
  184. DEVMETHOD(device_probe, oce_probe),
  185. DEVMETHOD(device_attach, oce_attach),
  186. DEVMETHOD(device_detach, oce_detach),
  187. DEVMETHOD(device_shutdown, oce_shutdown),
  188. DEVMETHOD_END
  189. };
  190. static driver_t oce_driver = {
  191. "oce",
  192. oce_dispatch,
  193. sizeof(OCE_SOFTC)
  194. };
  195. static devclass_t oce_devclass;
  196. /* global vars */
  197. const char component_revision[32] = {"///" COMPONENT_REVISION "///"};
  198. /* Module capabilites and parameters */
  199. uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED;
  200. uint32_t oce_enable_rss = OCE_MODCAP_RSS;
  201. uint32_t oce_rq_buf_size = 2048;
  202. TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled);
  203. TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss);
  204. /* Supported devices table */
  205. static uint32_t supportedDevices[] = {
  206. (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2,
  207. (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3,
  208. (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3,
  209. (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201,
  210. (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF,
  211. (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH
  212. };
  213. DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0);
  214. MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices,
  215. nitems(supportedDevices));
  216. MODULE_DEPEND(oce, pci, 1, 1, 1);
  217. MODULE_DEPEND(oce, ether, 1, 1, 1);
  218. MODULE_VERSION(oce, 1);
  219. POCE_SOFTC softc_head = NULL;
  220. POCE_SOFTC softc_tail = NULL;
  221. struct oce_rdma_if *oce_rdma_if = NULL;
  222. /*****************************************************************************
  223. * Driver entry points functions *
  224. *****************************************************************************/
  225. static int
  226. oce_probe(device_t dev)
  227. {
  228. uint16_t vendor = 0;
  229. uint16_t device = 0;
  230. int i = 0;
  231. char str[256] = {0};
  232. POCE_SOFTC sc;
  233. sc = device_get_softc(dev);
  234. bzero(sc, sizeof(OCE_SOFTC));
  235. sc->dev = dev;
  236. vendor = pci_get_vendor(dev);
  237. device = pci_get_device(dev);
  238. for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
  239. if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
  240. if (device == (supportedDevices[i] & 0xffff)) {
  241. sprintf(str, "%s:%s", "Emulex CNA NIC function",
  242. component_revision);
  243. device_set_desc_copy(dev, str);
  244. switch (device) {
  245. case PCI_PRODUCT_BE2:
  246. sc->flags |= OCE_FLAGS_BE2;
  247. break;
  248. case PCI_PRODUCT_BE3:
  249. sc->flags |= OCE_FLAGS_BE3;
  250. break;
  251. case PCI_PRODUCT_XE201:
  252. case PCI_PRODUCT_XE201_VF:
  253. sc->flags |= OCE_FLAGS_XE201;
  254. break;
  255. case PCI_PRODUCT_SH:
  256. sc->flags |= OCE_FLAGS_SH;
  257. break;
  258. default:
  259. return ENXIO;
  260. }
  261. return BUS_PROBE_DEFAULT;
  262. }
  263. }
  264. }
  265. return ENXIO;
  266. }
  267. static int
  268. oce_attach(device_t dev)
  269. {
  270. POCE_SOFTC sc;
  271. int rc = 0;
  272. sc = device_get_softc(dev);
  273. rc = oce_hw_pci_alloc(sc);
  274. if (rc)
  275. return rc;
  276. sc->tx_ring_size = OCE_TX_RING_SIZE;
  277. sc->rx_ring_size = OCE_RX_RING_SIZE;
  278. /* receive fragment size should be multiple of 2K */
  279. sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048);
  280. sc->flow_control = OCE_DEFAULT_FLOW_CONTROL;
  281. sc->promisc = OCE_DEFAULT_PROMISCUOUS;
  282. LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock");
  283. LOCK_CREATE(&sc->dev_lock, "Device_lock");
  284. /* initialise the hardware */
  285. rc = oce_hw_init(sc);
  286. if (rc)
  287. goto pci_res_free;
  288. oce_read_env_variables(sc);
  289. oce_get_config(sc);
  290. setup_max_queues_want(sc);
  291. rc = oce_setup_intr(sc);
  292. if (rc)
  293. goto mbox_free;
  294. rc = oce_queue_init_all(sc);
  295. if (rc)
  296. goto intr_free;
  297. rc = oce_attach_ifp(sc);
  298. if (rc)
  299. goto queues_free;
  300. #if defined(INET6) || defined(INET)
  301. rc = oce_init_lro(sc);
  302. if (rc)
  303. goto ifp_free;
  304. #endif
  305. rc = oce_hw_start(sc);
  306. if (rc)
  307. goto lro_free;
  308. sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
  309. oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
  310. sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
  311. oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST);
  312. rc = oce_stats_init(sc);
  313. if (rc)
  314. goto vlan_free;
  315. oce_add_sysctls(sc);
  316. callout_init(&sc->timer, CALLOUT_MPSAFE);
  317. rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
  318. if (rc)
  319. goto stats_free;
  320. sc->next =NULL;
  321. if (softc_tail != NULL) {
  322. softc_tail->next = sc;
  323. } else {
  324. softc_head = sc;
  325. }
  326. softc_tail = sc;
  327. return 0;
  328. stats_free:
  329. callout_drain(&sc->timer);
  330. oce_stats_free(sc);
  331. vlan_free:
  332. if (sc->vlan_attach)
  333. EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
  334. if (sc->vlan_detach)
  335. EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
  336. oce_hw_intr_disable(sc);
  337. lro_free:
  338. #if defined(INET6) || defined(INET)
  339. oce_free_lro(sc);
  340. ifp_free:
  341. #endif
  342. ether_ifdetach(sc->ifp);
  343. if_free(sc->ifp);
  344. queues_free:
  345. oce_queue_release_all(sc);
  346. intr_free:
  347. oce_intr_free(sc);
  348. mbox_free:
  349. oce_dma_free(sc, &sc->bsmbx);
  350. pci_res_free:
  351. oce_hw_pci_free(sc);
  352. LOCK_DESTROY(&sc->dev_lock);
  353. LOCK_DESTROY(&sc->bmbx_lock);
  354. return rc;
  355. }
  356. static int
  357. oce_detach(device_t dev)
  358. {
  359. POCE_SOFTC sc = device_get_softc(dev);
  360. POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL;
  361. poce_sc_tmp = softc_head;
  362. ppoce_sc_tmp1 = &softc_head;
  363. while (poce_sc_tmp != NULL) {
  364. if (poce_sc_tmp == sc) {
  365. *ppoce_sc_tmp1 = sc->next;
  366. if (sc->next == NULL) {
  367. softc_tail = poce_sc_tmp2;
  368. }
  369. break;
  370. }
  371. poce_sc_tmp2 = poce_sc_tmp;
  372. ppoce_sc_tmp1 = &poce_sc_tmp->next;
  373. poce_sc_tmp = poce_sc_tmp->next;
  374. }
  375. LOCK(&sc->dev_lock);
  376. oce_if_deactivate(sc);
  377. UNLOCK(&sc->dev_lock);
  378. callout_drain(&sc->timer);
  379. if (sc->vlan_attach != NULL)
  380. EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
  381. if (sc->vlan_detach != NULL)
  382. EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
  383. ether_ifdetach(sc->ifp);
  384. if_free(sc->ifp);
  385. oce_hw_shutdown(sc);
  386. bus_generic_detach(dev);
  387. return 0;
  388. }
  389. static int
  390. oce_shutdown(device_t dev)
  391. {
  392. int rc;
  393. rc = oce_detach(dev);
  394. return rc;
  395. }
  396. static int
  397. oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
  398. {
  399. struct ifreq *ifr = (struct ifreq *)data;
  400. POCE_SOFTC sc = ifp->if_softc;
  401. struct ifi2creq i2c;
  402. uint8_t offset = 0;
  403. int rc = 0;
  404. uint32_t u;
  405. switch (command) {
  406. case SIOCGIFMEDIA:
  407. rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
  408. break;
  409. case SIOCSIFMTU:
  410. if (ifr->ifr_mtu > OCE_MAX_MTU)
  411. rc = EINVAL;
  412. else
  413. ifp->if_mtu = ifr->ifr_mtu;
  414. break;
  415. case SIOCSIFFLAGS:
  416. if (ifp->if_flags & IFF_UP) {
  417. if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
  418. sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
  419. oce_init(sc);
  420. }
  421. device_printf(sc->dev, "Interface Up\n");
  422. } else {
  423. LOCK(&sc->dev_lock);
  424. sc->ifp->if_drv_flags &=
  425. ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
  426. oce_if_deactivate(sc);
  427. UNLOCK(&sc->dev_lock);
  428. device_printf(sc->dev, "Interface Down\n");
  429. }
  430. if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) {
  431. if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1))))
  432. sc->promisc = TRUE;
  433. } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) {
  434. if (!oce_rxf_set_promiscuous(sc, 0))
  435. sc->promisc = FALSE;
  436. }
  437. break;
  438. case SIOCADDMULTI:
  439. case SIOCDELMULTI:
  440. rc = oce_hw_update_multicast(sc);
  441. if (rc)
  442. device_printf(sc->dev,
  443. "Update multicast address failed\n");
  444. break;
  445. case SIOCSIFCAP:
  446. u = ifr->ifr_reqcap ^ ifp->if_capenable;
  447. if (u & IFCAP_TXCSUM) {
  448. ifp->if_capenable ^= IFCAP_TXCSUM;
  449. ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
  450. if (IFCAP_TSO & ifp->if_capenable &&
  451. !(IFCAP_TXCSUM & ifp->if_capenable)) {
  452. u &= ~IFCAP_TSO;
  453. ifp->if_capenable &= ~IFCAP_TSO;
  454. ifp->if_hwassist &= ~CSUM_TSO;
  455. if_printf(ifp,
  456. "TSO disabled due to -txcsum.\n");
  457. }
  458. }
  459. if (u & IFCAP_RXCSUM)
  460. ifp->if_capenable ^= IFCAP_RXCSUM;
  461. if (u & IFCAP_TSO4) {
  462. ifp->if_capenable ^= IFCAP_TSO4;
  463. if (IFCAP_TSO & ifp->if_capenable) {
  464. if (IFCAP_TXCSUM & ifp->if_capenable)
  465. ifp->if_hwassist |= CSUM_TSO;
  466. else {
  467. ifp->if_capenable &= ~IFCAP_TSO;
  468. ifp->if_hwassist &= ~CSUM_TSO;
  469. if_printf(ifp,
  470. "Enable txcsum first.\n");
  471. rc = EAGAIN;
  472. }
  473. } else
  474. ifp->if_hwassist &= ~CSUM_TSO;
  475. }
  476. if (u & IFCAP_VLAN_HWTAGGING)
  477. ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
  478. if (u & IFCAP_VLAN_HWFILTER) {
  479. ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
  480. oce_vid_config(sc);
  481. }
  482. #if defined(INET6) || defined(INET)
  483. if (u & IFCAP_LRO) {
  484. ifp->if_capenable ^= IFCAP_LRO;
  485. if(sc->enable_hwlro) {
  486. if(ifp->if_capenable & IFCAP_LRO) {
  487. rc = oce_mbox_nic_set_iface_lro_config(sc, 1);
  488. }else {
  489. rc = oce_mbox_nic_set_iface_lro_config(sc, 0);
  490. }
  491. }
  492. }
  493. #endif
  494. break;
  495. case SIOCGI2C:
  496. rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
  497. if (rc)
  498. break;
  499. if (i2c.dev_addr == PAGE_NUM_A0) {
  500. offset = i2c.offset;
  501. } else if (i2c.dev_addr == PAGE_NUM_A2) {
  502. offset = TRANSCEIVER_A0_SIZE + i2c.offset;
  503. } else {
  504. rc = EINVAL;
  505. break;
  506. }
  507. if (i2c.len > sizeof(i2c.data) ||
  508. i2c.len + offset > sizeof(sfp_vpd_dump_buffer)) {
  509. rc = EINVAL;
  510. break;
  511. }
  512. rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr);
  513. if (rc) {
  514. rc = -rc;
  515. break;
  516. }
  517. memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len);
  518. rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
  519. break;
  520. case SIOCGPRIVATE_0:
  521. rc = priv_check(curthread, PRIV_DRIVER);
  522. if (rc != 0)
  523. break;
  524. rc = oce_handle_passthrough(ifp, data);
  525. break;
  526. default:
  527. rc = ether_ioctl(ifp, command, data);
  528. break;
  529. }
  530. return rc;
  531. }
  532. static void
  533. oce_init(void *arg)
  534. {
  535. POCE_SOFTC sc = arg;
  536. LOCK(&sc->dev_lock);
  537. if (sc->ifp->if_flags & IFF_UP) {
  538. oce_if_deactivate(sc);
  539. oce_if_activate(sc);
  540. }
  541. UNLOCK(&sc->dev_lock);
  542. }
  543. static int
  544. oce_multiq_start(struct ifnet *ifp, struct mbuf *m)
  545. {
  546. POCE_SOFTC sc = ifp->if_softc;
  547. struct oce_wq *wq = NULL;
  548. int queue_index = 0;
  549. int status = 0;
  550. if (!sc->link_status)
  551. return ENXIO;
  552. if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
  553. queue_index = m->m_pkthdr.flowid % sc->nwqs;
  554. wq = sc->wq[queue_index];
  555. LOCK(&wq->tx_lock);
  556. status = oce_multiq_transmit(ifp, m, wq);
  557. UNLOCK(&wq->tx_lock);
  558. return status;
  559. }
  560. static void
  561. oce_multiq_flush(struct ifnet *ifp)
  562. {
  563. POCE_SOFTC sc = ifp->if_softc;
  564. struct mbuf *m;
  565. int i = 0;
  566. for (i = 0; i < sc->nwqs; i++) {
  567. while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)
  568. m_freem(m);
  569. }
  570. if_qflush(ifp);
  571. }
  572. /*****************************************************************************
  573. * Driver interrupt routines functions *
  574. *****************************************************************************/
  575. static void
  576. oce_intr(void *arg, int pending)
  577. {
  578. POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
  579. POCE_SOFTC sc = ii->sc;
  580. struct oce_eq *eq = ii->eq;
  581. struct oce_eqe *eqe;
  582. struct oce_cq *cq = NULL;
  583. int i, num_eqes = 0;
  584. bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
  585. BUS_DMASYNC_POSTWRITE);
  586. do {
  587. eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
  588. if (eqe->evnt == 0)
  589. break;
  590. eqe->evnt = 0;
  591. bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
  592. BUS_DMASYNC_POSTWRITE);
  593. RING_GET(eq->ring, 1);
  594. num_eqes++;
  595. } while (TRUE);
  596. if (!num_eqes)
  597. goto eq_arm; /* Spurious */
  598. /* Clear EQ entries, but dont arm */
  599. oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE);
  600. /* Process TX, RX and MCC. But dont arm CQ*/
  601. for (i = 0; i < eq->cq_valid; i++) {
  602. cq = eq->cq[i];
  603. (*cq->cq_handler)(cq->cb_arg);
  604. }
  605. /* Arm all cqs connected to this EQ */
  606. for (i = 0; i < eq->cq_valid; i++) {
  607. cq = eq->cq[i];
  608. oce_arm_cq(sc, cq->cq_id, 0, TRUE);
  609. }
  610. eq_arm:
  611. oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
  612. return;
  613. }
  614. static int
  615. oce_setup_intr(POCE_SOFTC sc)
  616. {
  617. int rc = 0, use_intx = 0;
  618. int vector = 0, req_vectors = 0;
  619. int tot_req_vectors, tot_vectors;
  620. if (is_rss_enabled(sc))
  621. req_vectors = MAX((sc->nrqs - 1), sc->nwqs);
  622. else
  623. req_vectors = 1;
  624. tot_req_vectors = req_vectors;
  625. if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
  626. if (req_vectors > 1) {
  627. tot_req_vectors += OCE_RDMA_VECTORS;
  628. sc->roce_intr_count = OCE_RDMA_VECTORS;
  629. }
  630. }
  631. if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) {
  632. sc->intr_count = req_vectors;
  633. tot_vectors = tot_req_vectors;
  634. rc = pci_alloc_msix(sc->dev, &tot_vectors);
  635. if (rc != 0) {
  636. use_intx = 1;
  637. pci_release_msi(sc->dev);
  638. } else {
  639. if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
  640. if (tot_vectors < tot_req_vectors) {
  641. if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) {
  642. sc->roce_intr_count = (tot_vectors / 2);
  643. }
  644. sc->intr_count = tot_vectors - sc->roce_intr_count;
  645. }
  646. } else {
  647. sc->intr_count = tot_vectors;
  648. }
  649. sc->flags |= OCE_FLAGS_USING_MSIX;
  650. }
  651. } else
  652. use_intx = 1;
  653. if (use_intx)
  654. sc->intr_count = 1;
  655. /* Scale number of queues based on intr we got */
  656. update_queues_got(sc);
  657. if (use_intx) {
  658. device_printf(sc->dev, "Using legacy interrupt\n");
  659. rc = oce_alloc_intr(sc, vector, oce_intr);
  660. if (rc)
  661. goto error;
  662. } else {
  663. for (; vector < sc->intr_count; vector++) {
  664. rc = oce_alloc_intr(sc, vector, oce_intr);
  665. if (rc)
  666. goto error;
  667. }
  668. }
  669. return 0;
  670. error:
  671. oce_intr_free(sc);
  672. return rc;
  673. }
  674. static int
  675. oce_fast_isr(void *arg)
  676. {
  677. POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
  678. POCE_SOFTC sc = ii->sc;
  679. if (ii->eq == NULL)
  680. return FILTER_STRAY;
  681. oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE);
  682. taskqueue_enqueue(ii->tq, &ii->task);
  683. ii->eq->intr++;
  684. return FILTER_HANDLED;
  685. }
  686. static int
  687. oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending))
  688. {
  689. POCE_INTR_INFO ii;
  690. int rc = 0, rr;
  691. if (vector >= OCE_MAX_EQ)
  692. return (EINVAL);
  693. ii = &sc->intrs[vector];
  694. /* Set the resource id for the interrupt.
  695. * MSIx is vector + 1 for the resource id,
  696. * INTx is 0 for the resource id.
  697. */
  698. if (sc->flags & OCE_FLAGS_USING_MSIX)
  699. rr = vector + 1;
  700. else
  701. rr = 0;
  702. ii->intr_res = bus_alloc_resource_any(sc->dev,
  703. SYS_RES_IRQ,
  704. &rr, RF_ACTIVE|RF_SHAREABLE);
  705. ii->irq_rr = rr;
  706. if (ii->intr_res == NULL) {
  707. device_printf(sc->dev,
  708. "Could not allocate interrupt\n");
  709. rc = ENXIO;
  710. return rc;
  711. }
  712. TASK_INIT(&ii->task, 0, isr, ii);
  713. ii->vector = vector;
  714. sprintf(ii->task_name, "oce_task[%d]", ii->vector);
  715. ii->tq = taskqueue_create_fast(ii->task_name,
  716. M_NOWAIT,
  717. taskqueue_thread_enqueue,
  718. &ii->tq);
  719. taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq",
  720. device_get_nameunit(sc->dev));
  721. ii->sc = sc;
  722. rc = bus_setup_intr(sc->dev,
  723. ii->intr_res,
  724. INTR_TYPE_NET,
  725. oce_fast_isr, NULL, ii, &ii->tag);
  726. return rc;
  727. }
  728. void
  729. oce_intr_free(POCE_SOFTC sc)
  730. {
  731. int i = 0;
  732. for (i = 0; i < sc->intr_count; i++) {
  733. if (sc->intrs[i].tag != NULL)
  734. bus_teardown_intr(sc->dev, sc->intrs[i].intr_res,
  735. sc->intrs[i].tag);
  736. if (sc->intrs[i].tq != NULL)
  737. taskqueue_free(sc->intrs[i].tq);
  738. if (sc->intrs[i].intr_res != NULL)
  739. bus_release_resource(sc->dev, SYS_RES_IRQ,
  740. sc->intrs[i].irq_rr,
  741. sc->intrs[i].intr_res);
  742. sc->intrs[i].tag = NULL;
  743. sc->intrs[i].intr_res = NULL;
  744. }
  745. if (sc->flags & OCE_FLAGS_USING_MSIX)
  746. pci_release_msi(sc->dev);
  747. }
  748. /******************************************************************************
  749. * Media callbacks functions *
  750. ******************************************************************************/
  751. static void
  752. oce_media_status(struct ifnet *ifp, struct ifmediareq *req)
  753. {
  754. POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc;
  755. req->ifm_status = IFM_AVALID;
  756. req->ifm_active = IFM_ETHER;
  757. if (sc->link_status == 1)
  758. req->ifm_status |= IFM_ACTIVE;
  759. else
  760. return;
  761. switch (sc->link_speed) {
  762. case 1: /* 10 Mbps */
  763. req->ifm_active |= IFM_10_T | IFM_FDX;
  764. sc->speed = 10;
  765. break;
  766. case 2: /* 100 Mbps */
  767. req->ifm_active |= IFM_100_TX | IFM_FDX;
  768. sc->speed = 100;
  769. break;
  770. case 3: /* 1 Gbps */
  771. req->ifm_active |= IFM_1000_T | IFM_FDX;
  772. sc->speed = 1000;
  773. break;
  774. case 4: /* 10 Gbps */
  775. req->ifm_active |= IFM_10G_SR | IFM_FDX;
  776. sc->speed = 10000;
  777. break;
  778. case 5: /* 20 Gbps */
  779. req->ifm_active |= IFM_10G_SR | IFM_FDX;
  780. sc->speed = 20000;
  781. break;
  782. case 6: /* 25 Gbps */
  783. req->ifm_active |= IFM_10G_SR | IFM_FDX;
  784. sc->speed = 25000;
  785. break;
  786. case 7: /* 40 Gbps */
  787. req->ifm_active |= IFM_40G_SR4 | IFM_FDX;
  788. sc->speed = 40000;
  789. break;
  790. default:
  791. sc->speed = 0;
  792. break;
  793. }
  794. return;
  795. }
  796. int
  797. oce_media_change(struct ifnet *ifp)
  798. {
  799. return 0;
  800. }
  801. static void oce_is_pkt_dest_bmc(POCE_SOFTC sc,
  802. struct mbuf *m, boolean_t *os2bmc,
  803. struct mbuf **m_new)
  804. {
  805. struct ether_header *eh = NULL;
  806. eh = mtod(m, struct ether_header *);
  807. if (!is_os2bmc_enabled(sc) || *os2bmc) {
  808. *os2bmc = FALSE;
  809. goto done;
  810. }
  811. if (!ETHER_IS_MULTICAST(eh->ether_dhost))
  812. goto done;
  813. if (is_mc_allowed_on_bmc(sc, eh) ||
  814. is_bc_allowed_on_bmc(sc, eh) ||
  815. is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) {
  816. *os2bmc = TRUE;
  817. goto done;
  818. }
  819. if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) {
  820. struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
  821. uint8_t nexthdr = ip6->ip6_nxt;
  822. if (nexthdr == IPPROTO_ICMPV6) {
  823. struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1);
  824. switch (icmp6->icmp6_type) {
  825. case ND_ROUTER_ADVERT:
  826. *os2bmc = is_ipv6_ra_filt_enabled(sc);
  827. goto done;
  828. case ND_NEIGHBOR_ADVERT:
  829. *os2bmc = is_ipv6_na_filt_enabled(sc);
  830. goto done;
  831. default:
  832. break;
  833. }
  834. }
  835. }
  836. if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) {
  837. struct ip *ip = mtod(m, struct ip *);
  838. int iphlen = ip->ip_hl << 2;
  839. struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen);
  840. switch (uh->uh_dport) {
  841. case DHCP_CLIENT_PORT:
  842. *os2bmc = is_dhcp_client_filt_enabled(sc);
  843. goto done;
  844. case DHCP_SERVER_PORT:
  845. *os2bmc = is_dhcp_srvr_filt_enabled(sc);
  846. goto done;
  847. case NET_BIOS_PORT1:
  848. case NET_BIOS_PORT2:
  849. *os2bmc = is_nbios_filt_enabled(sc);
  850. goto done;
  851. case DHCPV6_RAS_PORT:
  852. *os2bmc = is_ipv6_ras_filt_enabled(sc);
  853. goto done;
  854. default:
  855. break;
  856. }
  857. }
  858. done:
  859. if (*os2bmc) {
  860. *m_new = m_dup(m, M_NOWAIT);
  861. if (!*m_new) {
  862. *os2bmc = FALSE;
  863. return;
  864. }
  865. *m_new = oce_insert_vlan_tag(sc, *m_new, NULL);
  866. }
  867. }
  868. /*****************************************************************************
  869. * Transmit routines functions *
  870. *****************************************************************************/
  871. static int
  872. oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
  873. {
  874. int rc = 0, i, retry_cnt = 0;
  875. bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS];
  876. struct mbuf *m, *m_temp, *m_new = NULL;
  877. struct oce_wq *wq = sc->wq[wq_index];
  878. struct oce_packet_desc *pd;
  879. struct oce_nic_hdr_wqe *nichdr;
  880. struct oce_nic_frag_wqe *nicfrag;
  881. struct ether_header *eh = NULL;
  882. int num_wqes;
  883. uint32_t reg_value;
  884. boolean_t complete = TRUE;
  885. boolean_t os2bmc = FALSE;
  886. m = *mpp;
  887. if (!m)
  888. return EINVAL;
  889. if (!(m->m_flags & M_PKTHDR)) {
  890. rc = ENXIO;
  891. goto free_ret;
  892. }
  893. /* Don't allow non-TSO packets longer than MTU */
  894. if (!is_tso_pkt(m)) {
  895. eh = mtod(m, struct ether_header *);
  896. if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE))
  897. goto free_ret;
  898. }
  899. if(oce_tx_asic_stall_verify(sc, m)) {
  900. m = oce_insert_vlan_tag(sc, m, &complete);
  901. if(!m) {
  902. device_printf(sc->dev, "Insertion unsuccessful\n");
  903. return 0;
  904. }
  905. }
  906. /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less
  907. * may cause a transmit stall on that port. So the work-around is to
  908. * pad short packets (<= 32 bytes) to a 36-byte length.
  909. */
  910. if(IS_SH(sc) || IS_XE201(sc) ) {
  911. if(m->m_pkthdr.len <= 32) {
  912. char buf[36];
  913. bzero((void *)buf, 36);
  914. m_append(m, (36 - m->m_pkthdr.len), buf);
  915. }
  916. }
  917. tx_start:
  918. if (m->m_pkthdr.csum_flags & CSUM_TSO) {
  919. /* consolidate packet buffers for TSO/LSO segment offload */
  920. #if defined(INET6) || defined(INET)
  921. m = oce_tso_setup(sc, mpp);
  922. #else
  923. m = NULL;
  924. #endif
  925. if (m == NULL) {
  926. rc = ENXIO;
  927. goto free_ret;
  928. }
  929. }
  930. pd = &wq->pckts[wq->pkt_desc_head];
  931. retry:
  932. rc = bus_dmamap_load_mbuf_sg(wq->tag,
  933. pd->map,
  934. m, segs, &pd->nsegs, BUS_DMA_NOWAIT);
  935. if (rc == 0) {
  936. num_wqes = pd->nsegs + 1;
  937. if (IS_BE(sc) || IS_SH(sc)) {
  938. /*Dummy required only for BE3.*/
  939. if (num_wqes & 1)
  940. num_wqes++;
  941. }
  942. if (num_wqes >= RING_NUM_FREE(wq->ring)) {
  943. bus_dmamap_unload(wq->tag, pd->map);
  944. return EBUSY;
  945. }
  946. atomic_store_rel_int(&wq->pkt_desc_head,
  947. (wq->pkt_desc_head + 1) % \
  948. OCE_WQ_PACKET_ARRAY_SIZE);
  949. bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);
  950. pd->mbuf = m;
  951. nichdr =
  952. RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);
  953. nichdr->u0.dw[0] = 0;
  954. nichdr->u0.dw[1] = 0;
  955. nichdr->u0.dw[2] = 0;
  956. nichdr->u0.dw[3] = 0;
  957. nichdr->u0.s.complete = complete;
  958. nichdr->u0.s.mgmt = os2bmc;
  959. nichdr->u0.s.event = 1;
  960. nichdr->u0.s.crc = 1;
  961. nichdr->u0.s.forward = 0;
  962. nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0;
  963. nichdr->u0.s.udpcs =
  964. (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0;
  965. nichdr->u0.s.tcpcs =
  966. (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0;
  967. nichdr->u0.s.num_wqe = num_wqes;
  968. nichdr->u0.s.total_length = m->m_pkthdr.len;
  969. if (m->m_flags & M_VLANTAG) {
  970. nichdr->u0.s.vlan = 1; /*Vlan present*/
  971. nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag;
  972. }
  973. if (m->m_pkthdr.csum_flags & CSUM_TSO) {
  974. if (m->m_pkthdr.tso_segsz) {
  975. nichdr->u0.s.lso = 1;
  976. nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz;
  977. }
  978. if (!IS_BE(sc) || !IS_SH(sc))
  979. nichdr->u0.s.ipcs = 1;
  980. }
  981. RING_PUT(wq->ring, 1);
  982. atomic_add_int(&wq->ring->num_used, 1);
  983. for (i = 0; i < pd->nsegs; i++) {
  984. nicfrag =
  985. RING_GET_PRODUCER_ITEM_VA(wq->ring,
  986. struct oce_nic_frag_wqe);
  987. nicfrag->u0.s.rsvd0 = 0;
  988. nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr);
  989. nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr);
  990. nicfrag->u0.s.frag_len = segs[i].ds_len;
  991. pd->wqe_idx = wq->ring->pidx;
  992. RING_PUT(wq->ring, 1);
  993. atomic_add_int(&wq->ring->num_used, 1);
  994. }
  995. if (num_wqes > (pd->nsegs + 1)) {
  996. nicfrag =
  997. RING_GET_PRODUCER_ITEM_VA(wq->ring,
  998. struct oce_nic_frag_wqe);
  999. nicfrag->u0.dw[0] = 0;
  1000. nicfrag->u0.dw[1] = 0;
  1001. nicfrag->u0.dw[2] = 0;
  1002. nicfrag->u0.dw[3] = 0;
  1003. pd->wqe_idx = wq->ring->pidx;
  1004. RING_PUT(wq->ring, 1);
  1005. atomic_add_int(&wq->ring->num_used, 1);
  1006. pd->nsegs++;
  1007. }
  1008. if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
  1009. wq->tx_stats.tx_reqs++;
  1010. wq->tx_stats.tx_wrbs += num_wqes;
  1011. wq->tx_stats.tx_bytes += m->m_pkthdr.len;
  1012. wq->tx_stats.tx_pkts++;
  1013. bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,
  1014. BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
  1015. reg_value = (num_wqes << 16) | wq->wq_id;
  1016. /* if os2bmc is not enabled or if the pkt is already tagged as
  1017. bmc, do nothing
  1018. */
  1019. oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new);
  1020. OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);
  1021. } else if (rc == EFBIG) {
  1022. if (retry_cnt == 0) {
  1023. m_temp = m_defrag(m, M_NOWAIT);
  1024. if (m_temp == NULL)
  1025. goto free_ret;
  1026. m = m_temp;
  1027. *mpp = m_temp;
  1028. retry_cnt = retry_cnt + 1;
  1029. goto retry;
  1030. } else
  1031. goto free_ret;
  1032. } else if (rc == ENOMEM)
  1033. return rc;
  1034. else
  1035. goto free_ret;
  1036. if (os2bmc) {
  1037. m = m_new;
  1038. goto tx_start;
  1039. }
  1040. return 0;
  1041. free_ret:
  1042. m_freem(*mpp);
  1043. *mpp = NULL;
  1044. return rc;
  1045. }
  1046. static void
  1047. oce_process_tx_completion(struct oce_wq *wq)
  1048. {
  1049. struct oce_packet_desc *pd;
  1050. POCE_SOFTC sc = (POCE_SOFTC) wq->parent;
  1051. struct mbuf *m;
  1052. pd = &wq->pckts[wq->pkt_desc_tail];
  1053. atomic_store_rel_int(&wq->pkt_desc_tail,
  1054. (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);
  1055. atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);
  1056. bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
  1057. bus_dmamap_unload(wq->tag, pd->map);
  1058. m = pd->mbuf;
  1059. m_freem(m);
  1060. pd->mbuf = NULL;
  1061. if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) {
  1062. if (wq->ring->num_used < (wq->ring->num_items / 2)) {
  1063. sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
  1064. oce_tx_restart(sc, wq);
  1065. }
  1066. }
  1067. }
  1068. static void
  1069. oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
  1070. {
  1071. if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
  1072. return;
  1073. #if __FreeBSD_version >= 800000
  1074. if (!drbr_empty(sc->ifp, wq->br))
  1075. #else
  1076. if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd))
  1077. #endif
  1078. taskqueue_enqueue(taskqueue_swi, &wq->txtask);
  1079. }
  1080. #if defined(INET6) || defined(INET)
  1081. static struct mbuf *
  1082. oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
  1083. {
  1084. struct mbuf *m;
  1085. #ifdef INET
  1086. struct ip *ip;
  1087. #endif
  1088. #ifdef INET6
  1089. struct ip6_hdr *ip6;
  1090. #endif
  1091. struct ether_vlan_header *eh;
  1092. struct tcphdr *th;
  1093. uint16_t etype;
  1094. int total_len = 0, ehdrlen = 0;
  1095. m = *mpp;
  1096. if (M_WRITABLE(m) == 0) {
  1097. m = m_dup(*mpp, M_NOWAIT);
  1098. if (!m)
  1099. return NULL;
  1100. m_freem(*mpp);
  1101. *mpp = m;
  1102. }
  1103. eh = mtod(m, struct ether_vlan_header *);
  1104. if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
  1105. etype = ntohs(eh->evl_proto);
  1106. ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
  1107. } else {
  1108. etype = ntohs(eh->evl_encap_proto);
  1109. ehdrlen = ETHER_HDR_LEN;
  1110. }
  1111. switch (etype) {
  1112. #ifdef INET
  1113. case ETHERTYPE_IP:
  1114. ip = (struct ip *)(m->m_data + ehdrlen);
  1115. if (ip->ip_p != IPPROTO_TCP)
  1116. return NULL;
  1117. th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
  1118. total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2);
  1119. break;
  1120. #endif
  1121. #ifdef INET6
  1122. case ETHERTYPE_IPV6:
  1123. ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen);
  1124. if (ip6->ip6_nxt != IPPROTO_TCP)
  1125. return NULL;
  1126. th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
  1127. total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2);
  1128. break;
  1129. #endif
  1130. default:
  1131. return NULL;
  1132. }
  1133. m = m_pullup(m, total_len);
  1134. if (!m)
  1135. return NULL;
  1136. *mpp = m;
  1137. return m;
  1138. }
  1139. #endif /* INET6 || INET */
  1140. void
  1141. oce_tx_task(void *arg, int npending)
  1142. {
  1143. struct oce_wq *wq = arg;
  1144. POCE_SOFTC sc = wq->parent;
  1145. struct ifnet *ifp = sc->ifp;
  1146. int rc = 0;
  1147. #if __FreeBSD_version >= 800000
  1148. LOCK(&wq->tx_lock);
  1149. rc = oce_multiq_transmit(ifp, NULL, wq);
  1150. if (rc) {
  1151. device_printf(sc->dev,
  1152. "TX[%d] restart failed\n", wq->queue_index);
  1153. }
  1154. UNLOCK(&wq->tx_lock);
  1155. #else
  1156. oce_start(ifp);
  1157. #endif
  1158. }
  1159. void
  1160. oce_start(struct ifnet *ifp)
  1161. {
  1162. POCE_SOFTC sc = ifp->if_softc;
  1163. struct mbuf *m;
  1164. int rc = 0;
  1165. int def_q = 0; /* Defualt tx queue is 0*/
  1166. if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
  1167. IFF_DRV_RUNNING)
  1168. return;
  1169. if (!sc->link_status)
  1170. return;
  1171. do {
  1172. IF_DEQUEUE(&sc->ifp->if_snd, m);
  1173. if (m == NULL)
  1174. break;
  1175. LOCK(&sc->wq[def_q]->tx_lock);
  1176. rc = oce_tx(sc, &m, def_q);
  1177. UNLOCK(&sc->wq[def_q]->tx_lock);
  1178. if (rc) {
  1179. if (m != NULL) {
  1180. sc->wq[def_q]->tx_stats.tx_stops ++;
  1181. ifp->if_drv_flags |= IFF_DRV_OACTIVE;
  1182. IFQ_DRV_PREPEND(&ifp->if_snd, m);
  1183. m = NULL;
  1184. }
  1185. break;
  1186. }
  1187. if (m != NULL)
  1188. ETHER_BPF_MTAP(ifp, m);
  1189. } while (TRUE);
  1190. return;
  1191. }
  1192. /* Handle the Completion Queue for transmit */
  1193. uint16_t
  1194. oce_wq_handler(void *arg)
  1195. {
  1196. struct oce_wq *wq = (struct oce_wq *)arg;
  1197. POCE_SOFTC sc = wq->parent;
  1198. struct oce_cq *cq = wq->cq;
  1199. struct oce_nic_tx_cqe *cqe;
  1200. int num_cqes = 0;
  1201. LOCK(&wq->tx_compl_lock);
  1202. bus_dmamap_sync(cq->ring->dma.tag,
  1203. cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
  1204. cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
  1205. while (cqe->u0.dw[3]) {
  1206. DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe));
  1207. wq->ring->cidx = cqe->u0.s.wqe_index + 1;
  1208. if (wq->ring->cidx >= wq->ring->num_items)
  1209. wq->ring->cidx -= wq->ring->num_items;
  1210. oce_process_tx_completion(wq);
  1211. wq->tx_stats.tx_compl++;
  1212. cqe->u0.dw[3] = 0;
  1213. RING_GET(cq->ring, 1);
  1214. bus_dmamap_sync(cq->ring->dma.tag,
  1215. cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
  1216. cqe =
  1217. RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
  1218. num_cqes++;
  1219. }
  1220. if (num_cqes)
  1221. oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
  1222. UNLOCK(&wq->tx_compl_lock);
  1223. return num_cqes;
  1224. }
  1225. static int
  1226. oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq)
  1227. {
  1228. POCE_SOFTC sc = ifp->if_softc;
  1229. int status = 0, queue_index = 0;
  1230. struct mbuf *next = NULL;
  1231. struct buf_ring *br = NULL;
  1232. br = wq->br;
  1233. queue_index = wq->queue_index;
  1234. if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
  1235. IFF_DRV_RUNNING) {
  1236. if (m != NULL)
  1237. status = drbr_enqueue(ifp, br, m);
  1238. return status;
  1239. }
  1240. if (m != NULL) {
  1241. if ((status = drbr_enqueue(ifp, br, m)) != 0)
  1242. return status;
  1243. }
  1244. while ((next = drbr_peek(ifp, br)) != NULL) {
  1245. if (oce_tx(sc, &next, queue_index)) {
  1246. if (next == NULL) {
  1247. drbr_advance(ifp, br);
  1248. } else {
  1249. drbr_putback(ifp, br, next);
  1250. wq->tx_stats.tx_stops ++;
  1251. ifp->if_drv_flags |= IFF_DRV_OACTIVE;
  1252. }
  1253. break;
  1254. }
  1255. drbr_advance(ifp, br);
  1256. if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len);
  1257. if (next->m_flags & M_MCAST)
  1258. if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
  1259. ETHER_BPF_MTAP(ifp, next);
  1260. }
  1261. return 0;
  1262. }
  1263. /*****************************************************************************
  1264. * Receive routines functions *
  1265. *****************************************************************************/
  1266. static void
  1267. oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2)
  1268. {
  1269. uint32_t *p;
  1270. struct ether_header *eh = NULL;
  1271. struct tcphdr *tcp_hdr = NULL;
  1272. struct ip *ip4_hdr = NULL;
  1273. struct ip6_hdr *ip6 = NULL;
  1274. uint32_t payload_len = 0;
  1275. eh = mtod(m, struct ether_header *);
  1276. /* correct IP header */
  1277. if(!cqe2->ipv6_frame) {
  1278. ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header));
  1279. ip4_hdr->ip_ttl = cqe2->frame_lifespan;
  1280. ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header));
  1281. tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip));
  1282. }else {
  1283. ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header));
  1284. ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan;
  1285. payload_len = cqe2->coalesced_size - sizeof(struct ether_header)
  1286. - sizeof(struct ip6_hdr);
  1287. ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len);
  1288. tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr));
  1289. }
  1290. /* correct tcp header */
  1291. tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num);
  1292. if(cqe2->push) {
  1293. tcp_hdr->th_flags |= TH_PUSH;
  1294. }
  1295. tcp_hdr->th_win = htons(cqe2->tcp_window);
  1296. tcp_hdr->th_sum = 0xffff;
  1297. if(cqe2->ts_opt) {
  1298. p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2);
  1299. *p = cqe1->tcp_timestamp_val;
  1300. *(p+1) = cqe1->tcp_timestamp_ecr;
  1301. }
  1302. return;
  1303. }
  1304. static void
  1305. oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m)
  1306. {
  1307. POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
  1308. uint32_t i = 0, frag_len = 0;
  1309. uint32_t len = cqe_info->pkt_size;
  1310. struct oce_packet_desc *pd;
  1311. struct mbuf *tail = NULL;
  1312. for (i = 0; i < cqe_info->num_frags; i++) {
  1313. if (rq->ring->cidx == rq->ring->pidx) {
  1314. device_printf(sc->dev,
  1315. "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n");
  1316. return;
  1317. }
  1318. pd = &rq->pckts[rq->ring->cidx];
  1319. bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
  1320. bus_dmamap_unload(rq->tag, pd->map);
  1321. RING_GET(rq->ring, 1);
  1322. rq->pending--;
  1323. frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len;
  1324. pd->mbuf->m_len = frag_len;
  1325. if (tail != NULL) {
  1326. /* additional fragments */
  1327. pd->mbuf->m_flags &= ~M_PKTHDR;
  1328. tail->m_next = pd->mbuf;
  1329. if(rq->islro)
  1330. tail->m_nextpkt = NULL;
  1331. tail = pd->mbuf;
  1332. } else {
  1333. /* first fragment, fill out much of the packet header */
  1334. pd->mbuf->m_pkthdr.len = len;
  1335. if(rq->islro)
  1336. pd->mbuf->m_nextpkt = NULL;
  1337. pd->mbuf->m_pkthdr.csum_flags = 0;
  1338. if (IF_CSUM_ENABLED(sc)) {
  1339. if (cqe_info->l4_cksum_pass) {
  1340. if(!cqe_info->ipv6_frame) { /* IPV4 */
  1341. pd->mbuf->m_pkthdr.csum_flags |=
  1342. (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
  1343. }else { /* IPV6 frame */
  1344. if(rq->islro) {
  1345. pd->mbuf->m_pkthdr.csum_flags |=
  1346. (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
  1347. }
  1348. }
  1349. pd->mbuf->m_pkthdr.csum_data = 0xffff;
  1350. }
  1351. if (cqe_info->ip_cksum_pass) {
  1352. pd->mbuf->m_pkthdr.csum_flags |=
  1353. (CSUM_IP_CHECKED|CSUM_IP_VALID);
  1354. }
  1355. }
  1356. *m = tail = pd->mbuf;
  1357. }
  1358. pd->mbuf = NULL;
  1359. len -= frag_len;
  1360. }
  1361. return;
  1362. }
  1363. static void
  1364. oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2)
  1365. {
  1366. POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
  1367. struct nic_hwlro_cqe_part1 *cqe1 = NULL;
  1368. struct mbuf *m = NULL;
  1369. struct oce_common_cqe_info cq_info;
  1370. /* parse cqe */
  1371. if(cqe2 == NULL) {
  1372. cq_info.pkt_size = cqe->pkt_size;
  1373. cq_info.vtag = cqe->vlan_tag;
  1374. cq_info.l4_cksum_pass = cqe->l4_cksum_pass;
  1375. cq_info.ip_cksum_pass = cqe->ip_cksum_pass;
  1376. cq_info.ipv6_frame = cqe->ipv6_frame;
  1377. cq_info.vtp = cqe->vtp;
  1378. cq_info.qnq = cqe->qnq;
  1379. }else {
  1380. cqe1 = (struct nic_hwlro_cqe_part1 *)cqe;
  1381. cq_info.pkt_size = cqe2->coalesced_size;
  1382. cq_info.vtag = cqe2->vlan_tag;
  1383. cq_info.l4_cksum_pass = cqe2->l4_cksum_pass;
  1384. cq_info.ip_cksum_pass = cqe2->ip_cksum_pass;
  1385. cq_info.ipv6_frame = cqe2->ipv6_frame;
  1386. cq_info.vtp = cqe2->vtp;
  1387. cq_info.qnq = cqe1->qnq;
  1388. }
  1389. cq_info.vtag = BSWAP_16(cq_info.vtag);
  1390. cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size;
  1391. if(cq_info.pkt_size % rq->cfg.frag_size)
  1392. cq_info.num_frags++;
  1393. oce_rx_mbuf_chain(rq, &cq_info, &m);
  1394. if (m) {
  1395. if(cqe2) {
  1396. //assert(cqe2->valid != 0);
  1397. //assert(cqe2->cqe_type != 2);
  1398. oce_correct_header(m, cqe1, cqe2);
  1399. }
  1400. m->m_pkthdr.rcvif = sc->ifp;
  1401. #if __FreeBSD_version >= 800000
  1402. if (rq->queue_index)
  1403. m->m_pkthdr.flowid = (rq->queue_index - 1);
  1404. else
  1405. m->m_pkthdr.flowid = rq->queue_index;
  1406. M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
  1407. #endif
  1408. /* This deternies if vlan tag is Valid */
  1409. if (cq_info.vtp) {
  1410. if (sc->function_mode & FNM_FLEX10_MODE) {
  1411. /* FLEX10. If QnQ is not set, neglect VLAN */
  1412. if (cq_info.qnq) {
  1413. m->m_pkthdr.ether_vtag = cq_info.vtag;
  1414. m->m_flags |= M_VLANTAG;
  1415. }
  1416. } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) {
  1417. /* In UMC mode generally pvid will be striped by
  1418. hw. But in some cases we have seen it comes
  1419. with pvid. So if pvid == vlan, neglect vlan.
  1420. */
  1421. m->m_pkthdr.ether_vtag = cq_info.vtag;
  1422. m->m_flags |= M_VLANTAG;
  1423. }
  1424. }
  1425. if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
  1426. (*sc->ifp->if_input) (sc->ifp, m);
  1427. /* Update rx stats per queue */
  1428. rq->rx_stats.rx_pkts++;
  1429. rq->rx_stats.rx_bytes += cq_info.pkt_size;
  1430. rq->rx_stats.rx_frags += cq_info.num_frags;
  1431. rq->rx_stats.rx_ucast_pkts++;
  1432. }
  1433. return;
  1434. }
  1435. static void
  1436. oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe)
  1437. {
  1438. POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
  1439. int len;
  1440. struct mbuf *m = NULL;
  1441. struct oce_common_cqe_info cq_info;
  1442. uint16_t vtag = 0;
  1443. /* Is it a flush compl that has no data */
  1444. if(!cqe->u0.s.num_fragments)
  1445. goto exit;
  1446. len = cqe->u0.s.pkt_size;
  1447. if (!len) {
  1448. /*partial DMA workaround for Lancer*/
  1449. oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
  1450. goto exit;
  1451. }
  1452. if (!oce_cqe_portid_valid(sc, cqe)) {
  1453. oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
  1454. goto exit;
  1455. }
  1456. /* Get vlan_tag value */
  1457. if(IS_BE(sc) || IS_SH(sc))
  1458. vtag = BSWAP_16(cqe->u0.s.vlan_tag);
  1459. else
  1460. vtag = cqe->u0.s.vlan_tag;
  1461. cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass;
  1462. cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass;
  1463. cq_info.ipv6_frame = cqe->u0.s.ip_ver;
  1464. cq_info.num_frags = cqe->u0.s.num_fragments;
  1465. cq_info.pkt_size = cqe->u0.s.pkt_size;
  1466. oce_rx_mbuf_chain(rq, &cq_info, &m);
  1467. if (m) {
  1468. m->m_pkthdr.rcvif = sc->ifp;
  1469. #if __FreeBSD_version >= 800000
  1470. if (rq->queue_index)
  1471. m->m_pkthdr.flowid = (rq->queue_index - 1);
  1472. else
  1473. m->m_pkthdr.flowid = rq->queue_index;
  1474. M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
  1475. #endif
  1476. /* This deternies if vlan tag is Valid */
  1477. if (oce_cqe_vtp_valid(sc, cqe)) {
  1478. if (sc->function_mode & FNM_FLEX10_MODE) {
  1479. /* FLEX10. If QnQ is not set, neglect VLAN */
  1480. if (cqe->u0.s.qnq) {
  1481. m->m_pkthdr.ether_vtag = vtag;
  1482. m->m_flags |= M_VLANTAG;
  1483. }
  1484. } else if (sc->pvid != (vtag & VLAN_VID_MASK)) {
  1485. /* In UMC mode generally pvid will be striped by
  1486. hw. But in some cases we have seen it comes
  1487. with pvid. So if pvid == vlan, neglect vlan.
  1488. */
  1489. m->m_pkthdr.ether_vtag = vtag;
  1490. m->m_flags |= M_VLANTAG;
  1491. }
  1492. }
  1493. if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
  1494. #if defined(INET6) || defined(INET)
  1495. /* Try to queue to LRO */
  1496. if (IF_LRO_ENABLED(sc) &&
  1497. (cqe->u0.s.ip_cksum_pass) &&
  1498. (cqe->u0.s.l4_cksum_pass) &&
  1499. (!cqe->u0.s.ip_ver) &&
  1500. (rq->lro.lro_cnt != 0)) {
  1501. if (tcp_lro_rx(&rq->lro, m, 0) == 0) {
  1502. rq->lro_pkts_queued ++;
  1503. goto post_done;
  1504. }
  1505. /* If LRO posting fails then try to post to STACK */
  1506. }
  1507. #endif
  1508. (*sc->ifp->if_input) (sc->ifp, m);
  1509. #if defined(INET6) || defined(INET)
  1510. post_done:
  1511. #endif
  1512. /* Update rx stats per queue */
  1513. rq->rx_stats.rx_pkts++;
  1514. rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size;
  1515. rq->rx_stats.rx_frags += cqe->u0.s.num_fragments;
  1516. if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET)
  1517. rq->rx_stats.rx_mcast_pkts++;
  1518. if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET)
  1519. rq->rx_stats.rx_ucast_pkts++;
  1520. }
  1521. exit:
  1522. return;
  1523. }
  1524. void
  1525. oce_discard_rx_comp(struct oce_rq *rq, int num_frags)
  1526. {
  1527. uint32_t i = 0;
  1528. struct oce_packet_desc *pd;
  1529. POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
  1530. for (i = 0; i < num_frags; i++) {
  1531. if (rq->ring->cidx == rq->ring->pidx) {
  1532. device_printf(sc->dev,
  1533. "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n");
  1534. return;
  1535. }
  1536. pd = &rq->pckts[rq->ring->cidx];
  1537. bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
  1538. bus_dmamap_unload(rq->tag, pd->map);
  1539. if (pd->mbuf != NULL) {
  1540. m_freem(pd->mbuf);
  1541. pd->mbuf = NULL;
  1542. }
  1543. RING_GET(rq->ring, 1);
  1544. rq->pending--;
  1545. }
  1546. }
  1547. static int
  1548. oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
  1549. {
  1550. struct oce_nic_rx_cqe_v1 *cqe_v1;
  1551. int vtp = 0;
  1552. if (sc->be3_native) {
  1553. cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
  1554. vtp = cqe_v1->u0.s.vlan_tag_present;
  1555. } else
  1556. vtp = cqe->u0.s.vlan_tag_present;
  1557. return vtp;
  1558. }
  1559. static int
  1560. oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
  1561. {
  1562. struct oce_nic_rx_cqe_v1 *cqe_v1;
  1563. int port_id = 0;
  1564. if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) {
  1565. cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
  1566. port_id = cqe_v1->u0.s.port;
  1567. if (sc->port_id != port_id)
  1568. return 0;
  1569. } else
  1570. ;/* For BE3 legacy and Lancer this is dummy */
  1571. return 1;
  1572. }
  1573. #if defined(INET6) || defined(INET)
  1574. void
  1575. oce_rx_flush_lro(struct oce_rq *rq)
  1576. {
  1577. struct lro_ctrl *lro = &rq->lro;
  1578. POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
  1579. if (!IF_LRO_ENABLED(sc))
  1580. return;
  1581. tcp_lro_flush_all(lro);
  1582. rq->lro_pkts_queued = 0;
  1583. return;
  1584. }
  1585. static int
  1586. oce_init_lro(POCE_SOFTC sc)
  1587. {
  1588. struct lro_ctrl *lro = NULL;
  1589. int i = 0, rc = 0;
  1590. for (i = 0; i < sc->nrqs; i++) {
  1591. lro = &sc->rq[i]->lro;
  1592. rc = tcp_lro_init(lro);
  1593. if (rc != 0) {
  1594. device_printf(sc->dev, "LRO init failed\n");
  1595. return rc;
  1596. }
  1597. lro->ifp = sc->ifp;
  1598. }
  1599. return rc;
  1600. }
  1601. void
  1602. oce_free_lro(POCE_SOFTC sc)
  1603. {
  1604. struct lro_ctrl *lro = NULL;
  1605. int i = 0;
  1606. for (i = 0; i < sc->nrqs; i++) {
  1607. lro = &sc->rq[i]->lro;
  1608. if (lro)
  1609. tcp_lro_free(lro);
  1610. }
  1611. }
  1612. #endif
  1613. int
  1614. oce_alloc_rx_bufs(struct oce_rq *rq, int count)
  1615. {
  1616. POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
  1617. int i, in, rc;
  1618. struct oce_packet_desc *pd;
  1619. bus_dma_segment_t segs[6];
  1620. int nsegs, added = 0;
  1621. struct oce_nic_rqe *rqe;
  1622. pd_rxulp_db_t rxdb_reg;
  1623. uint32_t val = 0;
  1624. uint32_t oce_max_rq_posts = 64;
  1625. bzero(&rxdb_reg, sizeof(pd_rxulp_db_t));
  1626. for (i = 0; i < count; i++) {
  1627. in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE;
  1628. pd = &rq->pckts[rq->ring->pidx];
  1629. pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size);
  1630. if (pd->mbuf == NULL) {
  1631. device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size);
  1632. break;
  1633. }
  1634. pd->mbuf->m_nextpkt = NULL;
  1635. pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size;
  1636. rc = bus_dmamap_load_mbuf_sg(rq->tag,
  1637. pd->map,
  1638. pd->mbuf,
  1639. segs, &nsegs, BUS_DMA_NOWAIT);
  1640. if (rc) {
  1641. m_free(pd->mbuf);
  1642. device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc);
  1643. break;
  1644. }
  1645. if (nsegs != 1) {
  1646. i--;
  1647. continue;
  1648. }
  1649. bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD);
  1650. rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe);
  1651. rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr);
  1652. rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr);
  1653. DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe));
  1654. RING_PUT(rq->ring, 1);
  1655. added++;
  1656. rq->pending++;
  1657. }
  1658. oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS;
  1659. if (added != 0) {
  1660. for (i = added / oce_max_rq_posts; i > 0; i--) {
  1661. rxdb_reg.bits.num_posted = oce_max_rq_posts;
  1662. rxdb_reg.bits.qid = rq->rq_id;
  1663. if(rq->islro) {
  1664. val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
  1665. val |= oce_max_rq_posts << 16;
  1666. OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
  1667. }else {
  1668. OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
  1669. }
  1670. added -= oce_max_rq_posts;
  1671. }
  1672. if (added > 0) {
  1673. rxdb_reg.bits.qid = rq->rq_id;
  1674. rxdb_reg.bits.num_posted = added;
  1675. if(rq->islro) {
  1676. val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
  1677. val |= added << 16;
  1678. OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
  1679. }else {
  1680. OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
  1681. }
  1682. }
  1683. }
  1684. return 0;
  1685. }
  1686. static void
  1687. oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq)
  1688. {
  1689. if (num_cqes) {
  1690. oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE);
  1691. if(!sc->enable_hwlro) {
  1692. if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1)
  1693. oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1));
  1694. }else {
  1695. if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64)
  1696. oce_alloc_rx_bufs(rq, 64);
  1697. }
  1698. }
  1699. return;
  1700. }
  1701. uint16_t
  1702. oce_rq_handler_lro(void *arg)
  1703. {
  1704. struct oce_rq *rq = (struct oce_rq *)arg;
  1705. struct oce_cq *cq = rq->cq;
  1706. POCE_SOFTC sc = rq->parent;
  1707. struct nic_hwlro_singleton_cqe *cqe;
  1708. struct nic_hwlro_cqe_part2 *cqe2;
  1709. int num_cqes = 0;
  1710. LOCK(&rq->rx_lock);
  1711. bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
  1712. cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
  1713. while (cqe->valid) {
  1714. if(cqe->cqe_type == 0) { /* singleton cqe */
  1715. /* we should not get singleton cqe after cqe1 on same rq */
  1716. if(rq->cqe_firstpart != NULL) {
  1717. device_printf(sc->dev, "Got singleton cqe after cqe1 \n");
  1718. goto exit_rq_handler_lro;
  1719. }
  1720. if(cqe->error != 0) {
  1721. rq->rx_stats.rxcp_err++;
  1722. if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
  1723. }
  1724. oce_rx_lro(rq, cqe, NULL);
  1725. rq->rx_stats.rx_compl++;
  1726. cqe->valid = 0;
  1727. RING_GET(cq->ring, 1);
  1728. num_cqes++;
  1729. if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
  1730. break;
  1731. }else if(cqe->cqe_type == 0x1) { /* first part */
  1732. /* we should not get cqe1 after cqe1 on same rq */
  1733. if(rq->cqe_firstpart != NULL) {
  1734. device_printf(sc->dev, "Got cqe1 after cqe1 \n");
  1735. goto exit_rq_handler_lro;
  1736. }
  1737. rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe;
  1738. RING_GET(cq->ring, 1);
  1739. }else if(cqe->cqe_type == 0x2) { /* second part */
  1740. cqe2 = (struct nic_hwlro_cqe_part2 *)cqe;
  1741. if(cqe2->error != 0) {
  1742. rq->rx_stats.rxcp_err++;
  1743. if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
  1744. }
  1745. /* We should not get cqe2 without cqe1 */
  1746. if(rq->cqe_firstpart == NULL) {
  1747. device_printf(sc->dev, "Got cqe2 without cqe1 \n");
  1748. goto exit_rq_handler_lro;
  1749. }
  1750. oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2);
  1751. rq->rx_stats.rx_compl++;
  1752. rq->cqe_firstpart->valid = 0;
  1753. cqe2->valid = 0;
  1754. rq->cqe_firstpart = NULL;
  1755. RING_GET(cq->ring, 1);
  1756. num_cqes += 2;
  1757. if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
  1758. break;
  1759. }
  1760. bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
  1761. cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
  1762. }
  1763. oce_check_rx_bufs(sc, num_cqes, rq);
  1764. exit_rq_handler_lro:
  1765. UNLOCK(&rq->rx_lock);
  1766. return 0;
  1767. }
  1768. /* Handle the Completion Queue for receive */
  1769. uint16_t
  1770. oce_rq_handler(void *arg)
  1771. {
  1772. struct oce_rq *rq = (struct oce_rq *)arg;
  1773. struct oce_cq *cq = rq->cq;
  1774. POCE_SOFTC sc = rq->parent;
  1775. struct oce_nic_rx_cqe *cqe;
  1776. int num_cqes = 0;
  1777. if(rq->islro) {
  1778. oce_rq_handler_lro(arg);
  1779. return 0;
  1780. }
  1781. LOCK(&rq->rx_lock);
  1782. bus_dmamap_sync(cq->ring->dma.tag,
  1783. cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
  1784. cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
  1785. while (cqe->u0.dw[2]) {
  1786. DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe));
  1787. if (cqe->u0.s.error == 0) {
  1788. oce_rx(rq, cqe);
  1789. } else {
  1790. rq->rx_stats.rxcp_err++;
  1791. if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
  1792. /* Post L3/L4 errors to stack.*/
  1793. oce_rx(rq, cqe);
  1794. }
  1795. rq->rx_stats.rx_compl++;
  1796. cqe->u0.dw[2] = 0;
  1797. #if defined(INET6) || defined(INET)
  1798. if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) {
  1799. oce_rx_flush_lro(rq);
  1800. }
  1801. #endif
  1802. RING_GET(cq->ring, 1);
  1803. bus_dmamap_sync(cq->ring->dma.tag,
  1804. cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
  1805. cqe =
  1806. RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
  1807. num_cqes++;
  1808. if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
  1809. break;
  1810. }
  1811. #if defined(INET6) || defined(INET)
  1812. if (IF_LRO_ENABLED(sc))
  1813. oce_rx_flush_lro(rq);
  1814. #endif
  1815. oce_check_rx_bufs(sc, num_cqes, rq);
  1816. UNLOCK(&rq->rx_lock);
  1817. return 0;
  1818. }
  1819. /*****************************************************************************
  1820. * Helper function prototypes in this file *
  1821. *****************************************************************************/
  1822. static int
  1823. oce_attach_ifp(POCE_SOFTC sc)
  1824. {
  1825. sc->ifp = if_alloc(IFT_ETHER);
  1826. if (!sc->ifp)
  1827. return ENOMEM;
  1828. ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status);
  1829. ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
  1830. ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
  1831. sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST;
  1832. sc->ifp->if_ioctl = oce_ioctl;
  1833. sc->ifp->if_start = oce_start;
  1834. sc->ifp->if_init = oce_init;
  1835. sc->ifp->if_mtu = ETHERMTU;
  1836. sc->ifp->if_softc = sc;
  1837. #if __FreeBSD_version >= 800000
  1838. sc->ifp->if_transmit = oce_multiq_start;
  1839. sc->ifp->if_qflush = oce_multiq_flush;
  1840. #endif
  1841. if_initname(sc->ifp,
  1842. device_get_name(sc->dev), device_get_unit(sc->dev));
  1843. sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1;
  1844. IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen);
  1845. IFQ_SET_READY(&sc->ifp->if_snd);
  1846. sc->ifp->if_hwassist = OCE_IF_HWASSIST;
  1847. sc->ifp->if_hwassist |= CSUM_TSO;
  1848. sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP);
  1849. sc->ifp->if_capabilities = OCE_IF_CAPABILITIES;
  1850. sc->ifp->if_capabilities |= IFCAP_HWCSUM;
  1851. sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
  1852. #if defined(INET6) || defined(INET)
  1853. sc->ifp->if_capabilities |= IFCAP_TSO;
  1854. sc->ifp->if_capabilities |= IFCAP_LRO;
  1855. sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
  1856. #endif
  1857. sc->ifp->if_capenable = sc->ifp->if_capabilities;
  1858. sc->ifp->if_baudrate = IF_Gbps(10);
  1859. #if __FreeBSD_version >= 1000000
  1860. sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
  1861. sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS;
  1862. sc->ifp->if_hw_tsomaxsegsize = 4096;
  1863. #endif
  1864. ether_ifattach(sc->ifp, sc->macaddr.mac_addr);
  1865. return 0;
  1866. }
  1867. static void
  1868. oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
  1869. {
  1870. POCE_SOFTC sc = ifp->if_softc;
  1871. if (ifp->if_softc != arg)
  1872. return;
  1873. if ((vtag == 0) || (vtag > 4095))
  1874. return;
  1875. sc->vlan_tag[vtag] = 1;
  1876. sc->vlans_added++;
  1877. if (sc->vlans_added <= (sc->max_vlans + 1))
  1878. oce_vid_config(sc);
  1879. }
  1880. static void
  1881. oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag)
  1882. {
  1883. POCE_SOFTC sc = ifp->if_softc;
  1884. if (ifp->if_softc != arg)
  1885. return;
  1886. if ((vtag == 0) || (vtag > 4095))
  1887. return;
  1888. sc->vlan_tag[vtag] = 0;
  1889. sc->vlans_added--;
  1890. oce_vid_config(sc);
  1891. }
  1892. /*
  1893. * A max of 64 vlans can be configured in BE. If the user configures
  1894. * more, place the card in vlan promiscuous mode.
  1895. */
  1896. static int
  1897. oce_vid_config(POCE_SOFTC sc)
  1898. {
  1899. struct normal_vlan vtags[MAX_VLANFILTER_SIZE];
  1900. uint16_t ntags = 0, i;
  1901. int status = 0;
  1902. if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) &&
  1903. (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) {
  1904. for (i = 0; i < MAX_VLANS; i++) {
  1905. if (sc->vlan_tag[i]) {
  1906. vtags[ntags].vtag = i;
  1907. ntags++;
  1908. }
  1909. }
  1910. if (ntags)
  1911. status = oce_config_vlan(sc, (uint8_t) sc->if_id,
  1912. vtags, ntags, 1, 0);
  1913. } else
  1914. status = oce_config_vlan(sc, (uint8_t) sc->if_id,
  1915. NULL, 0, 1, 1);
  1916. return status;
  1917. }
  1918. static void
  1919. oce_mac_addr_set(POCE_SOFTC sc)
  1920. {
  1921. uint32_t old_pmac_id = sc->pmac_id;
  1922. int status = 0;
  1923. status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
  1924. sc->macaddr.size_of_struct);
  1925. if (!status)
  1926. return;
  1927. status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)),
  1928. sc->if_id, &sc->pmac_id);
  1929. if (!status) {
  1930. status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id);
  1931. bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr,
  1932. sc->macaddr.size_of_struct);
  1933. }
  1934. if (status)
  1935. device_printf(sc->dev, "Failed update macaddress\n");
  1936. }
  1937. static int
  1938. oce_handle_passthrough(struct ifnet *ifp, caddr_t data)
  1939. {
  1940. POCE_SOFTC sc = ifp->if_softc;
  1941. struct ifreq *ifr = (struct ifreq *)data;
  1942. int rc = ENXIO;
  1943. char cookie[32] = {0};
  1944. void *priv_data = ifr_data_get_ptr(ifr);
  1945. void *ioctl_ptr;
  1946. uint32_t req_size;
  1947. struct mbx_hdr req;
  1948. OCE_DMA_MEM dma_mem;
  1949. struct mbx_common_get_cntl_attr *fw_cmd;
  1950. if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE)))
  1951. return EFAULT;
  1952. if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE)))
  1953. return EINVAL;
  1954. ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE);
  1955. if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr)))
  1956. return EFAULT;
  1957. req_size = le32toh(req.u0.req.request_length);
  1958. if (req_size > 65536)
  1959. return EINVAL;
  1960. req_size += sizeof(struct mbx_hdr);
  1961. rc = oce_dma_alloc(sc, req_size, &dma_mem, 0);
  1962. if (rc)
  1963. return ENOMEM;
  1964. if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) {
  1965. rc = EFAULT;
  1966. goto dma_free;
  1967. }
  1968. rc = oce_pass_through_mbox(sc, &dma_mem, req_size);
  1969. if (rc) {
  1970. rc = EIO;
  1971. goto dma_free;
  1972. }
  1973. if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size))
  1974. rc = EFAULT;
  1975. /*
  1976. firmware is filling all the attributes for this ioctl except
  1977. the driver version..so fill it
  1978. */
  1979. if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) {
  1980. fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr;
  1981. strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str,
  1982. COMPONENT_REVISION, strlen(COMPONENT_REVISION));
  1983. }
  1984. dma_free:
  1985. oce_dma_free(sc, &dma_mem);
  1986. return rc;
  1987. }
  1988. static void
  1989. oce_eqd_set_periodic(POCE_SOFTC sc)
  1990. {
  1991. struct oce_set_eqd set_eqd[OCE_MAX_EQ];
  1992. struct oce_aic_obj *aic;
  1993. struct oce_eq *eqo;
  1994. uint64_t now = 0, delta;
  1995. int eqd, i, num = 0;
  1996. uint32_t tx_reqs = 0, rxpkts = 0, pps;
  1997. struct oce_wq *wq;
  1998. struct oce_rq *rq;
  1999. #define ticks_to_msecs(t) (1000 * (t) / hz)
  2000. for (i = 0 ; i < sc->neqs; i++) {
  2001. eqo = sc->eq[i];
  2002. aic = &sc->aic_obj[i];
  2003. /* When setting the static eq delay from the user space */
  2004. if (!aic->enable) {
  2005. if (aic->ticks)
  2006. aic->ticks = 0;
  2007. eqd = aic->et_eqd;
  2008. goto modify_eqd;
  2009. }
  2010. if (i == 0) {
  2011. rq = sc->rq[0];
  2012. rxpkts = rq->rx_stats.rx_pkts;
  2013. } else
  2014. rxpkts = 0;
  2015. if (i + 1 < sc->nrqs) {
  2016. rq = sc->rq[i + 1];
  2017. rxpkts += rq->rx_stats.rx_pkts;
  2018. }
  2019. if (i < sc->nwqs) {
  2020. wq = sc->wq[i];
  2021. tx_reqs = wq->tx_stats.tx_reqs;
  2022. } else
  2023. tx_reqs = 0;
  2024. now = ticks;
  2025. if (!aic->ticks || now < aic->ticks ||
  2026. rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) {
  2027. aic->prev_rxpkts = rxpkts;
  2028. aic->prev_txreqs = tx_reqs;
  2029. aic->ticks = now;
  2030. continue;
  2031. }
  2032. delta = ticks_to_msecs(now - aic->ticks);
  2033. pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) +
  2034. (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta);
  2035. eqd = (pps / 15000) << 2;
  2036. if (eqd < 8)
  2037. eqd = 0;
  2038. /* Make sure that the eq delay is in the known range */
  2039. eqd = min(eqd, aic->max_eqd);
  2040. eqd = max(eqd, aic->min_eqd);
  2041. aic->prev_rxpkts = rxpkts;
  2042. aic->prev_txreqs = tx_reqs;
  2043. aic->ticks = now;
  2044. modify_eqd:
  2045. if (eqd != aic->cur_eqd) {
  2046. set_eqd[num].delay_multiplier = (eqd * 65)/100;
  2047. set_eqd[num].eq_id = eqo->eq_id;
  2048. aic->cur_eqd = eqd;
  2049. num++;
  2050. }
  2051. }
  2052. /* Is there atleast one eq that needs to be modified? */
  2053. for(i = 0; i < num; i += 8) {
  2054. if((num - i) >=8 )
  2055. oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8);
  2056. else
  2057. oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i));
  2058. }
  2059. }
  2060. static void oce_detect_hw_error(POCE_SOFTC sc)
  2061. {
  2062. uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0;
  2063. uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
  2064. uint32_t i;
  2065. if (sc->hw_error)
  2066. return;
  2067. if (IS_XE201(sc)) {
  2068. sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET);
  2069. if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
  2070. sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET);
  2071. sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET);
  2072. }
  2073. } else {
  2074. ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW);
  2075. ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH);
  2076. ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK);
  2077. ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK);
  2078. ue_low = (ue_low & ~ue_low_mask);
  2079. ue_high = (ue_high & ~ue_high_mask);
  2080. }
  2081. /* On certain platforms BE hardware can indicate spurious UEs.
  2082. * Allow the h/w to stop working completely in case of a real UE.
  2083. * Hence not setting the hw_error for UE detection.
  2084. */
  2085. if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
  2086. sc->hw_error = TRUE;
  2087. device_printf(sc->dev, "Error detected in the card\n");
  2088. }
  2089. if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
  2090. device_printf(sc->dev,
  2091. "ERR: sliport status 0x%x\n", sliport_status);
  2092. device_printf(sc->dev,
  2093. "ERR: sliport error1 0x%x\n", sliport_err1);
  2094. device_printf(sc->dev,
  2095. "ERR: sliport error2 0x%x\n", sliport_err2);
  2096. }
  2097. if (ue_low) {
  2098. for (i = 0; ue_low; ue_low >>= 1, i++) {
  2099. if (ue_low & 1)
  2100. device_printf(sc->dev, "UE: %s bit set\n",
  2101. ue_status_low_desc[i]);
  2102. }
  2103. }
  2104. if (ue_high) {
  2105. for (i = 0; ue_high; ue_high >>= 1, i++) {
  2106. if (ue_high & 1)
  2107. device_printf(sc->dev, "UE: %s bit set\n",
  2108. ue_status_hi_desc[i]);
  2109. }
  2110. }
  2111. }
  2112. static void
  2113. oce_local_timer(void *arg)
  2114. {
  2115. POCE_SOFTC sc = arg;
  2116. int i = 0;
  2117. oce_detect_hw_error(sc);
  2118. oce_refresh_nic_stats(sc);
  2119. oce_refresh_queue_stats(sc);
  2120. oce_mac_addr_set(sc);
  2121. /* TX Watch Dog*/
  2122. for (i = 0; i < sc->nwqs; i++)
  2123. oce_tx_restart(sc, sc->wq[i]);
  2124. /* calculate and set the eq delay for optimal interrupt rate */
  2125. if (IS_BE(sc) || IS_SH(sc))
  2126. oce_eqd_set_periodic(sc);
  2127. callout_reset(&sc->timer, hz, oce_local_timer, sc);
  2128. }
  2129. static void
  2130. oce_tx_compl_clean(POCE_SOFTC sc)
  2131. {
  2132. struct oce_wq *wq;
  2133. int i = 0, timeo = 0, num_wqes = 0;
  2134. int pending_txqs = sc->nwqs;
  2135. /* Stop polling for compls when HW has been silent for 10ms or
  2136. * hw_error or no outstanding completions expected
  2137. */
  2138. do {
  2139. pending_txqs = sc->nwqs;
  2140. for_all_wq_queues(sc, wq, i) {
  2141. num_wqes = oce_wq_handler(wq);
  2142. if(num_wqes)
  2143. timeo = 0;
  2144. if(!wq->ring->num_used)
  2145. pending_txqs--;
  2146. }
  2147. if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error)
  2148. break;
  2149. DELAY(1000);
  2150. } while (TRUE);
  2151. for_all_wq_queues(sc, wq, i) {
  2152. while(wq->ring->num_used) {
  2153. LOCK(&wq->tx_compl_lock);
  2154. oce_process_tx_completion(wq);
  2155. UNLOCK(&wq->tx_compl_lock);
  2156. }
  2157. }
  2158. }
  2159. /* NOTE : This should only be called holding
  2160. * DEVICE_LOCK.
  2161. */
  2162. static void
  2163. oce_if_deactivate(POCE_SOFTC sc)
  2164. {
  2165. int i;
  2166. struct oce_rq *rq;
  2167. struct oce_wq *wq;
  2168. struct oce_eq *eq;
  2169. sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
  2170. oce_tx_compl_clean(sc);
  2171. /* Stop intrs and finish any bottom halves pending */
  2172. oce_hw_intr_disable(sc);
  2173. /* Since taskqueue_drain takes a Gaint Lock, We should not acquire
  2174. any other lock. So unlock device lock and require after
  2175. completing taskqueue_drain.
  2176. */
  2177. UNLOCK(&sc->dev_lock);
  2178. for (i = 0; i < sc->intr_count; i++) {
  2179. if (sc->intrs[i].tq != NULL) {
  2180. taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task);
  2181. }
  2182. }
  2183. LOCK(&sc->dev_lock);
  2184. /* Delete RX queue in card with flush param */
  2185. oce_stop_rx(sc);
  2186. /* Invalidate any pending cq and eq entries*/
  2187. for_all_evnt_queues(sc, eq, i)
  2188. oce_drain_eq(eq);
  2189. for_all_rq_queues(sc, rq, i)
  2190. oce_drain_rq_cq(rq);
  2191. for_all_wq_queues(sc, wq, i)
  2192. oce_drain_wq_cq(wq);
  2193. /* But still we need to get MCC aync events.
  2194. So enable intrs and also arm first EQ
  2195. */
  2196. oce_hw_intr_enable(sc);
  2197. oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
  2198. DELAY(10);
  2199. }
  2200. static void
  2201. oce_if_activate(POCE_SOFTC sc)
  2202. {
  2203. struct oce_eq *eq;
  2204. struct oce_rq *rq;
  2205. struct oce_wq *wq;
  2206. int i, rc = 0;
  2207. sc->ifp->if_drv_flags |= IFF_DRV_RUNNING;
  2208. oce_hw_intr_disable(sc);
  2209. oce_start_rx(sc);
  2210. for_all_rq_queues(sc, rq, i) {
  2211. rc = oce_start_rq(rq);
  2212. if (rc)
  2213. device_printf(sc->dev, "Unable to start RX\n");
  2214. }
  2215. for_all_wq_queues(sc, wq, i) {
  2216. rc = oce_start_wq(wq);
  2217. if (rc)
  2218. device_printf(sc->dev, "Unable to start TX\n");
  2219. }
  2220. for_all_evnt_queues(sc, eq, i)
  2221. oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
  2222. oce_hw_intr_enable(sc);
  2223. }
  2224. static void
  2225. process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
  2226. {
  2227. /* Update Link status */
  2228. if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
  2229. ASYNC_EVENT_LINK_UP) {
  2230. sc->link_status = ASYNC_EVENT_LINK_UP;
  2231. if_link_state_change(sc->ifp, LINK_STATE_UP);
  2232. } else {
  2233. sc->link_status = ASYNC_EVENT_LINK_DOWN;
  2234. if_link_state_change(sc->ifp, LINK_STATE_DOWN);
  2235. }
  2236. }
  2237. static void oce_async_grp5_osbmc_process(POCE_SOFTC sc,
  2238. struct oce_async_evt_grp5_os2bmc *evt)
  2239. {
  2240. DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc));
  2241. if (evt->u.s.mgmt_enable)
  2242. sc->flags |= OCE_FLAGS_OS2BMC;
  2243. else
  2244. return;
  2245. sc->bmc_filt_mask = evt->u.s.arp_filter;
  2246. sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1);
  2247. sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2);
  2248. sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3);
  2249. sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4);
  2250. sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5);
  2251. sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6);
  2252. sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7);
  2253. sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8);
  2254. }
  2255. static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe)
  2256. {
  2257. struct oce_async_event_grp5_pvid_state *gcqe;
  2258. struct oce_async_evt_grp5_os2bmc *bmccqe;
  2259. switch (cqe->u0.s.async_type) {
  2260. case ASYNC_EVENT_PVID_STATE:
  2261. /* GRP5 PVID */
  2262. gcqe = (struct oce_async_event_grp5_pvid_state *)cqe;
  2263. if (gcqe->enabled)
  2264. sc->pvid = gcqe->tag & VLAN_VID_MASK;
  2265. else
  2266. sc->pvid = 0;
  2267. break;
  2268. case ASYNC_EVENT_OS2BMC:
  2269. bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe;
  2270. oce_async_grp5_osbmc_process(sc, bmccqe);
  2271. break;
  2272. default:
  2273. break;
  2274. }
  2275. }
  2276. /* Handle the Completion Queue for the Mailbox/Async notifications */
  2277. uint16_t
  2278. oce_mq_handler(void *arg)
  2279. {
  2280. struct oce_mq *mq = (struct oce_mq *)arg;
  2281. POCE_SOFTC sc = mq->parent;
  2282. struct oce_cq *cq = mq->cq;
  2283. int num_cqes = 0, evt_type = 0, optype = 0;
  2284. struct oce_mq_cqe *cqe;
  2285. struct oce_async_cqe_link_state *acqe;
  2286. struct oce_async_event_qnq *dbgcqe;
  2287. bus_dmamap_sync(cq->ring->dma.tag,
  2288. cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
  2289. cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
  2290. while (cqe->u0.dw[3]) {
  2291. DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
  2292. if (cqe->u0.s.async_event) {
  2293. evt_type = cqe->u0.s.event_type;
  2294. optype = cqe->u0.s.async_type;
  2295. if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) {
  2296. /* Link status evt */
  2297. acqe = (struct oce_async_cqe_link_state *)cqe;
  2298. process_link_state(sc, acqe);
  2299. } else if (evt_type == ASYNC_EVENT_GRP5) {
  2300. oce_process_grp5_events(sc, cqe);
  2301. } else if (evt_type == ASYNC_EVENT_CODE_DEBUG &&
  2302. optype == ASYNC_EVENT_DEBUG_QNQ) {
  2303. dbgcqe = (struct oce_async_event_qnq *)cqe;
  2304. if(dbgcqe->valid)
  2305. sc->qnqid = dbgcqe->vlan_tag;
  2306. sc->qnq_debug_event = TRUE;
  2307. }
  2308. }
  2309. cqe->u0.dw[3] = 0;
  2310. RING_GET(cq->ring, 1);
  2311. bus_dmamap_sync(cq->ring->dma.tag,
  2312. cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
  2313. cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
  2314. num_cqes++;
  2315. }
  2316. if (num_cqes)
  2317. oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
  2318. return 0;
  2319. }
  2320. static void
  2321. setup_max_queues_want(POCE_SOFTC sc)
  2322. {
  2323. /* Check if it is FLEX machine. Is so dont use RSS */
  2324. if ((sc->function_mode & FNM_FLEX10_MODE) ||
  2325. (sc->function_mode & FNM_UMC_MODE) ||
  2326. (sc->function_mode & FNM_VNIC_MODE) ||
  2327. (!is_rss_enabled(sc)) ||
  2328. IS_BE2(sc)) {
  2329. sc->nrqs = 1;
  2330. sc->nwqs = 1;
  2331. } else {
  2332. sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
  2333. sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs);
  2334. }
  2335. if (IS_BE2(sc) && is_rss_enabled(sc))
  2336. sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
  2337. }
  2338. static void
  2339. update_queues_got(POCE_SOFTC sc)
  2340. {
  2341. if (is_rss_enabled(sc)) {
  2342. sc->nrqs = sc->intr_count + 1;
  2343. sc->nwqs = sc->intr_count;
  2344. } else {
  2345. sc->nrqs = 1;
  2346. sc->nwqs = 1;
  2347. }
  2348. if (IS_BE2(sc))
  2349. sc->nwqs = 1;
  2350. }
  2351. static int
  2352. oce_check_ipv6_ext_hdr(struct mbuf *m)
  2353. {
  2354. struct ether_header *eh = mtod(m, struct ether_header *);
  2355. caddr_t m_datatemp = m->m_data;
  2356. if (eh->ether_type == htons(ETHERTYPE_IPV6)) {
  2357. m->m_data += sizeof(struct ether_header);
  2358. struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
  2359. if((ip6->ip6_nxt != IPPROTO_TCP) && \
  2360. (ip6->ip6_nxt != IPPROTO_UDP)){
  2361. struct ip6_ext *ip6e = NULL;
  2362. m->m_data += sizeof(struct ip6_hdr);
  2363. ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *);
  2364. if(ip6e->ip6e_len == 0xff) {
  2365. m->m_data = m_datatemp;
  2366. return TRUE;
  2367. }
  2368. }
  2369. m->m_data = m_datatemp;
  2370. }
  2371. return FALSE;
  2372. }
  2373. static int
  2374. is_be3_a1(POCE_SOFTC sc)
  2375. {
  2376. if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) {
  2377. return TRUE;
  2378. }
  2379. return FALSE;
  2380. }
  2381. static struct mbuf *
  2382. oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete)
  2383. {
  2384. uint16_t vlan_tag = 0;
  2385. if(!M_WRITABLE(m))
  2386. return NULL;
  2387. /* Embed vlan tag in the packet if it is not part of it */
  2388. if(m->m_flags & M_VLANTAG) {
  2389. vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag);
  2390. m->m_flags &= ~M_VLANTAG;
  2391. }
  2392. /* if UMC, ignore vlan tag insertion and instead insert pvid */
  2393. if(sc->pvid) {
  2394. if(!vlan_tag)
  2395. vlan_tag = sc->pvid;
  2396. if (complete)
  2397. *complete = FALSE;
  2398. }
  2399. if(vlan_tag) {
  2400. m = ether_vlanencap(m, vlan_tag);
  2401. }
  2402. if(sc->qnqid) {
  2403. m = ether_vlanencap(m, sc->qnqid);
  2404. if (complete)
  2405. *complete = FALSE;
  2406. }
  2407. return m;
  2408. }
  2409. static int
  2410. oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m)
  2411. {
  2412. if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \
  2413. oce_check_ipv6_ext_hdr(m)) {
  2414. return TRUE;
  2415. }
  2416. return FALSE;
  2417. }
  2418. static void
  2419. oce_get_config(POCE_SOFTC sc)
  2420. {
  2421. int rc = 0;
  2422. uint32_t max_rss = 0;
  2423. if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native))
  2424. max_rss = OCE_LEGACY_MODE_RSS;
  2425. else
  2426. max_rss = OCE_MAX_RSS;
  2427. if (!IS_BE(sc)) {
  2428. rc = oce_get_profile_config(sc, max_rss);
  2429. if (rc) {
  2430. sc->nwqs = OCE_MAX_WQ;
  2431. sc->nrssqs = max_rss;
  2432. sc->nrqs = sc->nrssqs + 1;
  2433. }
  2434. }
  2435. else { /* For BE3 don't rely on fw for determining the resources */
  2436. sc->nrssqs = max_rss;
  2437. sc->nrqs = sc->nrssqs + 1;
  2438. sc->nwqs = OCE_MAX_WQ;
  2439. sc->max_vlans = MAX_VLANFILTER_SIZE;
  2440. }
  2441. }
  2442. static void
  2443. oce_rdma_close(void)
  2444. {
  2445. if (oce_rdma_if != NULL) {
  2446. oce_rdma_if = NULL;
  2447. }
  2448. }
  2449. static void
  2450. oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr)
  2451. {
  2452. memcpy(macaddr, sc->macaddr.mac_addr, 6);
  2453. }
  2454. int
  2455. oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if)
  2456. {
  2457. POCE_SOFTC sc;
  2458. struct oce_dev_info di;
  2459. int i;
  2460. if ((rdma_info == NULL) || (rdma_if == NULL)) {
  2461. return -EINVAL;
  2462. }
  2463. if ((rdma_info->size != OCE_RDMA_INFO_SIZE) ||
  2464. (rdma_if->size != OCE_RDMA_IF_SIZE)) {
  2465. return -ENXIO;
  2466. }
  2467. rdma_info->close = oce_rdma_close;
  2468. rdma_info->mbox_post = oce_mbox_post;
  2469. rdma_info->common_req_hdr_init = mbx_common_req_hdr_init;
  2470. rdma_info->get_mac_addr = oce_get_mac_addr;
  2471. oce_rdma_if = rdma_if;
  2472. sc = softc_head;
  2473. while (sc != NULL) {
  2474. if (oce_rdma_if->announce != NULL) {
  2475. memset(&di, 0, sizeof(di));
  2476. di.dev = sc->dev;
  2477. di.softc = sc;
  2478. di.ifp = sc->ifp;
  2479. di.db_bhandle = sc->db_bhandle;
  2480. di.db_btag = sc->db_btag;
  2481. di.db_page_size = 4096;
  2482. if (sc->flags & OCE_FLAGS_USING_MSIX) {
  2483. di.intr_mode = OCE_INTERRUPT_MODE_MSIX;
  2484. } else if (sc->flags & OCE_FLAGS_USING_MSI) {
  2485. di.intr_mode = OCE_INTERRUPT_MODE_MSI;
  2486. } else {
  2487. di.intr_mode = OCE_INTERRUPT_MODE_INTX;
  2488. }
  2489. di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk
  2490. if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) {
  2491. di.msix.num_vectors = sc->intr_count + sc->roce_intr_count;
  2492. di.msix.start_vector = sc->intr_count;
  2493. for (i=0; i<di.msix.num_vectors; i++) {
  2494. di.msix.vector_list[i] = sc->intrs[i].vector;
  2495. }
  2496. } else {
  2497. }
  2498. memcpy(di.mac_addr, sc->macaddr.mac_addr, 6);
  2499. di.vendor_id = pci_get_vendor(sc->dev);
  2500. di.dev_id = pci_get_device(sc->dev);
  2501. if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
  2502. di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED;
  2503. }
  2504. rdma_if->announce(&di);
  2505. sc = sc->next;
  2506. }
  2507. }
  2508. return 0;
  2509. }
  2510. static void
  2511. oce_read_env_variables( POCE_SOFTC sc )
  2512. {
  2513. char *value = NULL;
  2514. int rc = 0;
  2515. /* read if user wants to enable hwlro or swlro */
  2516. //value = getenv("oce_enable_hwlro");
  2517. if(value && IS_SH(sc)) {
  2518. sc->enable_hwlro = strtol(value, NULL, 10);
  2519. if(sc->enable_hwlro) {
  2520. rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL);
  2521. if(rc) {
  2522. device_printf(sc->dev, "no hardware lro support\n");
  2523. device_printf(sc->dev, "software lro enabled\n");
  2524. sc->enable_hwlro = 0;
  2525. }else {
  2526. device_printf(sc->dev, "hardware lro enabled\n");
  2527. oce_max_rsp_handled = 32;
  2528. }
  2529. }else {
  2530. device_printf(sc->dev, "software lro enabled\n");
  2531. }
  2532. }else {
  2533. sc->enable_hwlro = 0;
  2534. }
  2535. /* read mbuf size */
  2536. //value = getenv("oce_rq_buf_size");
  2537. if(value && IS_SH(sc)) {
  2538. oce_rq_buf_size = strtol(value, NULL, 10);
  2539. switch(oce_rq_buf_size) {
  2540. case 2048:
  2541. case 4096:
  2542. case 9216:
  2543. case 16384:
  2544. break;
  2545. default:
  2546. device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n");
  2547. oce_rq_buf_size = 2048;
  2548. }
  2549. }
  2550. return;
  2551. }