libpfctl: grow request buffer on ENOSPC

When we issue a request to pf and expect a serialised nvlist as a reply
we have to supply a suitable buffer to the kernel.
The required size for this buffer is difficult to predict, and may be
(slightly) different from request to request.
If it's insufficient the kernel will return ENOSPC. Teach libpfctl to
catch this and send the request again with a larger buffer.

MFC after:	2 weeks
Sponsored by:   Rubicon Communications, LLC ("Netgate")
Differential Revision:	https://reviews.freebsd.org/D34908

(cherry picked from commit 7ed19f5c)

lib/libpfctl/libpfctl.c

@@ -59,6 +59,49 @@ const char* PFCTL_SYNCOOKIES_MODE_NAMES[] = {
 static int	_pfctl_clear_states(int , const struct pfctl_kill *,
 		    unsigned int *, uint64_t);
 
+static int
+pfctl_do_ioctl(int dev, uint cmd, size_t size, nvlist_t **nvl)
+{
+	struct pfioc_nv nv;
+	void *data;
+	size_t nvlen;
+	int ret;
+
+	data = nvlist_pack(*nvl, &nvlen);
+
+retry:
+	nv.data = malloc(size);
+	memcpy(nv.data, data, nvlen);
+	free(data);
+
+	nv.len = nvlen;
+	nv.size = size;
+
+	ret = ioctl(dev, cmd, &nv);
+	if (ret == -1 && errno == ENOSPC) {
+		size *= 2;
+		free(nv.data);
+		goto retry;
+	}
+
+	nvlist_destroy(*nvl);
+	*nvl = NULL;
+
+	if (ret == 0) {
+		*nvl = nvlist_unpack(nv.data, nv.len, 0);
+		if (*nvl == NULL) {
+			free(nv.data);
+			return (EIO);
+		}
+	} else {
+		ret = errno;
+	}
+
+	free(nv.data);
+
+	return (ret);
+}
+
 static void
 pf_nvuint_8_array(const nvlist_t *nvl, const char *name, size_t maxelems,
     uint8_t *numbers, size_t *nelems)
@@ -159,7 +202,6 @@ _pfctl_get_status_counters(const nvlist_t *nvl,
 struct pfctl_status *
 pfctl_get_status(int dev)
 {
-	struct pfioc_nv	 nv;
 	struct pfctl_status	*status;
 	nvlist_t	*nvl;
 	size_t		 len;
@@ -169,18 +211,9 @@ pfctl_get_status(int dev)
 	if (status == NULL)
 		return (NULL);
 
-	nv.data = malloc(4096);
-	nv.len = nv.size = 4096;
-
-	if (ioctl(dev, DIOCGETSTATUSNV, &nv)) {
-		free(nv.data);
-		free(status);
-		return (NULL);
-	}
+	nvl = nvlist_create(0);
 
-	nvl = nvlist_unpack(nv.data, nv.len, 0);
-	free(nv.data);
-	if (nvl == NULL) {
+	if (pfctl_do_ioctl(dev, DIOCGETSTATUSNV, 4096, &nvl)) {
 		free(status);
 		return (NULL);
 	}
@@ -695,9 +728,7 @@ int	pfctl_get_clear_rule(int dev, uint32_t nr, uint32_t ticket,
 	    const char *anchor, uint32_t ruleset, struct pfctl_rule *rule,
 	    char *anchor_call, bool clear)
 {
-	struct pfioc_nv nv;
 	nvlist_t *nvl;
-	void *nvlpacked;
 	int ret;
 
 	nvl = nvlist_create(0);
@@ -712,30 +743,8 @@ int	pfctl_get_clear_rule(int dev, uint32_t nr, uint32_t ticket,
 	if (clear)
 		nvlist_add_bool(nvl, "clear_counter", true);
 
-	nvlpacked = nvlist_pack(nvl, &nv.len);
-	if (nvlpacked == NULL) {
-		nvlist_destroy(nvl);
-		return (ENOMEM);
-	}
-	nv.data = malloc(8182);
-	nv.size = 8192;
-	assert(nv.len <= nv.size);
-	memcpy(nv.data, nvlpacked, nv.len);
-	nvlist_destroy(nvl);
-	nvl = NULL;
-	free(nvlpacked);
-
-	ret = ioctl(dev, DIOCGETRULENV, &nv);
-	if (ret != 0) {
-		free(nv.data);
+	if ((ret = pfctl_do_ioctl(dev, DIOCGETRULENV, 8192, &nvl)) != 0)
 		return (ret);
-	}
-
-	nvl = nvlist_unpack(nv.data, nv.len, 0);
-	if (nvl == NULL) {
-		free(nv.data);
-		return (EIO);
-	}
 
 	pf_nvrule_to_rule(nvlist_get_nvlist(nvl, "rule"), rule);
 
@@ -743,7 +752,6 @@ int	pfctl_get_clear_rule(int dev, uint32_t nr, uint32_t ticket,
 		strlcpy(anchor_call, nvlist_get_string(nvl, "anchor_call"),
 		    MAXPATHLEN);
 
-	free(nv.data);
 	nvlist_destroy(nvl);
 
 	return (0);
@@ -929,22 +937,8 @@ _pfctl_clear_states(int dev, const struct pfctl_kill *kill,
 	nvlist_add_string(nvl, "label", kill->label);
 	nvlist_add_bool(nvl, "kill_match", kill->kill_match);
 
-	nv.data = nvlist_pack(nvl, &nv.len);
-	nv.size = nv.len;
-	nvlist_destroy(nvl);
-	nvl = NULL;
-
-	ret = ioctl(dev, ioctlval, &nv);
-	if (ret != 0) {
-		free(nv.data);
+	if ((ret = pfctl_do_ioctl(dev, ioctlval, 1024, &nvl)) != 0)
 		return (ret);
-	}
-
-	nvl = nvlist_unpack(nv.data, nv.len, 0);
-	if (nvl == NULL) {
-		free(nv.data);
-		return (EIO);
-	}
 
 	if (killed)
 		*killed = nvlist_get_number(nvl, "killed");
@@ -1082,7 +1076,6 @@ pfctl_set_syncookies(int dev, const struct pfctl_syncookies *s)
 int
 pfctl_get_syncookies(int dev, struct pfctl_syncookies *s)
 {
-	struct pfioc_nv	 nv;
 	nvlist_t	*nvl;
 	int		 ret;
 	uint		 state_limit;
@@ -1094,19 +1087,10 @@ pfctl_get_syncookies(int dev, struct pfctl_syncookies *s)
 
 	bzero(s, sizeof(*s));
 
-	nv.data = malloc(256);
-	nv.len = nv.size = 256;
+	nvl = nvlist_create(0);
 
-	if (ioctl(dev, DIOCGETSYNCOOKIES, &nv)) {
-		free(nv.data);
+	if ((ret = pfctl_do_ioctl(dev, DIOCGETSYNCOOKIES, 256, &nvl)) != 0)
 		return (errno);
-	}
-
-	nvl = nvlist_unpack(nv.data, nv.len, 0);
-	free(nv.data);
-	if (nvl == NULL) {
-		return (EIO);
-	}
 
 	enabled = nvlist_get_bool(nvl, "enabled");
 	adaptive = nvlist_get_bool(nvl, "adaptive");