vfs_bio.c

/*-
 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
 *
 * Copyright (c) 2004 Poul-Henning Kamp
 * Copyright (c) 1994,1997 John S. Dyson
 * Copyright (c) 2013 The FreeBSD Foundation
 * All rights reserved.
 *
 * Portions of this software were developed by Konstantin Belousov
 * under sponsorship from the FreeBSD Foundation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*
 * this file contains a new buffer I/O scheme implementing a coherent
 * VM object and buffer cache scheme.  Pains have been taken to make
 * sure that the performance degradation associated with schemes such
 * as this is not realized.
 *
 * Author:  John S. Dyson
 * Significant help during the development and debugging phases
 * had been provided by David Greenman, also of the FreeBSD core team.
 *
 * see man buf(9) for more info.
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bio.h>
#include <sys/bitset.h>
#include <sys/conf.h>
#include <sys/counter.h>
#include <sys/buf.h>
#include <sys/devicestat.h>
#include <sys/eventhandler.h>
#include <sys/fail.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/mutex.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/proc.h>
#include <sys/racct.h>
#include <sys/resourcevar.h>
#include <sys/rwlock.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/vmem.h>
#include <sys/vmmeter.h>
#include <sys/vnode.h>
#include <sys/watchdog.h>
#include <geom/geom.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_kern.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_pager.h>
#include <vm/vm_extern.h>
#include <vm/vm_map.h>
#include <vm/swap_pager.h>
#include "opt_swap.h"

static MALLOC_DEFINE(M_BIOBUF, "biobuf", "BIO buffer");

struct	bio_ops bioops;		/* I/O operation notification */

struct	buf_ops buf_ops_bio = {
	.bop_name	=	"buf_ops_bio",
	.bop_write	=	bufwrite,
	.bop_strategy	=	bufstrategy,
	.bop_sync	=	bufsync,
	.bop_bdflush	=	bufbdflush,
};

struct bufqueue {
	struct mtx_padalign	bq_lock;
	TAILQ_HEAD(, buf)	bq_queue;
	uint8_t			bq_index;
	uint16_t		bq_subqueue;
	int			bq_len;
} __aligned(CACHE_LINE_SIZE);

#define	BQ_LOCKPTR(bq)		(&(bq)->bq_lock)
#define	BQ_LOCK(bq)		mtx_lock(BQ_LOCKPTR((bq)))
#define	BQ_UNLOCK(bq)		mtx_unlock(BQ_LOCKPTR((bq)))
#define	BQ_ASSERT_LOCKED(bq)	mtx_assert(BQ_LOCKPTR((bq)), MA_OWNED)

struct bufdomain {
	struct bufqueue	bd_subq[MAXCPU + 1]; /* Per-cpu sub queues + global */
	struct bufqueue bd_dirtyq;
	struct bufqueue	*bd_cleanq;
	struct mtx_padalign bd_run_lock;
	/* Constants */
	long		bd_maxbufspace;
	long		bd_hibufspace;
	long 		bd_lobufspace;
	long 		bd_bufspacethresh;
	int		bd_hifreebuffers;
	int		bd_lofreebuffers;
	int		bd_hidirtybuffers;
	int		bd_lodirtybuffers;
	int		bd_dirtybufthresh;
	int		bd_lim;
	/* atomics */
	int		bd_wanted;
	int __aligned(CACHE_LINE_SIZE)	bd_numdirtybuffers;
	int __aligned(CACHE_LINE_SIZE)	bd_running;
	long __aligned(CACHE_LINE_SIZE) bd_bufspace;
	int __aligned(CACHE_LINE_SIZE)	bd_freebuffers;
} __aligned(CACHE_LINE_SIZE);

#define	BD_LOCKPTR(bd)		(&(bd)->bd_cleanq->bq_lock)
#define	BD_LOCK(bd)		mtx_lock(BD_LOCKPTR((bd)))
#define	BD_UNLOCK(bd)		mtx_unlock(BD_LOCKPTR((bd)))
#define	BD_ASSERT_LOCKED(bd)	mtx_assert(BD_LOCKPTR((bd)), MA_OWNED)
#define	BD_RUN_LOCKPTR(bd)	(&(bd)->bd_run_lock)
#define	BD_RUN_LOCK(bd)		mtx_lock(BD_RUN_LOCKPTR((bd)))
#define	BD_RUN_UNLOCK(bd)	mtx_unlock(BD_RUN_LOCKPTR((bd)))
#define	BD_DOMAIN(bd)		(bd - bdomain)

static struct buf *buf;		/* buffer header pool */
extern struct buf *swbuf;	/* Swap buffer header pool. */
caddr_t __read_mostly unmapped_buf;

/* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */
struct proc *bufdaemonproc;

static int inmem(struct vnode *vp, daddr_t blkno);
static void vm_hold_free_pages(struct buf *bp, int newbsize);
static void vm_hold_load_pages(struct buf *bp, vm_offset_t from,
		vm_offset_t to);
static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m);
static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off,
		vm_page_t m);
static void vfs_clean_pages_dirty_buf(struct buf *bp);
static void vfs_setdirty_locked_object(struct buf *bp);
static void vfs_vmio_invalidate(struct buf *bp);
static void vfs_vmio_truncate(struct buf *bp, int npages);
static void vfs_vmio_extend(struct buf *bp, int npages, int size);
static int vfs_bio_clcheck(struct vnode *vp, int size,
		daddr_t lblkno, daddr_t blkno);
static void breada(struct vnode *, daddr_t *, int *, int, struct ucred *, int,
		void (*)(struct buf *));
static int buf_flush(struct vnode *vp, struct bufdomain *, int);
static int flushbufqueues(struct vnode *, struct bufdomain *, int, int);
static void buf_daemon(void);
static __inline void bd_wakeup(void);
static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
static void bufkva_reclaim(vmem_t *, int);
static void bufkva_free(struct buf *);
static int buf_import(void *, void **, int, int, int);
static void buf_release(void *, void **, int);
static void maxbcachebuf_adjust(void);
static inline struct bufdomain *bufdomain(struct buf *);
static void bq_remove(struct bufqueue *bq, struct buf *bp);
static void bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock);
static int buf_recycle(struct bufdomain *, bool kva);
static void bq_init(struct bufqueue *bq, int qindex, int cpu,
	    const char *lockname);
static void bd_init(struct bufdomain *bd);
static int bd_flushall(struct bufdomain *bd);
static int sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS);
static int sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS);

static int sysctl_bufspace(SYSCTL_HANDLER_ARGS);
int vmiodirenable = TRUE;
SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW, &vmiodirenable, 0,
    "Use the VM system for directory writes");
long runningbufspace;
SYSCTL_LONG(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD, &runningbufspace, 0,
    "Amount of presently outstanding async buffer io");
SYSCTL_PROC(_vfs, OID_AUTO, bufspace, CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RD,
    NULL, 0, sysctl_bufspace, "L", "Physical memory used for buffers");
static counter_u64_t bufkvaspace;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace,
    "Kernel virtual memory used for buffers");
static long maxbufspace;
SYSCTL_PROC(_vfs, OID_AUTO, maxbufspace,
    CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &maxbufspace,
    __offsetof(struct bufdomain, bd_maxbufspace), sysctl_bufdomain_long, "L",
    "Maximum allowed value of bufspace (including metadata)");
static long bufmallocspace;
SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0,
    "Amount of malloced memory for buffers");
static long maxbufmallocspace;
SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace,
    0, "Maximum amount of malloced memory for buffers");
static long lobufspace;
SYSCTL_PROC(_vfs, OID_AUTO, lobufspace,
    CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &lobufspace,
    __offsetof(struct bufdomain, bd_lobufspace), sysctl_bufdomain_long, "L",
    "Minimum amount of buffers we want to have");
long hibufspace;
SYSCTL_PROC(_vfs, OID_AUTO, hibufspace,
    CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &hibufspace,
    __offsetof(struct bufdomain, bd_hibufspace), sysctl_bufdomain_long, "L",
    "Maximum allowed value of bufspace (excluding metadata)");
long bufspacethresh;
SYSCTL_PROC(_vfs, OID_AUTO, bufspacethresh,
    CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &bufspacethresh,
    __offsetof(struct bufdomain, bd_bufspacethresh), sysctl_bufdomain_long, "L",
    "Bufspace consumed before waking the daemon to free some");
static counter_u64_t buffreekvacnt;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt,
    "Number of times we have freed the KVA space from some buffer");
static counter_u64_t bufdefragcnt;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW, &bufdefragcnt,
    "Number of times we have had to repeat buffer allocation to defragment");
static long lorunningspace;
SYSCTL_PROC(_vfs, OID_AUTO, lorunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
    CTLFLAG_RW, &lorunningspace, 0, sysctl_runningspace, "L",
    "Minimum preferred space used for in-progress I/O");
static long hirunningspace;
SYSCTL_PROC(_vfs, OID_AUTO, hirunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
    CTLFLAG_RW, &hirunningspace, 0, sysctl_runningspace, "L",
    "Maximum amount of space to use for in-progress I/O");
int dirtybufferflushes;
SYSCTL_INT(_vfs, OID_AUTO, dirtybufferflushes, CTLFLAG_RW, &dirtybufferflushes,
    0, "Number of bdwrite to bawrite conversions to limit dirty buffers");
int bdwriteskip;
SYSCTL_INT(_vfs, OID_AUTO, bdwriteskip, CTLFLAG_RW, &bdwriteskip,
    0, "Number of buffers supplied to bdwrite with snapshot deadlock risk");
int altbufferflushes;
SYSCTL_INT(_vfs, OID_AUTO, altbufferflushes, CTLFLAG_RW, &altbufferflushes,
    0, "Number of fsync flushes to limit dirty buffers");
static int recursiveflushes;
SYSCTL_INT(_vfs, OID_AUTO, recursiveflushes, CTLFLAG_RW, &recursiveflushes,
    0, "Number of flushes skipped due to being recursive");
static int sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS);
SYSCTL_PROC(_vfs, OID_AUTO, numdirtybuffers,
    CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RD, NULL, 0, sysctl_numdirtybuffers, "I",
    "Number of buffers that are dirty (has unwritten changes) at the moment");
static int lodirtybuffers;
SYSCTL_PROC(_vfs, OID_AUTO, lodirtybuffers,
    CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lodirtybuffers,
    __offsetof(struct bufdomain, bd_lodirtybuffers), sysctl_bufdomain_int, "I",
    "How many buffers we want to have free before bufdaemon can sleep");
static int hidirtybuffers;
SYSCTL_PROC(_vfs, OID_AUTO, hidirtybuffers,
    CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hidirtybuffers,
    __offsetof(struct bufdomain, bd_hidirtybuffers), sysctl_bufdomain_int, "I",
    "When the number of dirty buffers is considered severe");
int dirtybufthresh;
SYSCTL_PROC(_vfs, OID_AUTO, dirtybufthresh,
    CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &dirtybufthresh,
    __offsetof(struct bufdomain, bd_dirtybufthresh), sysctl_bufdomain_int, "I",
    "Number of bdwrite to bawrite conversions to clear dirty buffers");
static int numfreebuffers;
SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, &numfreebuffers, 0,
    "Number of free buffers");
static int lofreebuffers;
SYSCTL_PROC(_vfs, OID_AUTO, lofreebuffers,
    CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lofreebuffers,
    __offsetof(struct bufdomain, bd_lofreebuffers), sysctl_bufdomain_int, "I",
   "Target number of free buffers");
static int hifreebuffers;
SYSCTL_PROC(_vfs, OID_AUTO, hifreebuffers,
    CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hifreebuffers,
    __offsetof(struct bufdomain, bd_hifreebuffers), sysctl_bufdomain_int, "I",
   "Threshold for clean buffer recycling");
static counter_u64_t getnewbufcalls;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RD,
   &getnewbufcalls, "Number of calls to getnewbuf");
static counter_u64_t getnewbufrestarts;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RD,
    &getnewbufrestarts,
    "Number of times getnewbuf has had to restart a buffer acquisition");
static counter_u64_t mappingrestarts;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, mappingrestarts, CTLFLAG_RD,
    &mappingrestarts,
    "Number of times getblk has had to restart a buffer mapping for "
    "unmapped buffer");
static counter_u64_t numbufallocfails;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, numbufallocfails, CTLFLAG_RW,
    &numbufallocfails, "Number of times buffer allocations failed");
static int flushbufqtarget = 100;
SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
    "Amount of work to do in flushbufqueues when helping bufdaemon");
static counter_u64_t notbufdflushes;
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, notbufdflushes, CTLFLAG_RD, &notbufdflushes,
    "Number of dirty buffer flushes done by the bufdaemon helpers");
static long barrierwrites;
SYSCTL_LONG(_vfs, OID_AUTO, barrierwrites, CTLFLAG_RW, &barrierwrites, 0,
    "Number of barrier writes");
SYSCTL_INT(_vfs, OID_AUTO, unmapped_buf_allowed, CTLFLAG_RD,
    &unmapped_buf_allowed, 0,
    "Permit the use of the unmapped i/o");
int maxbcachebuf = MAXBCACHEBUF;
SYSCTL_INT(_vfs, OID_AUTO, maxbcachebuf, CTLFLAG_RDTUN, &maxbcachebuf, 0,
    "Maximum size of a buffer cache block");

/*
 * This lock synchronizes access to bd_request.
 */
static struct mtx_padalign __exclusive_cache_line bdlock;

/*
 * This lock protects the runningbufreq and synchronizes runningbufwakeup and
 * waitrunningbufspace().
 */
static struct mtx_padalign __exclusive_cache_line rbreqlock;

/*
 * Lock that protects bdirtywait.
 */
static struct mtx_padalign __exclusive_cache_line bdirtylock;

/*
 * Wakeup point for bufdaemon, as well as indicator of whether it is already
 * active.  Set to 1 when the bufdaemon is already "on" the queue, 0 when it
 * is idling.
 */
static int bd_request;

/*
 * Request for the buf daemon to write more buffers than is indicated by
 * lodirtybuf.  This may be necessary to push out excess dependencies or
 * defragment the address space where a simple count of the number of dirty
 * buffers is insufficient to characterize the demand for flushing them.
 */
static int bd_speedupreq;

/*
 * Synchronization (sleep/wakeup) variable for active buffer space requests.
 * Set when wait starts, cleared prior to wakeup().
 * Used in runningbufwakeup() and waitrunningbufspace().
 */
static int runningbufreq;

/*
 * Synchronization for bwillwrite() waiters.
 */
static int bdirtywait;

/*
 * Definitions for the buffer free lists.
 */
#define QUEUE_NONE	0	/* on no queue */
#define QUEUE_EMPTY	1	/* empty buffer headers */
#define QUEUE_DIRTY	2	/* B_DELWRI buffers */
#define QUEUE_CLEAN	3	/* non-B_DELWRI buffers */
#define QUEUE_SENTINEL	4	/* not an queue index, but mark for sentinel */

/* Maximum number of buffer domains. */
#define	BUF_DOMAINS	8

struct bufdomainset bdlodirty;		/* Domains > lodirty */
struct bufdomainset bdhidirty;		/* Domains > hidirty */

/* Configured number of clean queues. */
static int __read_mostly buf_domains;

BITSET_DEFINE(bufdomainset, BUF_DOMAINS);
struct bufdomain __exclusive_cache_line bdomain[BUF_DOMAINS];
struct bufqueue __exclusive_cache_line bqempty;

/*
 * per-cpu empty buffer cache.
 */
uma_zone_t buf_zone;

/*
 * Single global constant for BUF_WMESG, to avoid getting multiple references.
 * buf_wmesg is referred from macros.
 */
const char *buf_wmesg = BUF_WMESG;

static int
sysctl_runningspace(SYSCTL_HANDLER_ARGS)
{
	long value;
	int error;

	value = *(long *)arg1;
	error = sysctl_handle_long(oidp, &value, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);
	mtx_lock(&rbreqlock);
	if (arg1 == &hirunningspace) {
		if (value < lorunningspace)
			error = EINVAL;
		else
			hirunningspace = value;
	} else {
		KASSERT(arg1 == &lorunningspace,
		    ("%s: unknown arg1", __func__));
		if (value > hirunningspace)
			error = EINVAL;
		else
			lorunningspace = value;
	}
	mtx_unlock(&rbreqlock);
	return (error);
}

static int
sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS)
{
	int error;
	int value;
	int i;

	value = *(int *)arg1;
	error = sysctl_handle_int(oidp, &value, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);
	*(int *)arg1 = value;
	for (i = 0; i < buf_domains; i++)
		*(int *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
		    value / buf_domains;

	return (error);
}

static int
sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS)
{
	long value;
	int error;
	int i;

	value = *(long *)arg1;
	error = sysctl_handle_long(oidp, &value, 0, req);
	if (error != 0 || req->newptr == NULL)
		return (error);
	*(long *)arg1 = value;
	for (i = 0; i < buf_domains; i++)
		*(long *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
		    value / buf_domains;

	return (error);
}

#if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
    defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
static int
sysctl_bufspace(SYSCTL_HANDLER_ARGS)
{
	long lvalue;
	int ivalue;
	int i;

	lvalue = 0;
	for (i = 0; i < buf_domains; i++)
		lvalue += bdomain[i].bd_bufspace;
	if (sizeof(int) == sizeof(long) || req->oldlen >= sizeof(long))
		return (sysctl_handle_long(oidp, &lvalue, 0, req));
	if (lvalue > INT_MAX)
		/* On overflow, still write out a long to trigger ENOMEM. */
		return (sysctl_handle_long(oidp, &lvalue, 0, req));
	ivalue = lvalue;
	return (sysctl_handle_int(oidp, &ivalue, 0, req));
}
#else
static int
sysctl_bufspace(SYSCTL_HANDLER_ARGS)
{
	long lvalue;
	int i;

	lvalue = 0;
	for (i = 0; i < buf_domains; i++)
		lvalue += bdomain[i].bd_bufspace;
	return (sysctl_handle_long(oidp, &lvalue, 0, req));
}
#endif

static int
sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS)
{
	int value;
	int i;

	value = 0;
	for (i = 0; i < buf_domains; i++)
		value += bdomain[i].bd_numdirtybuffers;
	return (sysctl_handle_int(oidp, &value, 0, req));
}

/*
 *	bdirtywakeup:
 *
 *	Wakeup any bwillwrite() waiters.
 */
static void
bdirtywakeup(void)
{
	mtx_lock(&bdirtylock);
	if (bdirtywait) {
		bdirtywait = 0;
		wakeup(&bdirtywait);
	}
	mtx_unlock(&bdirtylock);
}

/*
 *	bd_clear:
 *
 *	Clear a domain from the appropriate bitsets when dirtybuffers
 *	is decremented.
 */
static void
bd_clear(struct bufdomain *bd)
{

	mtx_lock(&bdirtylock);
	if (bd->bd_numdirtybuffers <= bd->bd_lodirtybuffers)
		BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
	if (bd->bd_numdirtybuffers <= bd->bd_hidirtybuffers)
		BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
	mtx_unlock(&bdirtylock);
}

/*
 *	bd_set:
 *
 *	Set a domain in the appropriate bitsets when dirtybuffers
 *	is incremented.
 */
static void
bd_set(struct bufdomain *bd)
{

	mtx_lock(&bdirtylock);
	if (bd->bd_numdirtybuffers > bd->bd_lodirtybuffers)
		BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
	if (bd->bd_numdirtybuffers > bd->bd_hidirtybuffers)
		BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
	mtx_unlock(&bdirtylock);
}

/*
 *	bdirtysub:
 *
 *	Decrement the numdirtybuffers count by one and wakeup any
 *	threads blocked in bwillwrite().
 */
static void
bdirtysub(struct buf *bp)
{
	struct bufdomain *bd;
	int num;

	bd = bufdomain(bp);
	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, -1);
	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
		bdirtywakeup();
	if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
		bd_clear(bd);
}

/*
 *	bdirtyadd:
 *
 *	Increment the numdirtybuffers count by one and wakeup the buf 
 *	daemon if needed.
 */
static void
bdirtyadd(struct buf *bp)
{
	struct bufdomain *bd;
	int num;

	/*
	 * Only do the wakeup once as we cross the boundary.  The
	 * buf daemon will keep running until the condition clears.
	 */
	bd = bufdomain(bp);
	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, 1);
	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
		bd_wakeup();
	if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
		bd_set(bd);
}

/*
 *	bufspace_daemon_wakeup:
 *
 *	Wakeup the daemons responsible for freeing clean bufs.
 */
static void
bufspace_daemon_wakeup(struct bufdomain *bd)
{

	/*
	 * avoid the lock if the daemon is running.
	 */
	if (atomic_fetchadd_int(&bd->bd_running, 1) == 0) {
		BD_RUN_LOCK(bd);
		atomic_store_int(&bd->bd_running, 1);
		wakeup(&bd->bd_running);
		BD_RUN_UNLOCK(bd);
	}
}

/*
 *	bufspace_daemon_wait:
 *
 *	Sleep until the domain falls below a limit or one second passes.
 */
static void
bufspace_daemon_wait(struct bufdomain *bd)
{
	/*
	 * Re-check our limits and sleep.  bd_running must be
	 * cleared prior to checking the limits to avoid missed
	 * wakeups.  The waker will adjust one of bufspace or
	 * freebuffers prior to checking bd_running.
	 */
	BD_RUN_LOCK(bd);
	atomic_store_int(&bd->bd_running, 0);
	if (bd->bd_bufspace < bd->bd_bufspacethresh &&
	    bd->bd_freebuffers > bd->bd_lofreebuffers) {
		msleep(&bd->bd_running, BD_RUN_LOCKPTR(bd), PRIBIO|PDROP,
		    "-", hz);
	} else {
		/* Avoid spurious wakeups while running. */
		atomic_store_int(&bd->bd_running, 1);
		BD_RUN_UNLOCK(bd);
	}
}

/*
 *	bufspace_adjust:
 *
 *	Adjust the reported bufspace for a KVA managed buffer, possibly
 * 	waking any waiters.
 */
static void
bufspace_adjust(struct buf *bp, int bufsize)
{
	struct bufdomain *bd;
	long space;
	int diff;

	KASSERT((bp->b_flags & B_MALLOC) == 0,
	    ("bufspace_adjust: malloc buf %p", bp));
	bd = bufdomain(bp);
	diff = bufsize - bp->b_bufsize;
	if (diff < 0) {
		atomic_subtract_long(&bd->bd_bufspace, -diff);
	} else if (diff > 0) {
		space = atomic_fetchadd_long(&bd->bd_bufspace, diff);
		/* Wake up the daemon on the transition. */
		if (space < bd->bd_bufspacethresh &&
		    space + diff >= bd->bd_bufspacethresh)
			bufspace_daemon_wakeup(bd);
	}
	bp->b_bufsize = bufsize;
}

/*
 *	bufspace_reserve:
 *
 *	Reserve bufspace before calling allocbuf().  metadata has a
 *	different space limit than data.
 */
static int
bufspace_reserve(struct bufdomain *bd, int size, bool metadata)
{
	long limit, new;
	long space;

	if (metadata)
		limit = bd->bd_maxbufspace;
	else
		limit = bd->bd_hibufspace;
	space = atomic_fetchadd_long(&bd->bd_bufspace, size);
	new = space + size;
	if (new > limit) {
		atomic_subtract_long(&bd->bd_bufspace, size);
		return (ENOSPC);
	}

	/* Wake up the daemon on the transition. */
	if (space < bd->bd_bufspacethresh && new >= bd->bd_bufspacethresh)
		bufspace_daemon_wakeup(bd);

	return (0);
}

/*
 *	bufspace_release:
 *
 *	Release reserved bufspace after bufspace_adjust() has consumed it.
 */
static void
bufspace_release(struct bufdomain *bd, int size)
{

	atomic_subtract_long(&bd->bd_bufspace, size);
}

/*
 *	bufspace_wait:
 *
 *	Wait for bufspace, acting as the buf daemon if a locked vnode is
 *	supplied.  bd_wanted must be set prior to polling for space.  The
 *	operation must be re-tried on return.
 */
static void
bufspace_wait(struct bufdomain *bd, struct vnode *vp, int gbflags,
    int slpflag, int slptimeo)
{
	struct thread *td;
	int error, fl, norunbuf;

	if ((gbflags & GB_NOWAIT_BD) != 0)
		return;

	td = curthread;
	BD_LOCK(bd);
	while (bd->bd_wanted) {
		if (vp != NULL && vp->v_type != VCHR &&
		    (td->td_pflags & TDP_BUFNEED) == 0) {
			BD_UNLOCK(bd);
			/*
			 * getblk() is called with a vnode locked, and
			 * some majority of the dirty buffers may as
			 * well belong to the vnode.  Flushing the
			 * buffers there would make a progress that
			 * cannot be achieved by the buf_daemon, that
			 * cannot lock the vnode.
			 */
			norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
			    (td->td_pflags & TDP_NORUNNINGBUF);

			/*
			 * Play bufdaemon.  The getnewbuf() function
			 * may be called while the thread owns lock
			 * for another dirty buffer for the same
			 * vnode, which makes it impossible to use
			 * VOP_FSYNC() there, due to the buffer lock
			 * recursion.
			 */
			td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
			fl = buf_flush(vp, bd, flushbufqtarget);
			td->td_pflags &= norunbuf;
			BD_LOCK(bd);
			if (fl != 0)
				continue;
			if (bd->bd_wanted == 0)
				break;
		}
		error = msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
		    (PRIBIO + 4) | slpflag, "newbuf", slptimeo);
		if (error != 0)
			break;
	}
	BD_UNLOCK(bd);
}


/*
 *	bufspace_daemon:
 *
 *	buffer space management daemon.  Tries to maintain some marginal
 *	amount of free buffer space so that requesting processes neither
 *	block nor work to reclaim buffers.
 */
static void
bufspace_daemon(void *arg)
{
	struct bufdomain *bd;

	EVENTHANDLER_REGISTER(shutdown_pre_sync, kthread_shutdown, curthread,
	    SHUTDOWN_PRI_LAST + 100);

	bd = arg;
	for (;;) {
		kthread_suspend_check();

		/*
		 * Free buffers from the clean queue until we meet our
		 * targets.
		 *
		 * Theory of operation:  The buffer cache is most efficient
		 * when some free buffer headers and space are always
		 * available to getnewbuf().  This daemon attempts to prevent
		 * the excessive blocking and synchronization associated
		 * with shortfall.  It goes through three phases according
		 * demand:
		 *
		 * 1)	The daemon wakes up voluntarily once per-second
		 *	during idle periods when the counters are below
		 *	the wakeup thresholds (bufspacethresh, lofreebuffers).
		 *
		 * 2)	The daemon wakes up as we cross the thresholds
		 *	ahead of any potential blocking.  This may bounce
		 *	slightly according to the rate of consumption and
		 *	release.
		 *
		 * 3)	The daemon and consumers are starved for working
		 *	clean buffers.  This is the 'bufspace' sleep below
		 *	which will inefficiently trade bufs with bqrelse
		 *	until we return to condition 2.
		 */
		while (bd->bd_bufspace > bd->bd_lobufspace ||
		    bd->bd_freebuffers < bd->bd_hifreebuffers) {
			if (buf_recycle(bd, false) != 0) {
				if (bd_flushall(bd))
					continue;
				/*
				 * Speedup dirty if we've run out of clean
				 * buffers.  This is possible in particular
				 * because softdep may held many bufs locked
				 * pending writes to other bufs which are
				 * marked for delayed write, exhausting
				 * clean space until they are written.
				 */
				bd_speedup();
				BD_LOCK(bd);
				if (bd->bd_wanted) {
					msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
					    PRIBIO|PDROP, "bufspace", hz/10);
				} else
					BD_UNLOCK(bd);
			}
			maybe_yield();
		}
		bufspace_daemon_wait(bd);
	}
}

/*
 *	bufmallocadjust:
 *
 *	Adjust the reported bufspace for a malloc managed buffer, possibly
 *	waking any waiters.
 */
static void
bufmallocadjust(struct buf *bp, int bufsize)
{
	int diff;

	KASSERT((bp->b_flags & B_MALLOC) != 0,
	    ("bufmallocadjust: non-malloc buf %p", bp));
	diff = bufsize - bp->b_bufsize;
	if (diff < 0)
		atomic_subtract_long(&bufmallocspace, -diff);
	else
		atomic_add_long(&bufmallocspace, diff);
	bp->b_bufsize = bufsize;
}

/*
 *	runningwakeup:
 *
 *	Wake up processes that are waiting on asynchronous writes to fall
 *	below lorunningspace.
 */
static void
runningwakeup(void)
{

	mtx_lock(&rbreqlock);
	if (runningbufreq) {
		runningbufreq = 0;
		wakeup(&runningbufreq);
	}
	mtx_unlock(&rbreqlock);
}

/*
 *	runningbufwakeup:
 *
 *	Decrement the outstanding write count according.
 */
void
runningbufwakeup(struct buf *bp)
{
	long space, bspace;

	bspace = bp->b_runningbufspace;
	if (bspace == 0)
		return;
	space = atomic_fetchadd_long(&runningbufspace, -bspace);
	KASSERT(space >= bspace, ("runningbufspace underflow %ld %ld",
	    space, bspace));
	bp->b_runningbufspace = 0;
	/*
	 * Only acquire the lock and wakeup on the transition from exceeding
	 * the threshold to falling below it.
	 */
	if (space < lorunningspace)
		return;
	if (space - bspace > lorunningspace)
		return;
	runningwakeup();
}

/*
 *	waitrunningbufspace()
 *
 *	runningbufspace is a measure of the amount of I/O currently
 *	running.  This routine is used in async-write situations to
 *	prevent creating huge backups of pending writes to a device.
 *	Only asynchronous writes are governed by this function.
 *
 *	This does NOT turn an async write into a sync write.  It waits  
 *	for earlier writes to complete and generally returns before the
 *	caller's write has reached the device.
 */
void
waitrunningbufspace(void)
{

	mtx_lock(&rbreqlock);
	while (runningbufspace > hirunningspace) {
		runningbufreq = 1;
		msleep(&runningbufreq, &rbreqlock, PVM, "wdrain", 0);
	}
	mtx_unlock(&rbreqlock);
}


/*
 *	vfs_buf_test_cache:
 *
 *	Called when a buffer is extended.  This function clears the B_CACHE
 *	bit if the newly extended portion of the buffer does not contain
 *	valid data.
 */
static __inline void
vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off,
    vm_offset_t size, vm_page_t m)
{

	VM_OBJECT_ASSERT_LOCKED(m->object);
	if (bp->b_flags & B_CACHE) {
		int base = (foff + off) & PAGE_MASK;
		if (vm_page_is_valid(m, base, size) == 0)
			bp->b_flags &= ~B_CACHE;
	}
}

/* Wake up the buffer daemon if necessary */
static void
bd_wakeup(void)
{

	mtx_lock(&bdlock);
	if (bd_request == 0) {
		bd_request = 1;
		wakeup(&bd_request);
	}
	mtx_unlock(&bdlock);
}

/*
 * Adjust the maxbcachbuf tunable.
 */
static void
maxbcachebuf_adjust(void)
{
	int i;

	/*
	 * maxbcachebuf must be a power of 2 >= MAXBSIZE.
	 */
	i = 2;
	while (i * 2 <= maxbcachebuf)
		i *= 2;
	maxbcachebuf = i;
	if (maxbcachebuf < MAXBSIZE)
		maxbcachebuf = MAXBSIZE;
	if (maxbcachebuf > MAXPHYS)
		maxbcachebuf = MAXPHYS;
	if (bootverbose != 0 && maxbcachebuf != MAXBCACHEBUF)
		printf("maxbcachebuf=%d\n", maxbcachebuf);
}