1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2020 Jeffrey Roberson <jeff@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice unmodified, this list of conditions, and the following
11 *    disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 * $FreeBSD$
28 *
29 */
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/kernel.h>
33#include <sys/libkern.h>
34#include <sys/module.h>
35#include <sys/mutex.h>
36#include <sys/proc.h>
37#include <sys/kthread.h>
38#include <sys/conf.h>
39#include <sys/mbuf.h>
40#include <sys/smp.h>
41#include <sys/smr.h>
42
43#include <vm/uma.h>
44
45#include <machine/stdarg.h>
46
47static uma_zone_t smrs_zone;
48static smr_t smrs_smr;
49
50static int smrs_cpus;
51static int smrs_writers;
52static int smrs_started;
53static int smrs_iterations = 10000000;
54static int smrs_failures = 0;
55static volatile int smrs_completed;
56
57struct smrs {
58	int		generation;
59	volatile u_int	count;
60};
61
62uintptr_t smrs_current;
63
64static void
65smrs_error(struct smrs *smrs, const char *fmt, ...)
66{
67	va_list ap;
68
69	atomic_add_int(&smrs_failures, 1);
70	printf("SMR ERROR: wr_seq %d, rd_seq %d, c_seq %d, generation %d, count %d ",
71	    smrs_smr->c_shared->s_wr.seq, smrs_smr->c_shared->s_rd_seq,
72	    zpcpu_get(smrs_smr)->c_seq, smrs->generation, smrs->count);
73	va_start(ap, fmt);
74	(void)vprintf(fmt, ap);
75	va_end(ap);
76}
77
78static void
79smrs_read(void)
80{
81	struct smrs *cur;
82	int cnt;
83
84	/* Wait for the writer to exit. */
85	while (smrs_completed == 0) {
86		smr_enter(smrs_smr);
87		cur = (void *)atomic_load_acq_ptr(&smrs_current);
88		if (cur->generation == -1)
89			smrs_error(cur, "read early: Use after free!\n");
90		atomic_add_int(&cur->count, 1);
91		DELAY(100);
92		cnt = atomic_fetchadd_int(&cur->count, -1);
93		if (cur->generation == -1)
94			smrs_error(cur, "read late: Use after free!\n");
95		else if (cnt <= 0)
96			smrs_error(cur, "Invalid ref\n");
97		smr_exit(smrs_smr);
98		maybe_yield();
99	}
100}
101
102static void
103smrs_write(void)
104{
105	struct smrs *cur;
106	int i;
107
108	for (i = 0; i < smrs_iterations; i++) {
109		cur = uma_zalloc_smr(smrs_zone, M_WAITOK);
110		atomic_thread_fence_rel();
111		cur = (void *)atomic_swap_ptr(&smrs_current, (uintptr_t)cur);
112		uma_zfree_smr(smrs_zone, cur);
113	}
114}
115
116static void
117smrs_thread(void *arg)
118{
119	int rw = (intptr_t)arg;
120
121	if (rw < smrs_writers)
122		smrs_write();
123	else
124		smrs_read();
125	atomic_add_int(&smrs_completed, 1);
126}
127
128static void
129smrs_start(void)
130{
131	struct smrs *cur;
132	int i;
133
134	smrs_cpus = mp_ncpus;
135	if (mp_ncpus > 3)
136		smrs_writers = 2;
137	else
138		smrs_writers = 1;
139	smrs_started = smrs_cpus;
140	smrs_completed = 0;
141	atomic_store_rel_ptr(&smrs_current,
142	    (uintptr_t)uma_zalloc_smr(smrs_zone, M_WAITOK));
143	for (i = 0; i < smrs_started; i++)
144		kthread_add((void (*)(void *))smrs_thread,
145		    (void *)(intptr_t)i, curproc, NULL, 0, 0, "smrs-%d", i);
146
147	while (smrs_completed != smrs_started)
148		pause("prf", hz/2);
149
150	cur = (void *)smrs_current;
151	smrs_current = (uintptr_t)NULL;
152	uma_zfree_smr(smrs_zone, cur);
153
154	printf("Completed %d loops with %d failures\n",
155	    smrs_iterations, smrs_failures);
156}
157
158static int
159smrs_ctor(void *mem, int size, void *arg, int flags)
160{
161	static int smr_generation = 0;
162	struct smrs *smrs = mem;
163
164	if (smrs->generation != -1 && smrs->generation != 0)
165		smrs_error(smrs, "ctor: Invalid smr generation on ctor\n");
166	else if (smrs->count != 0)
167		smrs_error(smrs, "ctor: Invalid count\n");
168	smrs->generation = ++smr_generation;
169
170	return (0);
171}
172
173
174static void
175smrs_dtor(void *mem, int size, void *arg)
176{
177	struct smrs *smrs = mem;
178
179	if (smrs->generation == -1)
180		smrs_error(smrs, "dtor: Invalid generation");
181	smrs->generation = -1;
182	if (smrs->count != 0)
183		smrs_error(smrs, "dtor: Invalid count\n");
184}
185
186
187static void
188smrs_init(void)
189{
190
191	smrs_zone = uma_zcreate("smrs", sizeof(struct smrs),
192	    smrs_ctor,  smrs_dtor, NULL, NULL, UMA_ALIGN_PTR,
193	    UMA_ZONE_SMR | UMA_ZONE_ZINIT);
194        smrs_smr = uma_zone_get_smr(smrs_zone);
195}
196
197static void
198smrs_fini(void)
199{
200
201	uma_zdestroy(smrs_zone);
202}
203
204static int
205smrs_modevent(module_t mod, int what, void *arg)
206{
207
208	switch (what) {
209	case MOD_LOAD:
210		smrs_init();
211		smrs_start();
212		break;
213	case MOD_UNLOAD:
214		smrs_fini();
215		break;
216	default:
217		break;
218	}
219	return (0);
220}
221
222moduledata_t smrs_meta = {
223	"smrstress",
224	smrs_modevent,
225	NULL
226};
227DECLARE_MODULE(smrstress, smrs_meta, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
228MODULE_VERSION(smrstress, 1);
229