libmultipath/checkers/tur: Fix races on tur_checker_context.thread
[multipath-tools/.git] / libmultipath / checkers / tur.c
1 /*
2  * Some code borrowed from sg-utils.
3  *
4  * Copyright (c) 2004 Christophe Varoqui
5  */
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <string.h>
9 #include <sys/types.h>
10 #include <sys/stat.h>
11 #include <unistd.h>
12 #include <fcntl.h>
13 #include <sys/ioctl.h>
14 #include <errno.h>
15 #include <sys/time.h>
16 #include <pthread.h>
17
18 #include "checkers.h"
19
20 #include "../libmultipath/debug.h"
21 #include "../libmultipath/sg_include.h"
22 #include "../libmultipath/uevent.h"
23 #include "../libmultipath/time-util.h"
24 #include "../libmultipath/util.h"
25
26 #define TUR_CMD_LEN 6
27 #define HEAVY_CHECK_COUNT       10
28
29 #define MSG_TUR_UP      "tur checker reports path is up"
30 #define MSG_TUR_DOWN    "tur checker reports path is down"
31 #define MSG_TUR_GHOST   "tur checker reports path is in standby state"
32 #define MSG_TUR_RUNNING "tur checker still running"
33 #define MSG_TUR_TIMEOUT "tur checker timed out"
34 #define MSG_TUR_FAILED  "tur checker failed to initialize"
35
36 struct tur_checker_context {
37         dev_t devt;
38         int state;
39         int running;
40         int fd;
41         unsigned int timeout;
42         time_t time;
43         pthread_t thread;
44         pthread_mutex_t lock;
45         pthread_cond_t active;
46         pthread_spinlock_t hldr_lock;
47         int holders;
48         char message[CHECKER_MSG_LEN];
49 };
50
51 static const char *tur_devt(char *devt_buf, int size,
52                             struct tur_checker_context *ct)
53 {
54         dev_t devt;
55
56         pthread_mutex_lock(&ct->lock);
57         devt = ct->devt;
58         pthread_mutex_unlock(&ct->lock);
59
60         snprintf(devt_buf, size, "%d:%d", major(devt), minor(devt));
61         return devt_buf;
62 }
63
64 int libcheck_init (struct checker * c)
65 {
66         struct tur_checker_context *ct;
67
68         ct = malloc(sizeof(struct tur_checker_context));
69         if (!ct)
70                 return 1;
71         memset(ct, 0, sizeof(struct tur_checker_context));
72
73         ct->state = PATH_UNCHECKED;
74         ct->fd = -1;
75         ct->holders = 1;
76         pthread_cond_init_mono(&ct->active);
77         pthread_mutex_init(&ct->lock, NULL);
78         pthread_spin_init(&ct->hldr_lock, PTHREAD_PROCESS_PRIVATE);
79         c->context = ct;
80
81         return 0;
82 }
83
84 static void cleanup_context(struct tur_checker_context *ct)
85 {
86         pthread_mutex_destroy(&ct->lock);
87         pthread_cond_destroy(&ct->active);
88         pthread_spin_destroy(&ct->hldr_lock);
89         free(ct);
90 }
91
92 void libcheck_free (struct checker * c)
93 {
94         if (c->context) {
95                 struct tur_checker_context *ct = c->context;
96                 int holders;
97                 pthread_t thread;
98
99                 pthread_spin_lock(&ct->hldr_lock);
100                 ct->holders--;
101                 holders = ct->holders;
102                 thread = ct->thread;
103                 pthread_spin_unlock(&ct->hldr_lock);
104                 if (holders)
105                         pthread_cancel(thread);
106                 else
107                         cleanup_context(ct);
108                 c->context = NULL;
109         }
110         return;
111 }
112
113 void libcheck_repair (struct checker * c)
114 {
115         return;
116 }
117
118 #define TUR_MSG(fmt, args...)                                   \
119         do {                                                    \
120                 char msg[CHECKER_MSG_LEN];                      \
121                                                                 \
122                 snprintf(msg, sizeof(msg), fmt, ##args);        \
123                 copy_message(cb_arg, msg);                      \
124         } while (0)
125
126 static int
127 tur_check(int fd, unsigned int timeout,
128           void (*copy_message)(void *, const char *), void *cb_arg)
129 {
130         struct sg_io_hdr io_hdr;
131         unsigned char turCmdBlk[TUR_CMD_LEN] = { 0x00, 0, 0, 0, 0, 0 };
132         unsigned char sense_buffer[32];
133         int retry_tur = 5;
134
135 retry:
136         memset(&io_hdr, 0, sizeof (struct sg_io_hdr));
137         memset(&sense_buffer, 0, 32);
138         io_hdr.interface_id = 'S';
139         io_hdr.cmd_len = sizeof (turCmdBlk);
140         io_hdr.mx_sb_len = sizeof (sense_buffer);
141         io_hdr.dxfer_direction = SG_DXFER_NONE;
142         io_hdr.cmdp = turCmdBlk;
143         io_hdr.sbp = sense_buffer;
144         io_hdr.timeout = timeout * 1000;
145         io_hdr.pack_id = 0;
146         if (ioctl(fd, SG_IO, &io_hdr) < 0) {
147                 TUR_MSG(MSG_TUR_DOWN);
148                 return PATH_DOWN;
149         }
150         if ((io_hdr.status & 0x7e) == 0x18) {
151                 /*
152                  * SCSI-3 arrays might return
153                  * reservation conflict on TUR
154                  */
155                 TUR_MSG(MSG_TUR_UP);
156                 return PATH_UP;
157         }
158         if (io_hdr.info & SG_INFO_OK_MASK) {
159                 int key = 0, asc, ascq;
160
161                 switch (io_hdr.host_status) {
162                 case DID_OK:
163                 case DID_NO_CONNECT:
164                 case DID_BAD_TARGET:
165                 case DID_ABORT:
166                 case DID_TRANSPORT_FAILFAST:
167                         break;
168                 default:
169                         /* Driver error, retry */
170                         if (--retry_tur)
171                                 goto retry;
172                         break;
173                 }
174                 if (io_hdr.sb_len_wr > 3) {
175                         if (io_hdr.sbp[0] == 0x72 || io_hdr.sbp[0] == 0x73) {
176                                 key = io_hdr.sbp[1] & 0x0f;
177                                 asc = io_hdr.sbp[2];
178                                 ascq = io_hdr.sbp[3];
179                         } else if (io_hdr.sb_len_wr > 13 &&
180                                    ((io_hdr.sbp[0] & 0x7f) == 0x70 ||
181                                     (io_hdr.sbp[0] & 0x7f) == 0x71)) {
182                                 key = io_hdr.sbp[2] & 0x0f;
183                                 asc = io_hdr.sbp[12];
184                                 ascq = io_hdr.sbp[13];
185                         }
186                 }
187                 if (key == 0x6) {
188                         /* Unit Attention, retry */
189                         if (--retry_tur)
190                                 goto retry;
191                 }
192                 else if (key == 0x2) {
193                         /* Not Ready */
194                         /* Note: Other ALUA states are either UP or DOWN */
195                         if( asc == 0x04 && ascq == 0x0b){
196                                 /*
197                                  * LOGICAL UNIT NOT ACCESSIBLE,
198                                  * TARGET PORT IN STANDBY STATE
199                                  */
200                                 TUR_MSG(MSG_TUR_GHOST);
201                                 return PATH_GHOST;
202                         }
203                 }
204                 TUR_MSG(MSG_TUR_DOWN);
205                 return PATH_DOWN;
206         }
207         TUR_MSG(MSG_TUR_UP);
208         return PATH_UP;
209 }
210
211 #define tur_thread_cleanup_push(ct) pthread_cleanup_push(cleanup_func, ct)
212 #define tur_thread_cleanup_pop(ct) pthread_cleanup_pop(1)
213
214 static void cleanup_func(void *data)
215 {
216         int holders;
217         struct tur_checker_context *ct = data;
218         pthread_spin_lock(&ct->hldr_lock);
219         ct->holders--;
220         holders = ct->holders;
221         ct->thread = 0;
222         pthread_spin_unlock(&ct->hldr_lock);
223         if (!holders)
224                 cleanup_context(ct);
225 }
226
227 static int tur_running(struct tur_checker_context *ct)
228 {
229         pthread_t thread;
230
231         pthread_spin_lock(&ct->hldr_lock);
232         thread = ct->thread;
233         pthread_spin_unlock(&ct->hldr_lock);
234
235         return thread != 0;
236 }
237
238 static void copy_msg_to_tcc(void *ct_p, const char *msg)
239 {
240         struct tur_checker_context *ct = ct_p;
241
242         pthread_mutex_lock(&ct->lock);
243         strlcpy(ct->message, msg, sizeof(ct->message));
244         pthread_mutex_unlock(&ct->lock);
245 }
246
247 static void *tur_thread(void *ctx)
248 {
249         struct tur_checker_context *ct = ctx;
250         int state;
251         char devt[32];
252
253         condlog(3, "%s: tur checker starting up",
254                 tur_devt(devt, sizeof(devt), ct));
255
256         /* This thread can be canceled, so setup clean up */
257         tur_thread_cleanup_push(ct);
258
259         /* TUR checker start up */
260         pthread_mutex_lock(&ct->lock);
261         ct->state = PATH_PENDING;
262         ct->message[0] = '\0';
263         pthread_mutex_unlock(&ct->lock);
264
265         state = tur_check(ct->fd, ct->timeout, copy_msg_to_tcc, ct->message);
266
267         /* TUR checker done */
268         pthread_mutex_lock(&ct->lock);
269         ct->state = state;
270         pthread_cond_signal(&ct->active);
271         pthread_mutex_unlock(&ct->lock);
272
273         condlog(3, "%s: tur checker finished, state %s",
274                 tur_devt(devt, sizeof(devt), ct), checker_state_name(state));
275         tur_thread_cleanup_pop(ct);
276
277         return ((void *)0);
278 }
279
280
281 static void tur_timeout(struct timespec *tsp)
282 {
283         clock_gettime(CLOCK_MONOTONIC, tsp);
284         tsp->tv_nsec += 1000 * 1000; /* 1 millisecond */
285         normalize_timespec(tsp);
286 }
287
288 static void tur_set_async_timeout(struct checker *c)
289 {
290         struct tur_checker_context *ct = c->context;
291         struct timespec now;
292
293         clock_gettime(CLOCK_MONOTONIC, &now);
294         ct->time = now.tv_sec + c->timeout;
295 }
296
297 static int tur_check_async_timeout(struct checker *c)
298 {
299         struct tur_checker_context *ct = c->context;
300         struct timespec now;
301
302         clock_gettime(CLOCK_MONOTONIC, &now);
303         return (now.tv_sec > ct->time);
304 }
305
306 static void copy_msg_to_checker(void *c_p, const char *msg)
307 {
308         struct checker *c = c_p;
309
310         strlcpy(c->message, msg, sizeof(c->message));
311 }
312
313 extern int
314 libcheck_check (struct checker * c)
315 {
316         struct tur_checker_context *ct = c->context;
317         struct timespec tsp;
318         struct stat sb;
319         pthread_attr_t attr;
320         int tur_status, r;
321         char devt[32];
322
323
324         if (!ct)
325                 return PATH_UNCHECKED;
326
327         if (fstat(c->fd, &sb) == 0) {
328                 pthread_mutex_lock(&ct->lock);
329                 ct->devt = sb.st_rdev;
330                 pthread_mutex_unlock(&ct->lock);
331         }
332
333         if (c->sync)
334                 return tur_check(c->fd, c->timeout, copy_msg_to_checker, c);
335
336         /*
337          * Async mode
338          */
339         r = pthread_mutex_lock(&ct->lock);
340         if (r != 0) {
341                 condlog(2, "%s: tur mutex lock failed with %d",
342                         tur_devt(devt, sizeof(devt), ct), r);
343                 MSG(c, MSG_TUR_FAILED);
344                 return PATH_WILD;
345         }
346
347         if (ct->running) {
348                 /*
349                  * Check if TUR checker is still running. Hold hldr_lock
350                  * around the pthread_cancel() call to avoid that
351                  * pthread_cancel() gets called after the (detached) TUR
352                  * thread has exited.
353                  */
354                 pthread_spin_lock(&ct->hldr_lock);
355                 if (ct->thread) {
356                         if (tur_check_async_timeout(c)) {
357                                 condlog(3, "%s: tur checker timeout",
358                                         tur_devt(devt, sizeof(devt), ct));
359                                 pthread_cancel(ct->thread);
360                                 ct->running = 0;
361                                 MSG(c, MSG_TUR_TIMEOUT);
362                                 tur_status = PATH_TIMEOUT;
363                         } else {
364                                 condlog(3, "%s: tur checker not finished",
365                                         tur_devt(devt, sizeof(devt), ct));
366                                 ct->running++;
367                                 tur_status = PATH_PENDING;
368                         }
369                 } else {
370                         /* TUR checker done */
371                         ct->running = 0;
372                         tur_status = ct->state;
373                         strlcpy(c->message, ct->message, sizeof(c->message));
374                 }
375                 pthread_spin_unlock(&ct->hldr_lock);
376                 pthread_mutex_unlock(&ct->lock);
377         } else {
378                 if (tur_running(ct)) {
379                         /* pthread cancel failed. continue in sync mode */
380                         pthread_mutex_unlock(&ct->lock);
381                         condlog(3, "%s: tur thread not responding",
382                                 tur_devt(devt, sizeof(devt), ct));
383                         return PATH_TIMEOUT;
384                 }
385                 /* Start new TUR checker */
386                 ct->state = PATH_UNCHECKED;
387                 ct->fd = c->fd;
388                 ct->timeout = c->timeout;
389                 pthread_spin_lock(&ct->hldr_lock);
390                 ct->holders++;
391                 pthread_spin_unlock(&ct->hldr_lock);
392                 tur_set_async_timeout(c);
393                 setup_thread_attr(&attr, 32 * 1024, 1);
394                 r = pthread_create(&ct->thread, &attr, tur_thread, ct);
395                 pthread_attr_destroy(&attr);
396                 if (r) {
397                         pthread_spin_lock(&ct->hldr_lock);
398                         ct->holders--;
399                         pthread_spin_unlock(&ct->hldr_lock);
400                         pthread_mutex_unlock(&ct->lock);
401                         ct->thread = 0;
402                         condlog(3, "%s: failed to start tur thread, using"
403                                 " sync mode", tur_devt(devt, sizeof(devt), ct));
404                         return tur_check(c->fd, c->timeout,
405                                          copy_msg_to_checker, c);
406                 }
407                 tur_timeout(&tsp);
408                 r = pthread_cond_timedwait(&ct->active, &ct->lock, &tsp);
409                 tur_status = ct->state;
410                 strlcpy(c->message, ct->message, sizeof(c->message));
411                 pthread_mutex_unlock(&ct->lock);
412                 if (tur_running(ct) &&
413                     (tur_status == PATH_PENDING || tur_status == PATH_UNCHECKED)) {
414                         condlog(3, "%s: tur checker still running",
415                                 tur_devt(devt, sizeof(devt), ct));
416                         ct->running = 1;
417                         tur_status = PATH_PENDING;
418                 }
419         }
420
421         return tur_status;
422 }