6 files changed, 694 insertions, 0 deletions

diff --git a/tests/runner.c b/tests/runner.c
index db62246..335d83d 100644
--- a/tests/runner.c
+++ b/tests/runner.c
@@ -23,6 +23,9 @@ static const struct module modules[] = {{"scan_urlencoded_query", test_scan_urle
                                         {"ot_buf", test_buf},
                                         {"ot_ip", test_ip},
                                         {"ot_socket", test_socket},
+                                        {"ot_mutex", test_mutex},
+                                        {"ot_batch", test_batch},
+                                        {"ot_fdstate", test_fdstate},
                                         {"ot_accesslist", test_accesslist},
                                         {NULL, NULL}};
 
diff --git a/tests/stubs.c b/tests/stubs.c
index ce93071..09a7312 100644
--- a/tests/stubs.c
+++ b/tests/stubs.c
@@ -18,3 +18,12 @@ void free_peerlist(ot_peerlist *peer_list) { (void)peer_list; }
    neither of which any test drives. */
 time_t       g_now_seconds;
 volatile int g_opentracker_running;
+
+/* ot_mutex.c wakes the main loop through the self pipe when a result is
+   ready. The queue tests never get that far. */
+int  g_self_pipe[2] = {-1, -1};
+long io_trywrite(long d, const char *buf, long len) {
+  (void)d;
+  (void)buf;
+  return len;
+}
diff --git a/tests/t_batch.c b/tests/t_batch.c
new file mode 100644
index 0000000..02743f1
--- /dev/null
+++ b/tests/t_batch.c
@@ -0,0 +1,348 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $id$ */
+
+/* System */
+#include <fcntl.h>
+#include <poll.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+/* Opentracker */
+#include "ot_batch.h"
+#include "test.h"
+
+/* The batch writes to a real socket pair, because what is worth pinning down
+   is what it does when the kernel takes only part of what it was offered. */
+
+static char *dup_of(const char *text) {
+  size_t len  = strlen(text);
+  char  *copy = malloc(len);
+
+  memcpy(copy, text, len);
+  return copy;
+}
+
+static void drain(int s, char *into, size_t len) {
+  size_t got = 0;
+
+  while (got < len) {
+    ssize_t r = read(s, into + got, len - got);
+
+    if (r <= 0)
+      break;
+    got += (size_t)r;
+  }
+  into[got] = 0;
+}
+
+static void case_queue_accounting(void) {
+  ot_batch batch;
+
+  memset(&batch, 0, sizeof(batch));
+
+  test_case("a zeroed batch is empty and sends nothing");
+  CHECK_INT(batch_pending(&batch), 0);
+  CHECK_INT(batch_send(-1, &batch), BATCH_DONE);
+
+  test_case("anything queued makes it pending");
+  CHECK_INT(batch_add(&batch, dup_of("hello "), 6), 0);
+  CHECK_INT(batch_add_copy(&batch, "world", 5), 0);
+  CHECK_INT(batch_pending(&batch), 1);
+  CHECK_INT(batch.count, 2);
+
+  batch_reset(&batch);
+  test_case("and a reset empties it");
+  CHECK_INT(batch_pending(&batch), 0);
+  CHECK_PTR(batch.entries, NULL);
+}
+
+static void case_send_in_one_go(void) {
+  ot_batch batch;
+  int      sv[2];
+  char     got[32];
+
+  memset(&batch, 0, sizeof(batch));
+  CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
+
+  test_case("several buffers arrive as one stream, in order");
+  CHECK_INT(batch_add(&batch, dup_of("opentracker"), 11), 0);
+  CHECK_INT(batch_add_copy(&batch, " says ", 6), 0);
+  CHECK_INT(batch_add(&batch, dup_of("hello"), 5), 0);
+  CHECK_INT(batch_send(sv[0], &batch), 22);
+  CHECK_INT(batch_pending(&batch), 0);
+
+  drain(sv[1], got, 22);
+  CHECK_STR(got, "opentracker says hello");
+
+  test_case("and an empty batch reports it has nothing left");
+  CHECK_INT(batch_send(sv[0], &batch), BATCH_DONE);
+
+  batch_reset(&batch);
+  close(sv[0]);
+  close(sv[1]);
+}
+
+/* Send everything the batch holds, reading the other end as we go so the
+   socket never stays full, and return what arrived. */
+static size_t pump(int writer, int reader, ot_batch *batch, char *into, size_t room) {
+  size_t got = 0;
+
+  while (batch_pending(batch) || got < room) {
+    ssize_t r;
+
+    if (batch_pending(batch)) {
+      ssize_t res = batch_send(writer, batch);
+
+      if (res == BATCH_FAILED)
+        break;
+      if (res > 0)
+        continue;
+    }
+
+    r = read(reader, into + got, room - got);
+    if (r <= 0)
+      break;
+    got += (size_t)r;
+  }
+  return got;
+}
+
+static void case_partial_write_resumes(void) {
+  const size_t size = 200000;
+  ot_batch     batch;
+  int          sv[2], sndbuf = 4096;
+  char        *sent, *got;
+  size_t       i, arrived;
+
+  memset(&batch, 0, sizeof(batch));
+  CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
+  setsockopt(sv[0], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf));
+  fcntl(sv[0], F_SETFL, fcntl(sv[0], F_GETFL, 0) | O_NONBLOCK);
+
+  sent = malloc(size);
+  got  = malloc(size);
+  for (i = 0; i < size; ++i)
+    sent[i] = (char)(i % 251);
+
+  test_case("a buffer the kernel takes in pieces arrives whole and in order");
+  CHECK_INT(batch_add(&batch, memcpy(malloc(size), sent, size), size), 0);
+  arrived = pump(sv[0], sv[1], &batch, got, size);
+  CHECK_INT(arrived, size);
+  CHECK_INT(batch_pending(&batch), 0);
+  CHECK_MEM(got, sent, size);
+
+  free(sent);
+  free(got);
+  batch_reset(&batch);
+  close(sv[0]);
+  close(sv[1]);
+}
+
+/* writev refuses more than IOV_MAX entries at once, so a long batch has to be
+   sent across several calls. */
+static void case_more_entries_than_one_writev(void) {
+  const size_t entries = 3000, each = 4;
+  ot_batch     batch;
+  int          sv[2], sndbuf = 4096;
+  char        *sent, *got;
+  size_t       i, arrived, refused = 0;
+
+  memset(&batch, 0, sizeof(batch));
+  CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
+  setsockopt(sv[0], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf));
+  fcntl(sv[0], F_SETFL, fcntl(sv[0], F_GETFL, 0) | O_NONBLOCK);
+
+  sent = malloc(entries * each);
+  got  = malloc(entries * each);
+
+  test_case("a batch of more entries than writev accepts still arrives whole");
+  for (i = 0; i < entries; ++i) {
+    char piece[4];
+
+    piece[0] = (char)(i), piece[1] = (char)(i >> 8), piece[2] = 'x', piece[3] = 'y';
+    memcpy(sent + i * each, piece, each);
+    if (batch_add(&batch, memcpy(malloc(each), piece, each), each))
+      ++refused;
+  }
+  CHECK_INT(refused, 0);
+  CHECK_INT(batch.count, entries);
+
+  arrived = pump(sv[0], sv[1], &batch, got, entries * each);
+  CHECK_INT(arrived, entries * each);
+  CHECK_MEM(got, sent, entries * each);
+
+  free(sent);
+  free(got);
+  batch_reset(&batch);
+  close(sv[0]);
+  close(sv[1]);
+}
+
+static void case_reset_frees_the_unsent(void) {
+  ot_batch batch;
+  int      sv[2];
+
+  memset(&batch, 0, sizeof(batch));
+  CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
+
+  test_case("a batch torn down mid flight keeps no bytes outstanding");
+  CHECK_INT(batch_add(&batch, dup_of("first"), 5), 0);
+  CHECK_INT(batch_add(&batch, dup_of("second"), 6), 0);
+  CHECK_INT(batch_send(sv[0], &batch), 11);
+  CHECK_INT(batch_add(&batch, dup_of("never sent"), 10), 0);
+  CHECK_INT(batch_pending(&batch), 1);
+
+  /* Whether the unsent buffer is freed is for the sanitizer to say */
+  batch_reset(&batch);
+  CHECK_INT(batch_pending(&batch), 0);
+  CHECK_INT(batch.count, 0);
+
+  close(sv[0]);
+  close(sv[1]);
+}
+
+static void case_closed_socket_is_fatal(void) {
+  ot_batch batch;
+  int      sv[2];
+
+  memset(&batch, 0, sizeof(batch));
+  CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
+  close(sv[1]);
+
+  test_case("writing to a socket whose peer is gone reports a dead connection");
+  CHECK_INT(batch_add(&batch, dup_of("nobody is listening"), 19), 0);
+  CHECK_INT(batch_send(sv[0], &batch), BATCH_FAILED);
+
+  batch_reset(&batch);
+  close(sv[0]);
+}
+
+static void case_vector_is_taken_whole(void) {
+  ot_batch      batch;
+  struct iovec *vector = malloc(3 * sizeof(struct iovec));
+  int           sv[2];
+  char          got[32];
+
+  memset(&batch, 0, sizeof(batch));
+  CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
+
+  test_case("a whole vector is queued in one go, in order");
+  vector[0].iov_base = dup_of("one "), vector[0].iov_len = 4;
+  vector[1].iov_base = dup_of("two "), vector[1].iov_len = 4;
+  vector[2].iov_base = dup_of("three"), vector[2].iov_len = 5;
+  CHECK_INT(batch_add_vector(&batch, vector, 3), 0);
+  free(vector);
+  CHECK_INT(batch.count, 3);
+  CHECK_INT(batch_send(sv[0], &batch), 13);
+
+  drain(sv[1], got, 13);
+  CHECK_STR(got, "one two three");
+
+  batch_reset(&batch);
+  close(sv[0]);
+  close(sv[1]);
+}
+
+/* The entry array starts small and doubles, so crossing that boundary one
+   buffer at a time is worth its own case. */
+static void case_growing_one_entry_at_a_time(void) {
+  const size_t entries = 100;
+  ot_batch     batch;
+  int          sv[2];
+  char         sent[400], got[401];
+  size_t       i, refused = 0;
+
+  memset(&batch, 0, sizeof(batch));
+  CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
+
+  test_case("a batch grown one buffer at a time keeps them all, in order");
+  for (i = 0; i < entries; ++i) {
+    char piece[4];
+
+    piece[0] = (char)('a' + i % 26), piece[1] = (char)('0' + i % 10);
+    piece[2] = (char)('A' + i % 26), piece[3] = '.';
+    memcpy(sent + i * 4, piece, 4);
+    if (batch_add(&batch, memcpy(malloc(4), piece, 4), 4))
+      ++refused;
+  }
+  CHECK_INT(refused, 0);
+  CHECK_INT(batch.count, entries);
+  CHECK_INT(batch_send(sv[0], &batch), (ssize_t)(entries * 4));
+
+  drain(sv[1], got, entries * 4);
+  CHECK_MEM(got, sent, entries * 4);
+
+  batch_reset(&batch);
+  close(sv[0]);
+  close(sv[1]);
+}
+
+/* Appending while earlier entries are still on their way is what a chunked
+   transfer does. Rather than depend on when a kernel decides to block, this
+   takes entries off the front the way a completed send would and checks that
+   the array reuses the space instead of growing. */
+static void case_append_while_sending(void) {
+  const size_t per_round = 12, each = 8;
+  ot_batch     batch;
+  int          sv[2];
+  char         expected[16], got[17];
+  size_t       i, round, refused = 0, space_after_first = 0;
+
+  memset(&batch, 0, sizeof(batch));
+  CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
+
+  test_case("an array whose front has gone out is reused, not grown");
+  for (round = 0; round < 8; ++round) {
+    for (i = 0; i < per_round; ++i) {
+      /* Distinct per entry, so a move from the wrong index shows up */
+      char *piece = malloc(each);
+
+      memset(piece, (int)('a' + (round * per_round + i) % 26), each);
+      piece[0] = (char)('0' + i);
+      piece[1] = (char)('0' + round);
+      if (batch_add(&batch, piece, each))
+        ++refused;
+    }
+
+    /* All but the last entry have left and been freed */
+    while (batch.count - batch.first > 1)
+      free(batch.entries[batch.first++].iov_base);
+
+    if (!round)
+      space_after_first = batch.space;
+  }
+  CHECK_INT(refused, 0);
+  CHECK_INT(batch.space, space_after_first);
+
+  test_case("and what is still queued survives the move, in order");
+  CHECK_INT(batch.count - batch.first, 1);
+  CHECK_INT(batch_add(&batch, memset(malloc(each), 'Z', each), each), 0);
+  CHECK_INT(batch_send(sv[0], &batch), (ssize_t)(2 * each));
+
+  memset(expected, (int)('a' + (7 * per_round + per_round - 1) % 26), each);
+  expected[0] = (char)('0' + per_round - 1);
+  expected[1] = '7';
+  memset(expected + each, 'Z', each);
+  drain(sv[1], got, 2 * each);
+  CHECK_MEM(got, expected, 2 * each);
+
+  batch_reset(&batch);
+  close(sv[0]);
+  close(sv[1]);
+}
+
+void test_batch(void) {
+  case_queue_accounting();
+  case_send_in_one_go();
+  case_vector_is_taken_whole();
+  case_growing_one_entry_at_a_time();
+  case_append_while_sending();
+  case_partial_write_resumes();
+  case_more_entries_than_one_writev();
+  case_reset_frees_the_unsent();
+  case_closed_socket_is_fatal();
+}
diff --git a/tests/t_fdstate.c b/tests/t_fdstate.c
new file mode 100644
index 0000000..bf8724a
--- /dev/null
+++ b/tests/t_fdstate.c
@@ -0,0 +1,224 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $id$ */
+
+/* System */
+#include <pthread.h>
+#include <stdatomic.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Opentracker */
+#include "ot_fdstate.h"
+#include "test.h"
+
+/* ------------------------------------------------------- one thread first */
+
+static void case_open_claim_release(void) {
+  char     cookie[4];
+  uint32_t generation;
+
+  test_case("a descriptor taken into the table can be claimed and gives back its cookie");
+  generation = fdstate_open(7, cookie, 0);
+  CHECK_INT(fdstate_claim(7, generation), 1);
+  CHECK_PTR(fdstate_cookie(7), cookie);
+  fdstate_release(7);
+
+  test_case("and can be claimed again once released");
+  CHECK_INT(fdstate_claim(7, generation), 1);
+  fdstate_release(7);
+  fdstate_close(7);
+}
+
+static void case_one_holder_at_a_time(void) {
+  uint32_t generation = fdstate_open(8, NULL, 0);
+
+  test_case("a descriptor somebody holds cannot be claimed again");
+  CHECK_INT(fdstate_claim(8, generation), 1);
+  CHECK_INT(fdstate_claim(8, generation), 0);
+  fdstate_release(8);
+  CHECK_INT(fdstate_claim(8, generation), 1);
+  fdstate_release(8);
+  fdstate_close(8);
+}
+
+static void case_stale_generation_is_refused(void) {
+  char     first[4], second[4];
+  uint32_t old, new;
+
+  test_case("a caller quoting the generation of a closed descriptor is turned away");
+  old = fdstate_open(9, first, 0);
+  fdstate_close(9);
+  new = fdstate_open(9, second, 0);
+  CHECK(old != new);
+  CHECK_INT(fdstate_claim(9, old), 0);
+
+  test_case("while the one quoting the new generation gets the new cookie");
+  CHECK_INT(fdstate_claim(9, new), 1);
+  CHECK_PTR(fdstate_cookie(9), second);
+  fdstate_release(9);
+  fdstate_close(9);
+}
+
+static void case_expiry(void) {
+  uint32_t generation = fdstate_open(10, NULL, 100);
+
+  test_case("a descriptor past its expiry is collected");
+  CHECK_INT(fdstate_claim_expired(10, 99), 0);
+  CHECK_INT(fdstate_claim_expired(10, 100), 1);
+  fdstate_close(10);
+
+  test_case("one that never expires is never collected");
+  generation = fdstate_open(10, NULL, 0);
+  CHECK_INT(fdstate_claim_expired(10, 1 << 30), 0);
+
+  test_case("and one somebody is working on is left alone");
+  fdstate_set_expiry(10, 1);
+  CHECK_INT(fdstate_claim(10, generation), 1);
+  CHECK_INT(fdstate_claim_expired(10, 1 << 30), 0);
+  fdstate_release(10);
+  CHECK_INT(fdstate_claim_expired(10, 1 << 30), 1);
+  fdstate_close(10);
+}
+
+static void case_out_of_range(void) {
+  test_case("a descriptor outside the table is refused rather than trusted");
+  CHECK_INT(fdstate_claim(-1, 0), 0);
+  CHECK_INT(fdstate_claim((int)fdstate_slots(), 0), 0);
+  CHECK_INT(fdstate_claim_expired(-1, 0), 0);
+  CHECK_PTR(fdstate_cookie(-1), NULL);
+}
+
+/* --------------------------------------------------------- then in anger */
+
+/* The race this is about only exists while a descriptor is being recycled,
+   so recycling is driven by work done rather than by a clock: every slot is
+   given up after RECYCLE_AFTER claims, whatever the machine's speed. */
+
+#define STRESS_SLOTS   64
+#define RECYCLE_AFTER  32
+#define STRESS_CLAIMS  200000
+#define STRESS_GIVEUP  (STRESS_CLAIMS * 100)
+
+static _Atomic long     stress_claims, stress_recycles, stress_violations, stress_attempts;
+static _Atomic int      stress_lives[STRESS_SLOTS];
+static _Atomic int      stress_live[STRESS_SLOTS];
+
+/* Generation and cookie are published in one word so a worker can never read
+   a cookie that belongs to a different generation than the one it claimed
+   with. Each connection gets a cookie nobody else ever gets. */
+static _Atomic uint64_t stress_conn[STRESS_SLOTS];
+static _Atomic uint64_t stress_next_cookie;
+
+#define STRESS_CONN(cookie, gen) (((uint64_t)(cookie) << 32) | (uint32_t)(gen))
+#define STRESS_CONN_COOKIE(w)    ((void *)(uintptr_t)((w) >> 32))
+#define STRESS_CONN_GEN(w)       ((uint32_t)(w))
+
+static int stress_running(void) {
+  return atomic_load(&stress_claims) < STRESS_CLAIMS && atomic_load(&stress_attempts) < STRESS_GIVEUP;
+}
+
+static void *stress_worker(void *arg) {
+  unsigned seed = (unsigned)(uintptr_t)arg;
+
+  while (stress_running()) {
+    int      fd   = (int)(rand_r(&seed) % STRESS_SLOTS);
+    uint64_t conn = atomic_load(&stress_conn[fd]);
+
+    atomic_fetch_add(&stress_attempts, 1);
+    if (!fdstate_claim(fd, STRESS_CONN_GEN(conn)))
+      continue;
+
+    if (fdstate_cookie(fd) != STRESS_CONN_COOKIE(conn))
+      atomic_fetch_add(&stress_violations, 1);
+    if (atomic_fetch_sub(&stress_lives[fd], 1) <= 1)
+      fdstate_set_expiry(fd, 1); /* retired by its holder, as a handler would */
+    if (fdstate_cookie(fd) != STRESS_CONN_COOKIE(conn))
+      atomic_fetch_add(&stress_violations, 1);
+
+    fdstate_release(fd);
+    atomic_fetch_add(&stress_claims, 1);
+  }
+  return NULL;
+}
+
+static void *stress_keeper(void *arg) {
+  (void)arg;
+  while (stress_running()) {
+    int fd;
+
+    for (fd = 0; fd < STRESS_SLOTS; ++fd) {
+      if (atomic_load(&stress_live[fd])) {
+        if (!fdstate_claim_expired(fd, 1))
+          continue;                            /* a worker has it, come back */
+        fdstate_close(fd);
+        atomic_store(&stress_live[fd], 0);
+        atomic_fetch_add(&stress_recycles, 1);
+      }
+      {
+        void    *cookie = (void *)(uintptr_t)(atomic_fetch_add(&stress_next_cookie, 1) + 1);
+        uint32_t gen    = fdstate_open(fd, cookie, 0);
+
+        atomic_store(&stress_lives[fd], RECYCLE_AFTER);
+        atomic_store(&stress_conn[fd], STRESS_CONN((uintptr_t)cookie, gen));
+        atomic_store(&stress_live[fd], 1);
+      }
+    }
+  }
+  return NULL;
+}
+
+static void case_recycling_under_load(void) {
+  pthread_t worker[4], keeper;
+  size_t    i;
+
+  test_case("no worker is ever handed a connection from another generation");
+  atomic_store(&stress_claims, 0);
+  atomic_store(&stress_recycles, 0);
+  atomic_store(&stress_violations, 0);
+  atomic_store(&stress_attempts, 0);
+  atomic_store(&stress_next_cookie, 0);
+  /* Every slot is opened and published before any worker runs: a worker only
+     ever learns of a descriptor from whoever registered it, and never
+     invents a generation of its own. */
+  for (i = 0; i < STRESS_SLOTS; ++i) {
+    void    *cookie = (void *)(uintptr_t)(atomic_fetch_add(&stress_next_cookie, 1) + 1);
+    uint32_t gen    = fdstate_open((int)i, cookie, 0);
+
+    atomic_store(&stress_lives[i], RECYCLE_AFTER);
+    atomic_store(&stress_conn[i], STRESS_CONN((uintptr_t)cookie, gen));
+    atomic_store(&stress_live[i], 1);
+  }
+
+  pthread_create(&keeper, NULL, stress_keeper, NULL);
+  for (i = 0; i < sizeof(worker) / sizeof(worker[0]); ++i)
+    pthread_create(&worker[i], NULL, stress_worker, (void *)(uintptr_t)(i + 1));
+  for (i = 0; i < sizeof(worker) / sizeof(worker[0]); ++i)
+    pthread_join(worker[i], NULL);
+  pthread_join(keeper, NULL);
+
+  CHECK_INT(atomic_load(&stress_violations), 0);
+
+  test_case("and the run actually recycled descriptors, or it proved nothing");
+  CHECK(atomic_load(&stress_claims) >= STRESS_CLAIMS);
+  CHECK(atomic_load(&stress_recycles) > STRESS_CLAIMS / (RECYCLE_AFTER * 4));
+
+  for (i = 0; i < STRESS_SLOTS; ++i)
+    fdstate_close((int)i);
+}
+
+void test_fdstate(void) {
+  CHECK_INT(fdstate_init(), 0);
+  CHECK(fdstate_slots() > STRESS_SLOTS);
+
+  case_open_claim_release();
+  case_one_holder_at_a_time();
+  case_stale_generation_is_refused();
+  case_expiry();
+  case_out_of_range();
+  case_recycling_under_load();
+
+  fdstate_deinit();
+}
diff --git a/tests/t_mutex.c b/tests/t_mutex.c
new file mode 100644
index 0000000..3307d70
--- /dev/null
+++ b/tests/t_mutex.c
@@ -0,0 +1,107 @@
+/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
+   It is considered beerware. Prost. Skol. Cheers or whatever.
+
+   $id$ */
+
+/* System */
+#include <stdlib.h>
+#include <string.h>
+
+/* Opentracker */
+#include "ot_mutex.h"
+#include "test.h"
+
+/* A worker asks for a class of task and blocks until one turns up, so a task
+   whose class matches no worker is never claimed and whoever is waiting for
+   its answer waits forever. These cases pin the matching. */
+
+static void case_task_goes_to_its_own_class(void) {
+  ot_tasktype asked;
+  ot_taskid   taskid;
+
+  test_case("a worker is handed the task of the class it asked for");
+  CHECK_INT(mutex_workqueue_pushtask(41, TASK_FULLSCRAPE_TPB_ASCII), 0);
+  asked  = TASK_FULLSCRAPE;
+  taskid = mutex_workqueue_poptask(&asked);
+  CHECK(taskid != 0);
+
+  test_case("and the full task type, not just its class");
+  CHECK_INT(asked, TASK_FULLSCRAPE_TPB_ASCII);
+  mutex_workqueue_canceltask(41);
+}
+
+static void case_flags_survive_the_queue(void) {
+  ot_tasktype asked;
+
+  test_case("flags outside the class mask are carried through");
+  CHECK_INT(mutex_workqueue_pushtask(42, TASK_FULLSCRAPE_TPB_URLENCODED | TASK_FLAG_GZIP), 0);
+  asked = TASK_FULLSCRAPE;
+  CHECK(mutex_workqueue_poptask(&asked) != 0);
+  CHECK_INT(asked, TASK_FULLSCRAPE_TPB_URLENCODED | TASK_FLAG_GZIP);
+  mutex_workqueue_canceltask(42);
+}
+
+static void case_classes_do_not_cross(void) {
+  ot_tasktype asked;
+  ot_taskid   taskid;
+
+  test_case("a stats worker passes over a fullscrape task");
+  CHECK_INT(mutex_workqueue_pushtask(43, TASK_FULLSCRAPE_TPB_BINARY), 0);
+  CHECK_INT(mutex_workqueue_pushtask(44, TASK_STATS_PEERS), 0);
+
+  asked  = TASK_STATS;
+  taskid = mutex_workqueue_poptask(&asked);
+  CHECK(taskid != 0);
+  CHECK_INT(asked, TASK_STATS_PEERS);
+
+  test_case("and leaves it for the one that wants it");
+  asked  = TASK_FULLSCRAPE;
+  taskid = mutex_workqueue_poptask(&asked);
+  CHECK(taskid != 0);
+  CHECK_INT(asked, TASK_FULLSCRAPE_TPB_BINARY);
+
+  mutex_workqueue_canceltask(43);
+  mutex_workqueue_canceltask(44);
+}
+
+static void case_every_task_gets_its_own_id(void) {
+  ot_tasktype asked;
+  ot_taskid   first, second;
+
+  test_case("two queued tasks are handed out under different ids");
+  CHECK_INT(mutex_workqueue_pushtask(45, TASK_FULLSCRAPE), 0);
+  CHECK_INT(mutex_workqueue_pushtask(46, TASK_FULLSCRAPE), 0);
+
+  asked  = TASK_FULLSCRAPE;
+  first  = mutex_workqueue_poptask(&asked);
+  asked  = TASK_FULLSCRAPE;
+  second = mutex_workqueue_poptask(&asked);
+  CHECK(first != 0);
+  CHECK(second != 0);
+  CHECK(first != second);
+
+  mutex_workqueue_canceltask(45);
+  mutex_workqueue_canceltask(46);
+}
+
+static void case_cancelled_task_is_not_handed_out(void) {
+  ot_tasktype asked;
+
+  test_case("a cancelled task is gone before any worker sees it");
+  CHECK_INT(mutex_workqueue_pushtask(47, TASK_FULLSCRAPE_TPB_ASCII_PLUS), 0);
+  CHECK_INT(mutex_workqueue_pushtask(48, TASK_FULLSCRAPE), 0);
+  mutex_workqueue_canceltask(47);
+
+  asked = TASK_FULLSCRAPE;
+  CHECK(mutex_workqueue_poptask(&asked) != 0);
+  CHECK_INT(asked, TASK_FULLSCRAPE);
+  mutex_workqueue_canceltask(48);
+}
+
+void test_mutex(void) {
+  case_task_goes_to_its_own_class();
+  case_flags_survive_the_queue();
+  case_classes_do_not_cross();
+  case_every_task_gets_its_own_id();
+  case_cancelled_task_is_not_handed_out();
+}
diff --git a/tests/test.h b/tests/test.h
index d79c4d2..bc8c3bd 100644
--- a/tests/test.h
+++ b/tests/test.h
@@ -86,5 +86,8 @@ void test_scan(void);
 void test_buf(void);
 void test_ip(void);
 void test_socket(void);
+void test_mutex(void);
+void test_batch(void);
+void test_fdstate(void);
 
 #endif