diff options
| -rw-r--r-- | Makefile | 12 | ||||
| -rw-r--r-- | opentracker.c | 40 | ||||
| -rw-r--r-- | ot_accesslist.c | 8 | ||||
| -rw-r--r-- | ot_accesslist.h | 6 | ||||
| -rw-r--r-- | ot_batch.c | 158 | ||||
| -rw-r--r-- | ot_batch.h | 48 | ||||
| -rw-r--r-- | ot_config.c | 12 | ||||
| -rw-r--r-- | ot_config.h | 8 | ||||
| -rw-r--r-- | ot_fullscrape.c | 2 | ||||
| -rw-r--r-- | ot_http.c | 85 | ||||
| -rw-r--r-- | ot_http.h | 9 | ||||
| -rw-r--r-- | ot_stats.c | 2 | ||||
| -rw-r--r-- | ot_timeout.h | 39 | ||||
| -rw-r--r-- | proxy.c | 16 | ||||
| -rw-r--r-- | scan_urlencoded_query.h | 4 | ||||
| -rw-r--r-- | tests/runner.c | 2 | ||||
| -rw-r--r-- | tests/stubs.c | 9 | ||||
| -rw-r--r-- | tests/t_batch.c | 348 | ||||
| -rw-r--r-- | tests/t_mutex.c | 107 | ||||
| -rw-r--r-- | tests/test.h | 2 | ||||
| -rw-r--r-- | trackerlogic.c | 5 | ||||
| -rw-r--r-- | trackerlogic.h | 2 |
22 files changed, 789 insertions, 135 deletions
| @@ -64,13 +64,13 @@ GIT_VERSION=`sh -c 'command -v git >/dev/null && test -d .git && git rev-parse H | |||
| 64 | OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage | 64 | OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage |
| 65 | OPTS_production=-O3 | 65 | OPTS_production=-O3 |
| 66 | 66 | ||
| 67 | CFLAGS+=-I$(LIBOWFAT_HEADERS) -DGIT_VERSION=$(GIT_VERSION) -Wall -pipe -pthread -Wextra #-ansi -pedantic | 67 | CFLAGS+=-I$(LIBOWFAT_HEADERS) -DGIT_VERSION=$(GIT_VERSION) -Wall -pipe -pthread -Wextra -Wwrite-strings #-ansi -pedantic |
| 68 | LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread | 68 | LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread |
| 69 | 69 | ||
| 70 | BINARY =opentracker | 70 | BINARY =opentracker |
| 71 | HEADERS=trackerlogic.h scan_urlencoded_query.h ot_buf.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h | 71 | HEADERS=trackerlogic.h scan_urlencoded_query.h ot_batch.h ot_buf.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h |
| 72 | SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c ot_buf.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c | 72 | SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c ot_batch.c ot_buf.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c |
| 73 | SOURCES_proxy=proxy.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c | 73 | SOURCES_proxy=proxy.c ot_batch.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c |
| 74 | 74 | ||
| 75 | OBJECTS = $(SOURCES:%.c=%.o) | 75 | OBJECTS = $(SOURCES:%.c=%.o) |
| 76 | OBJECTS_debug = $(SOURCES:%.c=%.debug.o) | 76 | OBJECTS_debug = $(SOURCES:%.c=%.debug.o) |
| @@ -119,8 +119,8 @@ $(OBJECTS_proxy_debug): $(HEADERS) | |||
| 119 | # has to. tests/stubs.c supplies the few symbols those modules reference but | 119 | # has to. tests/stubs.c supplies the few symbols those modules reference but |
| 120 | # the tests never reach. | 120 | # the tests never reach. |
| 121 | TESTBIN=tests/runner | 121 | TESTBIN=tests/runner |
| 122 | TEST_SOURCES=tests/runner.c tests/test.c tests/stubs.c tests/t_scan_urlencoded_query.c tests/t_iovec.c tests/t_vector.c tests/t_random.c tests/t_hex.c tests/t_accesslist.c tests/t_scan.c tests/t_buf.c tests/t_ip.c tests/t_socket.c | 122 | TEST_SOURCES=tests/runner.c tests/test.c tests/stubs.c tests/t_scan_urlencoded_query.c tests/t_iovec.c tests/t_vector.c tests/t_random.c tests/t_hex.c tests/t_accesslist.c tests/t_scan.c tests/t_buf.c tests/t_ip.c tests/t_socket.c tests/t_mutex.c tests/t_batch.c |
| 123 | TEST_UNITS=scan_urlencoded_query.c ot_buf.c ot_ip.c ot_socket.c ot_iovec.c ot_vector.c ot_random.c ot_hex.c ot_scan.c ot_accesslist.c | 123 | TEST_UNITS=scan_urlencoded_query.c ot_batch.c ot_buf.c ot_ip.c ot_socket.c ot_iovec.c ot_vector.c ot_random.c ot_hex.c ot_scan.c ot_accesslist.c ot_mutex.c |
| 124 | TEST_HEADERS=tests/test.h $(HEADERS) | 124 | TEST_HEADERS=tests/test.h $(HEADERS) |
| 125 | # Tests reach code that sits behind feature flags, so the units under test are | 125 | # Tests reach code that sits behind feature flags, so the units under test are |
| 126 | # built with those features enabled. A test binary is not a production binary. | 126 | # built with those features enabled. A test binary is not a production binary. |
diff --git a/opentracker.c b/opentracker.c index 0af0d17..60bdceb 100644 --- a/opentracker.c +++ b/opentracker.c | |||
| @@ -23,7 +23,6 @@ | |||
| 23 | 23 | ||
| 24 | /* Libowfat */ | 24 | /* Libowfat */ |
| 25 | #include "io.h" | 25 | #include "io.h" |
| 26 | #include "iob.h" | ||
| 27 | 26 | ||
| 28 | /* Opentracker */ | 27 | /* Opentracker */ |
| 29 | #include "ot_accesslist.h" | 28 | #include "ot_accesslist.h" |
| @@ -36,6 +35,7 @@ | |||
| 36 | #include "ot_random.h" | 35 | #include "ot_random.h" |
| 37 | #include "ot_scan.h" | 36 | #include "ot_scan.h" |
| 38 | #include "ot_socket.h" | 37 | #include "ot_socket.h" |
| 38 | #include "ot_timeout.h" | ||
| 39 | #include "ot_stats.h" | 39 | #include "ot_stats.h" |
| 40 | #include "ot_udp.h" | 40 | #include "ot_udp.h" |
| 41 | #include "trackerlogic.h" | 41 | #include "trackerlogic.h" |
| @@ -120,9 +120,8 @@ static ssize_t header_complete(char *request, ssize_t byte_count) { | |||
| 120 | static void handle_dead(const int64_t sock) { | 120 | static void handle_dead(const int64_t sock) { |
| 121 | struct http_data *cookie = io_getcookie(sock); | 121 | struct http_data *cookie = io_getcookie(sock); |
| 122 | if (cookie) { | 122 | if (cookie) { |
| 123 | size_t i; | 123 | if (cookie->batch) |
| 124 | for (i = 0; i < cookie->batches; ++i) | 124 | batch_reset(cookie->batch); |
| 125 | iob_reset(cookie->batch + i); | ||
| 126 | free(cookie->batch); | 125 | free(cookie->batch); |
| 127 | buf_reset(&cookie->request); | 126 | buf_reset(&cookie->request); |
| 128 | if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) | 127 | if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) |
| @@ -178,33 +177,23 @@ static void handle_read(const int64_t sock, struct ot_workstruct *ws) { | |||
| 178 | 177 | ||
| 179 | static void handle_write(const int64_t sock) { | 178 | static void handle_write(const int64_t sock) { |
| 180 | struct http_data *cookie = io_getcookie(sock); | 179 | struct http_data *cookie = io_getcookie(sock); |
| 181 | size_t i; | ||
| 182 | int chunked = 0; | 180 | int chunked = 0; |
| 183 | 181 | ||
| 184 | /* Look for the first io_batch still containing bytes to write */ | ||
| 185 | if (cookie) { | 182 | if (cookie) { |
| 186 | if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) | 183 | if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) |
| 187 | chunked = 1; | 184 | chunked = 1; |
| 188 | 185 | ||
| 189 | for (i = 0; i < cookie->batches; ++i) { | 186 | if (cookie->batch) { |
| 190 | if (cookie->batch[i].bytesleft) { | 187 | if (batch_send(sock, cookie->batch) == BATCH_FAILED) { |
| 191 | int64_t res = iob_send(sock, cookie->batch + i); | 188 | handle_dead(sock); |
| 192 | 189 | return; | |
| 193 | if (res == -3) { | ||
| 194 | handle_dead(sock); | ||
| 195 | return; | ||
| 196 | } | ||
| 197 | |||
| 198 | if (!cookie->batch[i].bytesleft) | ||
| 199 | continue; | ||
| 200 | |||
| 201 | if (res == -1 || res > 0 || i < cookie->batches - 1) | ||
| 202 | return; | ||
| 203 | } | 190 | } |
| 191 | if (batch_pending(cookie->batch)) | ||
| 192 | return; | ||
| 204 | } | 193 | } |
| 205 | } | 194 | } |
| 206 | 195 | ||
| 207 | /* In a chunked transfer after all batches accumulated have been sent, wait for the next one */ | 196 | /* In a chunked transfer, wait for the next piece once everything queued has gone out */ |
| 208 | if (chunked) | 197 | if (chunked) |
| 209 | io_dontwantwrite(sock); | 198 | io_dontwantwrite(sock); |
| 210 | else | 199 | else |
| @@ -216,7 +205,6 @@ static void handle_accept(const int64_t serversocket) { | |||
| 216 | int64_t sock; | 205 | int64_t sock; |
| 217 | ot_ip6 ip; | 206 | ot_ip6 ip; |
| 218 | uint16_t port; | 207 | uint16_t port; |
| 219 | tai6464 t; | ||
| 220 | 208 | ||
| 221 | while ((sock = socket_accept(serversocket, ip, &port)) != -1) { | 209 | while ((sock = socket_accept(serversocket, ip, &port)) != -1) { |
| 222 | 210 | ||
| @@ -235,11 +223,7 @@ static void handle_accept(const int64_t serversocket) { | |||
| 235 | 223 | ||
| 236 | stats_issue_event(EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip); | 224 | stats_issue_event(EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip); |
| 237 | 225 | ||
| 238 | /* That breaks taia encapsulation. But there is no way to take system | 226 | timeout_in(sock, OT_CLIENT_TIMEOUT); |
| 239 | time this often in FreeBSD and libowfat does not allow to set unix time */ | ||
| 240 | taia_uint(&t, 0); /* Clear t */ | ||
| 241 | tai_unix(&(t.sec), (g_now_seconds + OT_CLIENT_TIMEOUT)); | ||
| 242 | io_timeout(sock, t); | ||
| 243 | } | 227 | } |
| 244 | io_eagain(serversocket); | 228 | io_eagain(serversocket); |
| 245 | } | 229 | } |
| @@ -305,7 +289,7 @@ static void listen_bind_one(ot_listen *entry) { | |||
| 305 | 289 | ||
| 306 | #ifdef _DEBUG | 290 | #ifdef _DEBUG |
| 307 | { | 291 | { |
| 308 | char *protos[] = {"TCP", "UDP", "UDP mcast"}; | 292 | const char *protos[] = {"TCP", "UDP", "UDP mcast"}; |
| 309 | char _debug[512]; | 293 | char _debug[512]; |
| 310 | int off = snprintf(_debug, sizeof(_debug), "Binding socket type %s to address [", protos[entry->proto]); | 294 | int off = snprintf(_debug, sizeof(_debug), "Binding socket type %s to address [", protos[entry->proto]); |
| 311 | off += ip_format(_debug + off, entry->ip); | 295 | off += ip_format(_debug + off, entry->ip); |
diff --git a/ot_accesslist.c b/ot_accesslist.c index 2465ada..d7e88ee 100644 --- a/ot_accesslist.c +++ b/ot_accesslist.c | |||
| @@ -32,10 +32,10 @@ | |||
| 32 | 32 | ||
| 33 | /* GLOBAL VARIABLES */ | 33 | /* GLOBAL VARIABLES */ |
| 34 | #ifdef WANT_ACCESSLIST | 34 | #ifdef WANT_ACCESSLIST |
| 35 | char *g_accesslist_filename = NULL; | 35 | const char *g_accesslist_filename = NULL; |
| 36 | #ifdef WANT_DYNAMIC_ACCESSLIST | 36 | #ifdef WANT_DYNAMIC_ACCESSLIST |
| 37 | char *g_accesslist_pipe_add = NULL; | 37 | const char *g_accesslist_pipe_add = NULL; |
| 38 | char *g_accesslist_pipe_delete = NULL; | 38 | const char *g_accesslist_pipe_delete = NULL; |
| 39 | #endif | 39 | #endif |
| 40 | static pthread_mutex_t g_accesslist_mutex; | 40 | static pthread_mutex_t g_accesslist_mutex; |
| 41 | 41 | ||
| @@ -272,7 +272,7 @@ static void *accesslist_worker(void *args) { | |||
| 272 | 272 | ||
| 273 | #ifdef WANT_DYNAMIC_ACCESSLIST | 273 | #ifdef WANT_DYNAMIC_ACCESSLIST |
| 274 | static pthread_t thread_adder_id, thread_deleter_id; | 274 | static pthread_t thread_adder_id, thread_deleter_id; |
| 275 | static void *accesslist_adddel_worker(char *fifoname, ot_accesslist *_Atomic *adding_to, ot_accesslist *_Atomic *removing_from) { | 275 | static void *accesslist_adddel_worker(const char *fifoname, ot_accesslist *_Atomic *adding_to, ot_accesslist *_Atomic *removing_from) { |
| 276 | struct stat st; | 276 | struct stat st; |
| 277 | 277 | ||
| 278 | if (!stat(fifoname, &st)) { | 278 | if (!stat(fifoname, &st)) { |
diff --git a/ot_accesslist.h b/ot_accesslist.h index 30aaebd..9dfdd95 100644 --- a/ot_accesslist.h +++ b/ot_accesslist.h | |||
| @@ -23,10 +23,10 @@ void accesslist_cleanup(void); | |||
| 23 | line, and return how many were written to dest */ | 23 | line, and return how many were written to dest */ |
| 24 | size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len); | 24 | size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len); |
| 25 | 25 | ||
| 26 | extern char *g_accesslist_filename; | 26 | extern const char *g_accesslist_filename; |
| 27 | #ifdef WANT_DYNAMIC_ACCESSLIST | 27 | #ifdef WANT_DYNAMIC_ACCESSLIST |
| 28 | extern char *g_accesslist_pipe_add; | 28 | extern const char *g_accesslist_pipe_add; |
| 29 | extern char *g_accesslist_pipe_delete; | 29 | extern const char *g_accesslist_pipe_delete; |
| 30 | #endif | 30 | #endif |
| 31 | 31 | ||
| 32 | #else | 32 | #else |
diff --git a/ot_batch.c b/ot_batch.c new file mode 100644 index 0000000..84eb4a8 --- /dev/null +++ b/ot_batch.c | |||
| @@ -0,0 +1,158 @@ | |||
| 1 | /* This software was written by Dirk Engling <erdgeist@erdgeist.org> | ||
| 2 | It is considered beerware. Prost. Skol. Cheers or whatever. | ||
| 3 | |||
| 4 | $id$ */ | ||
| 5 | |||
| 6 | /* System */ | ||
| 7 | #include <errno.h> | ||
| 8 | #include <limits.h> | ||
| 9 | #include <stdlib.h> | ||
| 10 | #include <string.h> | ||
| 11 | #include <sys/socket.h> | ||
| 12 | #include <sys/uio.h> | ||
| 13 | #include <unistd.h> | ||
| 14 | |||
| 15 | /* Opentracker */ | ||
| 16 | #include "ot_batch.h" | ||
| 17 | |||
| 18 | #ifndef IOV_MAX | ||
| 19 | #define IOV_MAX 1024 | ||
| 20 | #endif | ||
| 21 | |||
| 22 | #define OT_BATCH_MIN_ENTRIES 16 | ||
| 23 | |||
| 24 | static int batch_room_for(ot_batch *batch, size_t wanted) { | ||
| 25 | size_t space = batch->space ? batch->space : OT_BATCH_MIN_ENTRIES; | ||
| 26 | struct iovec *grown; | ||
| 27 | |||
| 28 | if (batch->count + wanted <= batch->space) | ||
| 29 | return 0; | ||
| 30 | |||
| 31 | /* During a chunked transfer entries are appended while earlier ones are | ||
| 32 | still going out, so the front of the array frees up as we go. Slide the | ||
| 33 | rest down before asking for more. */ | ||
| 34 | if (batch->first && batch->count - batch->first + wanted <= batch->space) { | ||
| 35 | memmove(batch->entries, batch->entries + batch->first, (batch->count - batch->first) * sizeof(struct iovec)); | ||
| 36 | batch->count -= batch->first; | ||
| 37 | batch->first = 0; | ||
| 38 | return 0; | ||
| 39 | } | ||
| 40 | |||
| 41 | while (space < batch->count + wanted) | ||
| 42 | space *= 2; | ||
| 43 | |||
| 44 | if (!(grown = realloc(batch->entries, space * sizeof(struct iovec)))) | ||
| 45 | return -1; | ||
| 46 | |||
| 47 | batch->entries = grown; | ||
| 48 | batch->space = space; | ||
| 49 | return 0; | ||
| 50 | } | ||
| 51 | |||
| 52 | int batch_add(ot_batch *batch, void *buf, size_t len) { | ||
| 53 | if (batch_room_for(batch, 1)) { | ||
| 54 | free(buf); | ||
| 55 | return -1; | ||
| 56 | } | ||
| 57 | |||
| 58 | batch->entries[batch->count].iov_base = buf; | ||
| 59 | batch->entries[batch->count].iov_len = len; | ||
| 60 | ++batch->count; | ||
| 61 | return 0; | ||
| 62 | } | ||
| 63 | |||
| 64 | int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count) { | ||
| 65 | if (batch_room_for(batch, count)) { | ||
| 66 | while (count--) | ||
| 67 | free(entries[count].iov_base); | ||
| 68 | return -1; | ||
| 69 | } | ||
| 70 | |||
| 71 | memcpy(batch->entries + batch->count, entries, count * sizeof(struct iovec)); | ||
| 72 | batch->count += count; | ||
| 73 | return 0; | ||
| 74 | } | ||
| 75 | |||
| 76 | /* Drop everything the kernel accepted, freeing each buffer as it goes */ | ||
| 77 | static void batch_consume(ot_batch *batch, size_t written) { | ||
| 78 | while (written) { | ||
| 79 | struct iovec *entry = batch->entries + batch->first; | ||
| 80 | size_t left = entry->iov_len - batch->offset; | ||
| 81 | |||
| 82 | if (written < left) { | ||
| 83 | batch->offset += written; | ||
| 84 | return; | ||
| 85 | } | ||
| 86 | |||
| 87 | free(entry->iov_base); | ||
| 88 | written -= left; | ||
| 89 | batch->offset = 0; | ||
| 90 | ++batch->first; | ||
| 91 | } | ||
| 92 | |||
| 93 | /* Nothing is outstanding, so the array can start over from the front */ | ||
| 94 | if (batch->first == batch->count) | ||
| 95 | batch->first = batch->count = 0; | ||
| 96 | } | ||
| 97 | |||
| 98 | /* A peer that has gone away must come back as an error rather than as a | ||
| 99 | signal that kills the tracker, so ask the kernel not to raise one where it | ||
| 100 | can be asked. macOS cannot, and relies on SIGPIPE being blocked. */ | ||
| 101 | static ssize_t batch_writev(int s, struct iovec *entries, int count) { | ||
| 102 | #ifdef MSG_NOSIGNAL | ||
| 103 | struct msghdr msg; | ||
| 104 | |||
| 105 | memset(&msg, 0, sizeof(msg)); | ||
| 106 | msg.msg_iov = entries; | ||
| 107 | msg.msg_iovlen = count; | ||
| 108 | return sendmsg(s, &msg, MSG_NOSIGNAL); | ||
| 109 | #else | ||
| 110 | return writev(s, entries, count); | ||
| 111 | #endif | ||
| 112 | } | ||
| 113 | |||
| 114 | ssize_t batch_send(int s, ot_batch *batch) { | ||
| 115 | size_t entries = batch->count - batch->first; | ||
| 116 | ssize_t written; | ||
| 117 | |||
| 118 | if (!batch_pending(batch)) | ||
| 119 | return BATCH_DONE; | ||
| 120 | |||
| 121 | if (entries > IOV_MAX) | ||
| 122 | entries = IOV_MAX; | ||
| 123 | |||
| 124 | /* The first entry may be half way out already */ | ||
| 125 | batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base + batch->offset; | ||
| 126 | batch->entries[batch->first].iov_len -= batch->offset; | ||
| 127 | written = batch_writev(s, batch->entries + batch->first, (int)entries); | ||
| 128 | batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base - batch->offset; | ||
| 129 | batch->entries[batch->first].iov_len += batch->offset; | ||
| 130 | |||
| 131 | if (written < 0) | ||
| 132 | return (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) ? BATCH_AGAIN : BATCH_FAILED; | ||
| 133 | if (!written) | ||
| 134 | return BATCH_FAILED; | ||
| 135 | |||
| 136 | batch_consume(batch, (size_t)written); | ||
| 137 | return written; | ||
| 138 | } | ||
| 139 | |||
| 140 | void batch_reset(ot_batch *batch) { | ||
| 141 | while (batch->first < batch->count) | ||
| 142 | free(batch->entries[batch->first++].iov_base); | ||
| 143 | |||
| 144 | free(batch->entries); | ||
| 145 | batch->entries = NULL; | ||
| 146 | batch->count = batch->first = batch->offset = batch->space = 0; | ||
| 147 | } | ||
| 148 | |||
| 149 | /* For the few constants the chunked encoding needs, which the batch frees | ||
| 150 | along with everything else */ | ||
| 151 | int batch_add_copy(ot_batch *batch, const void *buf, size_t len) { | ||
| 152 | void *copy = malloc(len); | ||
| 153 | |||
| 154 | if (!copy) | ||
| 155 | return -1; | ||
| 156 | memcpy(copy, buf, len); | ||
| 157 | return batch_add(batch, copy, len); | ||
| 158 | } | ||
diff --git a/ot_batch.h b/ot_batch.h new file mode 100644 index 0000000..c855671 --- /dev/null +++ b/ot_batch.h | |||
| @@ -0,0 +1,48 @@ | |||
| 1 | /* This software was written by Dirk Engling <erdgeist@erdgeist.org> | ||
| 2 | It is considered beerware. Prost. Skol. Cheers or whatever. | ||
| 3 | |||
| 4 | $id$ */ | ||
| 5 | |||
| 6 | #ifndef OT_BATCH_H__ | ||
| 7 | #define OT_BATCH_H__ | ||
| 8 | |||
| 9 | #include <stddef.h> | ||
| 10 | #include <sys/types.h> | ||
| 11 | #include <sys/uio.h> | ||
| 12 | |||
| 13 | /* A queue of buffers on their way to one socket. The batch owns every buffer | ||
| 14 | handed to it and frees each one as soon as it has left, so a client that | ||
| 15 | reads slowly costs memory only for what it has not received yet. | ||
| 16 | |||
| 17 | A zeroed ot_batch is ready to use. */ | ||
| 18 | typedef struct { | ||
| 19 | struct iovec *entries; | ||
| 20 | size_t space; | ||
| 21 | size_t count; | ||
| 22 | size_t first; /* entries before this one are sent and freed */ | ||
| 23 | size_t offset; /* bytes of entries[first] already sent */ | ||
| 24 | } ot_batch; | ||
| 25 | |||
| 26 | static inline int batch_pending(const ot_batch *batch) { return batch->first != batch->count; } | ||
| 27 | |||
| 28 | /* What batch_send reports instead of a byte count */ | ||
| 29 | enum { BATCH_DONE = 0, BATCH_AGAIN = -1, BATCH_FAILED = -2 }; | ||
| 30 | |||
| 31 | /* Queue buf and take ownership of it. Returns 0, or -1 having freed buf. */ | ||
| 32 | int batch_add(ot_batch *batch, void *buf, size_t len); | ||
| 33 | |||
| 34 | /* Write as much as the socket takes. Returns the number of bytes written, or | ||
| 35 | BATCH_DONE, BATCH_AGAIN or BATCH_FAILED. */ | ||
| 36 | ssize_t batch_send(int s, ot_batch *batch); | ||
| 37 | |||
| 38 | /* Queue every entry of a vector, taking ownership of each buffer but not of | ||
| 39 | the vector itself. Returns 0, or -1 having freed all of them. */ | ||
| 40 | int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count); | ||
| 41 | |||
| 42 | /* Queue a copy of buf, which the batch then owns. Returns 0, or -1. */ | ||
| 43 | int batch_add_copy(ot_batch *batch, const void *buf, size_t len); | ||
| 44 | |||
| 45 | /* Free whatever has not left yet, along with the batch's own array */ | ||
| 46 | void batch_reset(ot_batch *batch); | ||
| 47 | |||
| 48 | #endif | ||
diff --git a/ot_config.c b/ot_config.c index 43c07c5..e9666f9 100644 --- a/ot_config.c +++ b/ot_config.c | |||
| @@ -22,10 +22,10 @@ | |||
| 22 | #include "ot_stats.h" | 22 | #include "ot_stats.h" |
| 23 | #include "trackerlogic.h" | 23 | #include "trackerlogic.h" |
| 24 | 24 | ||
| 25 | char *g_redirecturl; | 25 | const char *g_redirecturl; |
| 26 | char *g_serverdir; | 26 | const char *g_serverdir; |
| 27 | char *g_serveruser; | 27 | const char *g_serveruser; |
| 28 | char *g_statefile; | 28 | const char *g_statefile; |
| 29 | 29 | ||
| 30 | #define OT_LISTEN_INHERIT (-1) | 30 | #define OT_LISTEN_INHERIT (-1) |
| 31 | 31 | ||
| @@ -122,13 +122,13 @@ static int listen_scan_options(char *p, int *shared, int *workers) { | |||
| 122 | } | 122 | } |
| 123 | 123 | ||
| 124 | 124 | ||
| 125 | char *set_config_option(char **option, char *value) { | 125 | static const char *set_config_option(const char **option, char *value) { |
| 126 | #ifdef _DEBUG | 126 | #ifdef _DEBUG |
| 127 | fprintf(stderr, "Setting config option: %s\n", value); | 127 | fprintf(stderr, "Setting config option: %s\n", value); |
| 128 | #endif | 128 | #endif |
| 129 | while (isspace(*value)) | 129 | while (isspace(*value)) |
| 130 | ++value; | 130 | ++value; |
| 131 | free(*option); | 131 | free((char *)*option); |
| 132 | return *option = strdup(value); | 132 | return *option = strdup(value); |
| 133 | } | 133 | } |
| 134 | 134 | ||
diff --git a/ot_config.h b/ot_config.h index ab25e2c..059573a 100644 --- a/ot_config.h +++ b/ot_config.h | |||
| @@ -22,10 +22,10 @@ typedef struct { | |||
| 22 | int64_t sock; | 22 | int64_t sock; |
| 23 | } ot_listen; | 23 | } ot_listen; |
| 24 | 24 | ||
| 25 | extern char *g_redirecturl; | 25 | extern const char *g_redirecturl; |
| 26 | extern char *g_serverdir; | 26 | extern const char *g_serverdir; |
| 27 | extern char *g_serveruser; | 27 | extern const char *g_serveruser; |
| 28 | extern char *g_statefile; | 28 | extern const char *g_statefile; |
| 29 | 29 | ||
| 30 | /* Reads the command line and every config file it names. Exits on error. */ | 30 | /* Reads the command line and every config file it names. Exits on error. */ |
| 31 | void config_parse(int argc, char **argv); | 31 | void config_parse(int argc, char **argv); |
diff --git a/ot_fullscrape.c b/ot_fullscrape.c index 9dcbe68..eade79d 100644 --- a/ot_fullscrape.c +++ b/ot_fullscrape.c | |||
| @@ -83,7 +83,7 @@ void fullscrape_deinit(void) { | |||
| 83 | } | 83 | } |
| 84 | 84 | ||
| 85 | void fullscrape_deliver( int64_t sock, ot_tasktype tasktype ) { | 85 | void fullscrape_deliver( int64_t sock, ot_tasktype tasktype ) { |
| 86 | mutex_workqueue_pushtask( sock, tasktype ); | 86 | mutex_workqueue_pushtask( sock, TASK_FULLSCRAPE | tasktype ); |
| 87 | } | 87 | } |
| 88 | 88 | ||
| 89 | static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_torrent *torrent, ot_hash *hash ) { | 89 | static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_torrent *torrent, ot_hash *hash ) { |
| @@ -15,7 +15,7 @@ | |||
| 15 | #include <unistd.h> | 15 | #include <unistd.h> |
| 16 | 16 | ||
| 17 | /* Libowfat */ | 17 | /* Libowfat */ |
| 18 | #include "iob.h" | 18 | #include "io.h" |
| 19 | 19 | ||
| 20 | /* Opentracker */ | 20 | /* Opentracker */ |
| 21 | #include "ot_accesslist.h" | 21 | #include "ot_accesslist.h" |
| @@ -26,20 +26,14 @@ | |||
| 26 | #include "ot_mutex.h" | 26 | #include "ot_mutex.h" |
| 27 | #include "ot_scan.h" | 27 | #include "ot_scan.h" |
| 28 | #include "ot_stats.h" | 28 | #include "ot_stats.h" |
| 29 | #include "ot_timeout.h" | ||
| 29 | #include "scan_urlencoded_query.h" | 30 | #include "scan_urlencoded_query.h" |
| 30 | #include "trackerlogic.h" | 31 | #include "trackerlogic.h" |
| 31 | 32 | ||
| 32 | #ifdef WANT_NO_AUTO_FREE | ||
| 33 | #define OT_IOB_INIT(B) (bzero(B, sizeof(io_batch)), 0) | ||
| 34 | #else | ||
| 35 | #define OT_IOB_INIT(B) iob_init_autofree(B, 0) | ||
| 36 | #endif | ||
| 37 | |||
| 38 | #define OT_MAXMULTISCRAPE_COUNT 64 | 33 | #define OT_MAXMULTISCRAPE_COUNT 64 |
| 39 | #define OT_BATCH_LIMIT (1024 * 1024 * 16) | 34 | extern const char *g_redirecturl; |
| 40 | extern char *g_redirecturl; | ||
| 41 | 35 | ||
| 42 | char *g_stats_path; | 36 | const char *g_stats_path; |
| 43 | ssize_t g_stats_path_len; | 37 | ssize_t g_stats_path_len; |
| 44 | 38 | ||
| 45 | enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 }; | 39 | enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 }; |
| @@ -79,7 +73,6 @@ static void http_senddata(const int64_t sock, struct ot_workstruct *ws) { | |||
| 79 | 73 | ||
| 80 | if (written_size < ws->reply_size) { | 74 | if (written_size < ws->reply_size) { |
| 81 | char *outbuf; | 75 | char *outbuf; |
| 82 | tai6464 t; | ||
| 83 | 76 | ||
| 84 | if (!(outbuf = malloc(ws->reply_size - written_size))) { | 77 | if (!(outbuf = malloc(ws->reply_size - written_size))) { |
| 85 | buf_reset(&cookie->request); | 78 | buf_reset(&cookie->request); |
| @@ -90,25 +83,21 @@ static void http_senddata(const int64_t sock, struct ot_workstruct *ws) { | |||
| 90 | 83 | ||
| 91 | memcpy(outbuf, ws->reply + written_size, ws->reply_size - written_size); | 84 | memcpy(outbuf, ws->reply + written_size, ws->reply_size - written_size); |
| 92 | if (!cookie->batch) { | 85 | if (!cookie->batch) { |
| 93 | cookie->batch = malloc(sizeof(io_batch)); | 86 | cookie->batch = calloc(1, sizeof(ot_batch)); |
| 94 | if (!cookie->batch || OT_IOB_INIT(cookie->batch) == -1) { | 87 | if (!cookie->batch) { |
| 95 | free(cookie->batch); | ||
| 96 | free(outbuf); | 88 | free(outbuf); |
| 97 | buf_reset(&cookie->request); | 89 | buf_reset(&cookie->request); |
| 98 | free(cookie); | 90 | free(cookie); |
| 99 | io_close(sock); | 91 | io_close(sock); |
| 100 | return; | 92 | return; |
| 101 | } | 93 | } |
| 102 | |||
| 103 | cookie->batches = 1; | ||
| 104 | } | 94 | } |
| 105 | 95 | ||
| 106 | iob_addbuf_free(cookie->batch, outbuf, ws->reply_size - written_size); | 96 | batch_add(cookie->batch, outbuf, ws->reply_size - written_size); |
| 107 | 97 | ||
| 108 | /* writeable short data sockets just have a tcp timeout */ | 98 | /* writeable short data sockets just have a tcp timeout */ |
| 109 | if (!ws->keep_alive) { | 99 | if (!ws->keep_alive) { |
| 110 | taia_uint(&t, 0); | 100 | timeout_never(sock); |
| 111 | io_timeout(sock, t); | ||
| 112 | io_dontwantread(sock); | 101 | io_dontwantread(sock); |
| 113 | } | 102 | } |
| 114 | io_wantwrite(sock); | 103 | io_wantwrite(sock); |
| @@ -125,9 +114,9 @@ static void http_senddata(const int64_t sock, struct ot_workstruct *ws) { | |||
| 125 | #define HTTPERROR_404 return http_issue_error(sock, ws, CODE_HTTPERROR_404) | 114 | #define HTTPERROR_404 return http_issue_error(sock, ws, CODE_HTTPERROR_404) |
| 126 | #define HTTPERROR_500 return http_issue_error(sock, ws, CODE_HTTPERROR_500) | 115 | #define HTTPERROR_500 return http_issue_error(sock, ws, CODE_HTTPERROR_500) |
| 127 | ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code) { | 116 | ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code) { |
| 128 | char *error_code[] = {"302 Found", "400 Invalid Request", "400 Invalid Request", "400 Invalid Request", "402 Payment Required", | 117 | const char *error_code[] = {"302 Found", "400 Invalid Request", "400 Invalid Request", "400 Invalid Request", "402 Payment Required", |
| 129 | "403 Not Modest", "403 Access Denied", "404 Not Found", "500 Internal Server Error"}; | 118 | "403 Not Modest", "403 Access Denied", "404 Not Found", "500 Internal Server Error"}; |
| 130 | char *title = error_code[code]; | 119 | const char *title = error_code[code]; |
| 131 | 120 | ||
| 132 | ws->reply = ws->outbuf; | 121 | ws->reply = ws->outbuf; |
| 133 | if (code == CODE_HTTPERROR_302) | 122 | if (code == CODE_HTTPERROR_302) |
| @@ -146,11 +135,9 @@ ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code) | |||
| 146 | 135 | ||
| 147 | ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial) { | 136 | ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial) { |
| 148 | struct http_data *cookie = io_getcookie(sock); | 137 | struct http_data *cookie = io_getcookie(sock); |
| 149 | io_batch *current; | ||
| 150 | char *header; | 138 | char *header; |
| 151 | const char *encoding = ""; | 139 | const char *encoding = ""; |
| 152 | size_t i, header_size, size = iovec_length(&iovec_entries, (const struct iovec **)&iovector); | 140 | size_t header_size, size = iovec_length(&iovec_entries, (const struct iovec **)&iovector); |
| 153 | tai6464 t; | ||
| 154 | 141 | ||
| 155 | /* No cookie? Bad socket. Leave. */ | 142 | /* No cookie? Bad socket. Leave. */ |
| 156 | if (!cookie) { | 143 | if (!cookie) { |
| @@ -199,46 +186,28 @@ ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t | |||
| 199 | } | 186 | } |
| 200 | 187 | ||
| 201 | if (!cookie->batch) { | 188 | if (!cookie->batch) { |
| 202 | cookie->batch = malloc(sizeof(io_batch)); | 189 | cookie->batch = calloc(1, sizeof(ot_batch)); |
| 203 | if (!cookie->batch || OT_IOB_INIT(cookie->batch) == -1) { | 190 | if (!cookie->batch) { |
| 204 | free(cookie->batch); | ||
| 205 | free(header); | 191 | free(header); |
| 206 | iovec_free(&iovec_entries, &iovector); | 192 | iovec_free(&iovec_entries, &iovector); |
| 207 | HTTPERROR_500; | 193 | HTTPERROR_500; |
| 208 | } | 194 | } |
| 209 | cookie->batches = 1; | ||
| 210 | } | 195 | } |
| 211 | current = cookie->batch + cookie->batches - 1; | ||
| 212 | iob_addbuf_free(current, header, header_size); | ||
| 213 | 196 | ||
| 214 | /* Split huge iovectors into separate io_batches */ | 197 | batch_add(cookie->batch, header, header_size); |
| 215 | for (i = 0; i < (size_t)iovec_entries; ++i) { | 198 | batch_add_vector(cookie->batch, iovector, iovec_entries); |
| 216 | /* If the current batch's limit is reached, try to reallocate a new batch to work on */ | ||
| 217 | if (current->bytesleft > OT_BATCH_LIMIT) { | ||
| 218 | io_batch *new_batch = realloc(cookie->batch, (cookie->batches + 1) * sizeof(io_batch)); | ||
| 219 | if (new_batch) { | ||
| 220 | cookie->batch = new_batch; | ||
| 221 | if (OT_IOB_INIT(current) != -1) | ||
| 222 | current = cookie->batch + cookie->batches++; | ||
| 223 | } | ||
| 224 | } | ||
| 225 | iob_addbuf_free(current, iovector[i].iov_base, iovector[i].iov_len); | ||
| 226 | } | ||
| 227 | free(iovector); | 199 | free(iovector); |
| 200 | |||
| 228 | if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) | 201 | if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) |
| 229 | iob_addbuf(current, "\r\n", 2); | 202 | batch_add_copy(cookie->batch, "\r\n", 2); |
| 230 | } | 203 | } |
| 231 | 204 | ||
| 232 | if ((cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) && cookie->batch && !is_partial) { | 205 | if ((cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) && cookie->batch && !is_partial) { |
| 233 | current = cookie->batch + cookie->batches - 1; | 206 | batch_add_copy(cookie->batch, "0\r\n\r\n", 5); |
| 234 | iob_addbuf(current, "0\r\n\r\n", 5); | ||
| 235 | cookie->flag &= ~STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; | 207 | cookie->flag &= ~STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; |
| 236 | } | 208 | } |
| 237 | 209 | ||
| 238 | /* writeable sockets timeout after 10 minutes */ | 210 | timeout_in(sock, OT_CLIENT_TIMEOUT_SEND); |
| 239 | taia_now(&t); | ||
| 240 | taia_addsec(&t, &t, OT_CLIENT_TIMEOUT_SEND); | ||
| 241 | io_timeout(sock, t); | ||
| 242 | io_wantwrite(sock); | 211 | io_wantwrite(sock); |
| 243 | return 0; | 212 | return 0; |
| 244 | } | 213 | } |
| @@ -317,7 +286,6 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c | |||
| 317 | } | 286 | } |
| 318 | 287 | ||
| 319 | if (mode == TASK_STATS_TPB) { | 288 | if (mode == TASK_STATS_TPB) { |
| 320 | tai6464 t; | ||
| 321 | #ifdef WANT_COMPRESSION_GZIP | 289 | #ifdef WANT_COMPRESSION_GZIP |
| 322 | ws->request[ws->request_size - 1] = 0; | 290 | ws->request[ws->request_size - 1] = 0; |
| 323 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS | 291 | #ifndef WANT_COMPRESSION_GZIP_ALWAYS |
| @@ -333,8 +301,7 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c | |||
| 333 | cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; | 301 | cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; |
| 334 | 302 | ||
| 335 | /* Clients waiting for us should not easily timeout */ | 303 | /* Clients waiting for us should not easily timeout */ |
| 336 | taia_uint(&t, 0); | 304 | timeout_never(sock); |
| 337 | io_timeout(sock, t); | ||
| 338 | fullscrape_deliver(sock, format); | 305 | fullscrape_deliver(sock, format); |
| 339 | io_dontwantread(sock); | 306 | io_dontwantread(sock); |
| 340 | return ws->reply_size = -2; | 307 | return ws->reply_size = -2; |
| @@ -343,13 +310,11 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c | |||
| 343 | 310 | ||
| 344 | /* default format for now */ | 311 | /* default format for now */ |
| 345 | if ((mode & TASK_CLASS_MASK) == TASK_STATS) { | 312 | if ((mode & TASK_CLASS_MASK) == TASK_STATS) { |
| 346 | tai6464 t; | ||
| 347 | 313 | ||
| 348 | cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK; | 314 | cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK; |
| 349 | 315 | ||
| 350 | /* Complex stats also include expensive memory debugging tools */ | 316 | /* Complex stats also include expensive memory debugging tools */ |
| 351 | taia_uint(&t, 0); | 317 | timeout_never(sock); |
| 352 | io_timeout(sock, t); | ||
| 353 | stats_deliver(sock, mode); | 318 | stats_deliver(sock, mode); |
| 354 | return ws->reply_size = -2; | 319 | return ws->reply_size = -2; |
| 355 | } | 320 | } |
| @@ -371,7 +336,6 @@ typedef struct { | |||
| 371 | static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *ws) { | 336 | static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *ws) { |
| 372 | struct http_data *cookie = io_getcookie(sock); | 337 | struct http_data *cookie = io_getcookie(sock); |
| 373 | int format = 0; | 338 | int format = 0; |
| 374 | tai6464 t; | ||
| 375 | 339 | ||
| 376 | #ifdef WANT_MODEST_FULLSCRAPES | 340 | #ifdef WANT_MODEST_FULLSCRAPES |
| 377 | { | 341 | { |
| @@ -430,9 +394,8 @@ static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct * | |||
| 430 | /* Pass this task to the worker thread */ | 394 | /* Pass this task to the worker thread */ |
| 431 | cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; | 395 | cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; |
| 432 | /* Clients waiting for us should not easily timeout */ | 396 | /* Clients waiting for us should not easily timeout */ |
| 433 | taia_uint(&t, 0); | 397 | timeout_never(sock); |
| 434 | io_timeout(sock, t); | 398 | fullscrape_deliver(sock, format); |
| 435 | fullscrape_deliver(sock, TASK_FULLSCRAPE | format); | ||
| 436 | io_dontwantread(sock); | 399 | io_dontwantread(sock); |
| 437 | return ws->reply_size = -2; | 400 | return ws->reply_size = -2; |
| 438 | } | 401 | } |
| @@ -491,7 +454,7 @@ unsigned long long numwants[201]; | |||
| 491 | #endif | 454 | #endif |
| 492 | 455 | ||
| 493 | #if defined(WANT_KEEPALIVE) || defined(WANT_IP_FROM_PROXY) | 456 | #if defined(WANT_KEEPALIVE) || defined(WANT_IP_FROM_PROXY) |
| 494 | static char *http_header(char *data, size_t byte_count, char *header) { | 457 | static char *http_header(char *data, size_t byte_count, const char *header) { |
| 495 | size_t i; | 458 | size_t i; |
| 496 | long sl = strlen(header); | 459 | long sl = strlen(header); |
| 497 | for (i = 0; i + sl + 2 < byte_count; ++i) { | 460 | for (i = 0; i + sl + 2 < byte_count; ++i) { |
| @@ -10,9 +10,7 @@ | |||
| 10 | #include <sys/types.h> | 10 | #include <sys/types.h> |
| 11 | #include <sys/uio.h> | 11 | #include <sys/uio.h> |
| 12 | 12 | ||
| 13 | /* Libowfat */ | 13 | #include "ot_batch.h" |
| 14 | #include "iob.h" | ||
| 15 | |||
| 16 | #include "ot_buf.h" | 14 | #include "ot_buf.h" |
| 17 | #include "trackerlogic.h" | 15 | #include "trackerlogic.h" |
| 18 | 16 | ||
| @@ -27,8 +25,7 @@ typedef enum { | |||
| 27 | 25 | ||
| 28 | struct http_data { | 26 | struct http_data { |
| 29 | ot_buf request; | 27 | ot_buf request; |
| 30 | io_batch *batch; | 28 | ot_batch *batch; |
| 31 | size_t batches; | ||
| 32 | ot_ip6 ip; | 29 | ot_ip6 ip; |
| 33 | STRUCT_HTTP_FLAG flag; | 30 | STRUCT_HTTP_FLAG flag; |
| 34 | }; | 31 | }; |
| @@ -37,7 +34,7 @@ ssize_t http_handle_request(const int64_t s, struct ot_workstruct *ws); | |||
| 37 | ssize_t http_sendiovecdata(const int64_t s, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial); | 34 | ssize_t http_sendiovecdata(const int64_t s, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial); |
| 38 | ssize_t http_issue_error(const int64_t s, struct ot_workstruct *ws, int code); | 35 | ssize_t http_issue_error(const int64_t s, struct ot_workstruct *ws, int code); |
| 39 | 36 | ||
| 40 | extern char *g_stats_path; | 37 | extern const char *g_stats_path; |
| 41 | extern ssize_t g_stats_path_len; | 38 | extern ssize_t g_stats_path_len; |
| 42 | 39 | ||
| 43 | #endif | 40 | #endif |
| @@ -53,7 +53,7 @@ static unsigned long long ot_full_scrape_count; | |||
| 53 | static unsigned long long ot_full_scrape_request_count; | 53 | static unsigned long long ot_full_scrape_request_count; |
| 54 | static unsigned long long ot_full_scrape_size; | 54 | static unsigned long long ot_full_scrape_size; |
| 55 | static unsigned long long ot_failed_request_counts[CODE_HTTPERROR_COUNT]; | 55 | static unsigned long long ot_failed_request_counts[CODE_HTTPERROR_COUNT]; |
| 56 | static char *ot_failed_request_names[] = { | 56 | static const char *ot_failed_request_names[] = { |
| 57 | "302 Redirect", "400 Parse Error", "400 Invalid Parameter", "400 Invalid Parameter (compact=0)", "400 Not Modest", | 57 | "302 Redirect", "400 Parse Error", "400 Invalid Parameter", "400 Invalid Parameter (compact=0)", "400 Not Modest", |
| 58 | "402 Payment Required", "403 Access Denied", "404 Not found", "500 Internal Server Error"}; | 58 | "402 Payment Required", "403 Access Denied", "404 Not found", "500 Internal Server Error"}; |
| 59 | static unsigned long long ot_renewed[OT_PEER_TIMEOUT]; | 59 | static unsigned long long ot_renewed[OT_PEER_TIMEOUT]; |
diff --git a/ot_timeout.h b/ot_timeout.h new file mode 100644 index 0000000..1027c2e --- /dev/null +++ b/ot_timeout.h | |||
| @@ -0,0 +1,39 @@ | |||
| 1 | /* This software was written by Dirk Engling <erdgeist@erdgeist.org> | ||
| 2 | It is considered beerware. Prost. Skol. Cheers or whatever. | ||
| 3 | |||
| 4 | $id$ */ | ||
| 5 | |||
| 6 | #ifndef OT_TIMEOUT_H__ | ||
| 7 | #define OT_TIMEOUT_H__ | ||
| 8 | |||
| 9 | #include <stdint.h> | ||
| 10 | #include <string.h> | ||
| 11 | |||
| 12 | /* Libowfat */ | ||
| 13 | #include "io.h" | ||
| 14 | |||
| 15 | #include "trackerlogic.h" | ||
| 16 | |||
| 17 | /* io_timeout wants an absolute TAI64 deadline and io_timeouted skips any | ||
| 18 | socket whose deadline is zero, so that is how a socket is told to wait | ||
| 19 | indefinitely. | ||
| 20 | |||
| 21 | The deadline is taken from the cached clock rather than from | ||
| 22 | gettimeofday, which is too expensive to call at this rate on FreeBSD. */ | ||
| 23 | |||
| 24 | static inline void timeout_never(int64_t sock) { | ||
| 25 | tai6464 t; | ||
| 26 | |||
| 27 | memset(&t, 0, sizeof(t)); | ||
| 28 | io_timeout(sock, t); | ||
| 29 | } | ||
| 30 | |||
| 31 | static inline void timeout_in(int64_t sock, unsigned int seconds) { | ||
| 32 | tai6464 t; | ||
| 33 | |||
| 34 | memset(&t, 0, sizeof(t)); | ||
| 35 | tai_unix(&t.sec, g_now_seconds + seconds); | ||
| 36 | io_timeout(sock, t); | ||
| 37 | } | ||
| 38 | |||
| 39 | #endif | ||
| @@ -16,9 +16,9 @@ | |||
| 16 | 16 | ||
| 17 | /* Libowfat */ | 17 | /* Libowfat */ |
| 18 | #include "io.h" | 18 | #include "io.h" |
| 19 | #include "iob.h" | ||
| 20 | 19 | ||
| 21 | /* Opentracker */ | 20 | /* Opentracker */ |
| 21 | #include "ot_batch.h" | ||
| 22 | #include "ot_ip.h" | 22 | #include "ot_ip.h" |
| 23 | #include "ot_mutex.h" | 23 | #include "ot_mutex.h" |
| 24 | #include "ot_random.h" | 24 | #include "ot_random.h" |
| @@ -72,7 +72,7 @@ static void *livesync_worker(void *args); | |||
| 72 | static void *streamsync_worker(void *args); | 72 | static void *streamsync_worker(void *args); |
| 73 | static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer); | 73 | static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer); |
| 74 | 74 | ||
| 75 | void exerr(char *message) { | 75 | void exerr(const char *message) { |
| 76 | fprintf(stderr, "%s\n", message); | 76 | fprintf(stderr, "%s\n", message); |
| 77 | exit(111); | 77 | exit(111); |
| 78 | } | 78 | } |
| @@ -228,7 +228,7 @@ typedef struct { | |||
| 228 | size_t indata_length; /* Length of unprocessed data */ | 228 | size_t indata_length; /* Length of unprocessed data */ |
| 229 | uint32_t tracker_id; /* How the other end greeted */ | 229 | uint32_t tracker_id; /* How the other end greeted */ |
| 230 | int64_t fd; /* A file handle, if connected, <= 0 is disconnected (0 initially, -1 else) */ | 230 | int64_t fd; /* A file handle, if connected, <= 0 is disconnected (0 initially, -1 else) */ |
| 231 | io_batch outdata; /* The iobatch containing our sync data */ | 231 | ot_batch outdata; /* The batch containing our sync data */ |
| 232 | 232 | ||
| 233 | size_t packet_tcount; /* Number of unprocessed torrents in packet we currently receive */ | 233 | size_t packet_tcount; /* Number of unprocessed torrents in packet we currently receive */ |
| 234 | uint8_t packet_tprefix; /* Prefix byte for all torrents in current packet */ | 234 | uint8_t packet_tprefix; /* Prefix byte for all torrents in current packet */ |
| @@ -243,7 +243,7 @@ void reset_info_block(proxy_peer *peer) { | |||
| 243 | peer->tracker_id = 0; | 243 | peer->tracker_id = 0; |
| 244 | peer->fd = -1; | 244 | peer->fd = -1; |
| 245 | peer->packet_tcount = 0; | 245 | peer->packet_tcount = 0; |
| 246 | iob_reset(&peer->outdata); | 246 | batch_reset(&peer->outdata); |
| 247 | PROXYPEER_SETDISCONNECTED(peer->state); | 247 | PROXYPEER_SETDISCONNECTED(peer->state); |
| 248 | } | 248 | } |
| 249 | 249 | ||
| @@ -424,11 +424,11 @@ static void handle_write(int64_t peersocket) { | |||
| 424 | } | 424 | } |
| 425 | break; | 425 | break; |
| 426 | case FLAG_CONNECTED: | 426 | case FLAG_CONNECTED: |
| 427 | switch (iob_send(peersocket, &peer->outdata)) { | 427 | switch (batch_send(peersocket, &peer->outdata)) { |
| 428 | case 0: /* all data sent */ | 428 | case BATCH_DONE: |
| 429 | io_dontwantwrite(peersocket); | 429 | io_dontwantwrite(peersocket); |
| 430 | break; | 430 | break; |
| 431 | case -3: /* an error occured */ | 431 | case BATCH_FAILED: |
| 432 | io_close(peersocket); | 432 | io_close(peersocket); |
| 433 | reset_info_block(peer); | 433 | reset_info_block(peer); |
| 434 | break; | 434 | break; |
| @@ -728,7 +728,7 @@ static void *streamsync_worker(void *args) { | |||
| 728 | void *tmp = malloc(mem); | 728 | void *tmp = malloc(mem); |
| 729 | if (tmp) { | 729 | if (tmp) { |
| 730 | memcpy(tmp, ptr, mem); | 730 | memcpy(tmp, ptr, mem); |
| 731 | iob_addbuf_free(&g_connections[i].outdata, tmp, mem); | 731 | batch_add(&g_connections[i].outdata, tmp, mem); |
| 732 | io_wantwrite(g_connections[i].fd); | 732 | io_wantwrite(g_connections[i].fd); |
| 733 | } | 733 | } |
| 734 | } | 734 | } |
diff --git a/scan_urlencoded_query.h b/scan_urlencoded_query.h index d0929cf..f71752d 100644 --- a/scan_urlencoded_query.h +++ b/scan_urlencoded_query.h | |||
| @@ -11,8 +11,8 @@ | |||
| 11 | #include <sys/types.h> | 11 | #include <sys/types.h> |
| 12 | 12 | ||
| 13 | typedef struct { | 13 | typedef struct { |
| 14 | char *key; | 14 | const char *key; |
| 15 | int value; | 15 | int value; |
| 16 | } ot_keywords; | 16 | } ot_keywords; |
| 17 | 17 | ||
| 18 | typedef enum { | 18 | typedef enum { |
diff --git a/tests/runner.c b/tests/runner.c index db62246..148b928 100644 --- a/tests/runner.c +++ b/tests/runner.c | |||
| @@ -23,6 +23,8 @@ static const struct module modules[] = {{"scan_urlencoded_query", test_scan_urle | |||
| 23 | {"ot_buf", test_buf}, | 23 | {"ot_buf", test_buf}, |
| 24 | {"ot_ip", test_ip}, | 24 | {"ot_ip", test_ip}, |
| 25 | {"ot_socket", test_socket}, | 25 | {"ot_socket", test_socket}, |
| 26 | {"ot_mutex", test_mutex}, | ||
| 27 | {"ot_batch", test_batch}, | ||
| 26 | {"ot_accesslist", test_accesslist}, | 28 | {"ot_accesslist", test_accesslist}, |
| 27 | {NULL, NULL}}; | 29 | {NULL, NULL}}; |
| 28 | 30 | ||
diff --git a/tests/stubs.c b/tests/stubs.c index ce93071..09a7312 100644 --- a/tests/stubs.c +++ b/tests/stubs.c | |||
| @@ -18,3 +18,12 @@ void free_peerlist(ot_peerlist *peer_list) { (void)peer_list; } | |||
| 18 | neither of which any test drives. */ | 18 | neither of which any test drives. */ |
| 19 | time_t g_now_seconds; | 19 | time_t g_now_seconds; |
| 20 | volatile int g_opentracker_running; | 20 | volatile int g_opentracker_running; |
| 21 | |||
| 22 | /* ot_mutex.c wakes the main loop through the self pipe when a result is | ||
| 23 | ready. The queue tests never get that far. */ | ||
| 24 | int g_self_pipe[2] = {-1, -1}; | ||
| 25 | long io_trywrite(long d, const char *buf, long len) { | ||
| 26 | (void)d; | ||
| 27 | (void)buf; | ||
| 28 | return len; | ||
| 29 | } | ||
diff --git a/tests/t_batch.c b/tests/t_batch.c new file mode 100644 index 0000000..02743f1 --- /dev/null +++ b/tests/t_batch.c | |||
| @@ -0,0 +1,348 @@ | |||
| 1 | /* This software was written by Dirk Engling <erdgeist@erdgeist.org> | ||
| 2 | It is considered beerware. Prost. Skol. Cheers or whatever. | ||
| 3 | |||
| 4 | $id$ */ | ||
| 5 | |||
| 6 | /* System */ | ||
| 7 | #include <fcntl.h> | ||
| 8 | #include <poll.h> | ||
| 9 | #include <stdlib.h> | ||
| 10 | #include <string.h> | ||
| 11 | #include <sys/socket.h> | ||
| 12 | #include <unistd.h> | ||
| 13 | |||
| 14 | /* Opentracker */ | ||
| 15 | #include "ot_batch.h" | ||
| 16 | #include "test.h" | ||
| 17 | |||
| 18 | /* The batch writes to a real socket pair, because what is worth pinning down | ||
| 19 | is what it does when the kernel takes only part of what it was offered. */ | ||
| 20 | |||
| 21 | static char *dup_of(const char *text) { | ||
| 22 | size_t len = strlen(text); | ||
| 23 | char *copy = malloc(len); | ||
| 24 | |||
| 25 | memcpy(copy, text, len); | ||
| 26 | return copy; | ||
| 27 | } | ||
| 28 | |||
| 29 | static void drain(int s, char *into, size_t len) { | ||
| 30 | size_t got = 0; | ||
| 31 | |||
| 32 | while (got < len) { | ||
| 33 | ssize_t r = read(s, into + got, len - got); | ||
| 34 | |||
| 35 | if (r <= 0) | ||
| 36 | break; | ||
| 37 | got += (size_t)r; | ||
| 38 | } | ||
| 39 | into[got] = 0; | ||
| 40 | } | ||
| 41 | |||
| 42 | static void case_queue_accounting(void) { | ||
| 43 | ot_batch batch; | ||
| 44 | |||
| 45 | memset(&batch, 0, sizeof(batch)); | ||
| 46 | |||
| 47 | test_case("a zeroed batch is empty and sends nothing"); | ||
| 48 | CHECK_INT(batch_pending(&batch), 0); | ||
| 49 | CHECK_INT(batch_send(-1, &batch), BATCH_DONE); | ||
| 50 | |||
| 51 | test_case("anything queued makes it pending"); | ||
| 52 | CHECK_INT(batch_add(&batch, dup_of("hello "), 6), 0); | ||
| 53 | CHECK_INT(batch_add_copy(&batch, "world", 5), 0); | ||
| 54 | CHECK_INT(batch_pending(&batch), 1); | ||
| 55 | CHECK_INT(batch.count, 2); | ||
| 56 | |||
| 57 | batch_reset(&batch); | ||
| 58 | test_case("and a reset empties it"); | ||
| 59 | CHECK_INT(batch_pending(&batch), 0); | ||
| 60 | CHECK_PTR(batch.entries, NULL); | ||
| 61 | } | ||
| 62 | |||
| 63 | static void case_send_in_one_go(void) { | ||
| 64 | ot_batch batch; | ||
| 65 | int sv[2]; | ||
| 66 | char got[32]; | ||
| 67 | |||
| 68 | memset(&batch, 0, sizeof(batch)); | ||
| 69 | CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0); | ||
| 70 | |||
| 71 | test_case("several buffers arrive as one stream, in order"); | ||
| 72 | CHECK_INT(batch_add(&batch, dup_of("opentracker"), 11), 0); | ||
| 73 | CHECK_INT(batch_add_copy(&batch, " says ", 6), 0); | ||
| 74 | CHECK_INT(batch_add(&batch, dup_of("hello"), 5), 0); | ||
| 75 | CHECK_INT(batch_send(sv[0], &batch), 22); | ||
| 76 | CHECK_INT(batch_pending(&batch), 0); | ||
| 77 | |||
| 78 | drain(sv[1], got, 22); | ||
| 79 | CHECK_STR(got, "opentracker says hello"); | ||
| 80 | |||
| 81 | test_case("and an empty batch reports it has nothing left"); | ||
| 82 | CHECK_INT(batch_send(sv[0], &batch), BATCH_DONE); | ||
| 83 | |||
| 84 | batch_reset(&batch); | ||
| 85 | close(sv[0]); | ||
| 86 | close(sv[1]); | ||
| 87 | } | ||
| 88 | |||
| 89 | /* Send everything the batch holds, reading the other end as we go so the | ||
| 90 | socket never stays full, and return what arrived. */ | ||
| 91 | static size_t pump(int writer, int reader, ot_batch *batch, char *into, size_t room) { | ||
| 92 | size_t got = 0; | ||
| 93 | |||
| 94 | while (batch_pending(batch) || got < room) { | ||
| 95 | ssize_t r; | ||
| 96 | |||
| 97 | if (batch_pending(batch)) { | ||
| 98 | ssize_t res = batch_send(writer, batch); | ||
| 99 | |||
| 100 | if (res == BATCH_FAILED) | ||
| 101 | break; | ||
| 102 | if (res > 0) | ||
| 103 | continue; | ||
| 104 | } | ||
| 105 | |||
| 106 | r = read(reader, into + got, room - got); | ||
| 107 | if (r <= 0) | ||
| 108 | break; | ||
| 109 | got += (size_t)r; | ||
| 110 | } | ||
| 111 | return got; | ||
| 112 | } | ||
| 113 | |||
| 114 | static void case_partial_write_resumes(void) { | ||
| 115 | const size_t size = 200000; | ||
| 116 | ot_batch batch; | ||
| 117 | int sv[2], sndbuf = 4096; | ||
| 118 | char *sent, *got; | ||
| 119 | size_t i, arrived; | ||
| 120 | |||
| 121 | memset(&batch, 0, sizeof(batch)); | ||
| 122 | CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0); | ||
| 123 | setsockopt(sv[0], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf)); | ||
| 124 | fcntl(sv[0], F_SETFL, fcntl(sv[0], F_GETFL, 0) | O_NONBLOCK); | ||
| 125 | |||
| 126 | sent = malloc(size); | ||
| 127 | got = malloc(size); | ||
| 128 | for (i = 0; i < size; ++i) | ||
| 129 | sent[i] = (char)(i % 251); | ||
| 130 | |||
| 131 | test_case("a buffer the kernel takes in pieces arrives whole and in order"); | ||
| 132 | CHECK_INT(batch_add(&batch, memcpy(malloc(size), sent, size), size), 0); | ||
| 133 | arrived = pump(sv[0], sv[1], &batch, got, size); | ||
| 134 | CHECK_INT(arrived, size); | ||
| 135 | CHECK_INT(batch_pending(&batch), 0); | ||
| 136 | CHECK_MEM(got, sent, size); | ||
| 137 | |||
| 138 | free(sent); | ||
| 139 | free(got); | ||
| 140 | batch_reset(&batch); | ||
| 141 | close(sv[0]); | ||
| 142 | close(sv[1]); | ||
| 143 | } | ||
| 144 | |||
| 145 | /* writev refuses more than IOV_MAX entries at once, so a long batch has to be | ||
| 146 | sent across several calls. */ | ||
| 147 | static void case_more_entries_than_one_writev(void) { | ||
| 148 | const size_t entries = 3000, each = 4; | ||
| 149 | ot_batch batch; | ||
| 150 | int sv[2], sndbuf = 4096; | ||
| 151 | char *sent, *got; | ||
| 152 | size_t i, arrived, refused = 0; | ||
| 153 | |||
| 154 | memset(&batch, 0, sizeof(batch)); | ||
| 155 | CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0); | ||
| 156 | setsockopt(sv[0], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf)); | ||
| 157 | fcntl(sv[0], F_SETFL, fcntl(sv[0], F_GETFL, 0) | O_NONBLOCK); | ||
| 158 | |||
| 159 | sent = malloc(entries * each); | ||
| 160 | got = malloc(entries * each); | ||
| 161 | |||
| 162 | test_case("a batch of more entries than writev accepts still arrives whole"); | ||
| 163 | for (i = 0; i < entries; ++i) { | ||
| 164 | char piece[4]; | ||
| 165 | |||
| 166 | piece[0] = (char)(i), piece[1] = (char)(i >> 8), piece[2] = 'x', piece[3] = 'y'; | ||
| 167 | memcpy(sent + i * each, piece, each); | ||
| 168 | if (batch_add(&batch, memcpy(malloc(each), piece, each), each)) | ||
| 169 | ++refused; | ||
| 170 | } | ||
| 171 | CHECK_INT(refused, 0); | ||
| 172 | CHECK_INT(batch.count, entries); | ||
| 173 | |||
| 174 | arrived = pump(sv[0], sv[1], &batch, got, entries * each); | ||
| 175 | CHECK_INT(arrived, entries * each); | ||
| 176 | CHECK_MEM(got, sent, entries * each); | ||
| 177 | |||
| 178 | free(sent); | ||
| 179 | free(got); | ||
| 180 | batch_reset(&batch); | ||
| 181 | close(sv[0]); | ||
| 182 | close(sv[1]); | ||
| 183 | } | ||
| 184 | |||
| 185 | static void case_reset_frees_the_unsent(void) { | ||
| 186 | ot_batch batch; | ||
| 187 | int sv[2]; | ||
| 188 | |||
| 189 | memset(&batch, 0, sizeof(batch)); | ||
| 190 | CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0); | ||
| 191 | |||
| 192 | test_case("a batch torn down mid flight keeps no bytes outstanding"); | ||
| 193 | CHECK_INT(batch_add(&batch, dup_of("first"), 5), 0); | ||
| 194 | CHECK_INT(batch_add(&batch, dup_of("second"), 6), 0); | ||
| 195 | CHECK_INT(batch_send(sv[0], &batch), 11); | ||
| 196 | CHECK_INT(batch_add(&batch, dup_of("never sent"), 10), 0); | ||
| 197 | CHECK_INT(batch_pending(&batch), 1); | ||
| 198 | |||
| 199 | /* Whether the unsent buffer is freed is for the sanitizer to say */ | ||
| 200 | batch_reset(&batch); | ||
| 201 | CHECK_INT(batch_pending(&batch), 0); | ||
| 202 | CHECK_INT(batch.count, 0); | ||
| 203 | |||
| 204 | close(sv[0]); | ||
| 205 | close(sv[1]); | ||
| 206 | } | ||
| 207 | |||
| 208 | static void case_closed_socket_is_fatal(void) { | ||
| 209 | ot_batch batch; | ||
| 210 | int sv[2]; | ||
| 211 | |||
| 212 | memset(&batch, 0, sizeof(batch)); | ||
| 213 | CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0); | ||
| 214 | close(sv[1]); | ||
| 215 | |||
| 216 | test_case("writing to a socket whose peer is gone reports a dead connection"); | ||
| 217 | CHECK_INT(batch_add(&batch, dup_of("nobody is listening"), 19), 0); | ||
| 218 | CHECK_INT(batch_send(sv[0], &batch), BATCH_FAILED); | ||
| 219 | |||
| 220 | batch_reset(&batch); | ||
| 221 | close(sv[0]); | ||
| 222 | } | ||
| 223 | |||
| 224 | static void case_vector_is_taken_whole(void) { | ||
| 225 | ot_batch batch; | ||
| 226 | struct iovec *vector = malloc(3 * sizeof(struct iovec)); | ||
| 227 | int sv[2]; | ||
| 228 | char got[32]; | ||
| 229 | |||
| 230 | memset(&batch, 0, sizeof(batch)); | ||
| 231 | CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0); | ||
| 232 | |||
| 233 | test_case("a whole vector is queued in one go, in order"); | ||
| 234 | vector[0].iov_base = dup_of("one "), vector[0].iov_len = 4; | ||
| 235 | vector[1].iov_base = dup_of("two "), vector[1].iov_len = 4; | ||
| 236 | vector[2].iov_base = dup_of("three"), vector[2].iov_len = 5; | ||
| 237 | CHECK_INT(batch_add_vector(&batch, vector, 3), 0); | ||
| 238 | free(vector); | ||
| 239 | CHECK_INT(batch.count, 3); | ||
| 240 | CHECK_INT(batch_send(sv[0], &batch), 13); | ||
| 241 | |||
| 242 | drain(sv[1], got, 13); | ||
| 243 | CHECK_STR(got, "one two three"); | ||
| 244 | |||
| 245 | batch_reset(&batch); | ||
| 246 | close(sv[0]); | ||
| 247 | close(sv[1]); | ||
| 248 | } | ||
| 249 | |||
| 250 | /* The entry array starts small and doubles, so crossing that boundary one | ||
| 251 | buffer at a time is worth its own case. */ | ||
| 252 | static void case_growing_one_entry_at_a_time(void) { | ||
| 253 | const size_t entries = 100; | ||
| 254 | ot_batch batch; | ||
| 255 | int sv[2]; | ||
| 256 | char sent[400], got[401]; | ||
| 257 | size_t i, refused = 0; | ||
| 258 | |||
| 259 | memset(&batch, 0, sizeof(batch)); | ||
| 260 | CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0); | ||
| 261 | |||
| 262 | test_case("a batch grown one buffer at a time keeps them all, in order"); | ||
| 263 | for (i = 0; i < entries; ++i) { | ||
| 264 | char piece[4]; | ||
| 265 | |||
| 266 | piece[0] = (char)('a' + i % 26), piece[1] = (char)('0' + i % 10); | ||
| 267 | piece[2] = (char)('A' + i % 26), piece[3] = '.'; | ||
| 268 | memcpy(sent + i * 4, piece, 4); | ||
| 269 | if (batch_add(&batch, memcpy(malloc(4), piece, 4), 4)) | ||
| 270 | ++refused; | ||
| 271 | } | ||
| 272 | CHECK_INT(refused, 0); | ||
| 273 | CHECK_INT(batch.count, entries); | ||
| 274 | CHECK_INT(batch_send(sv[0], &batch), (ssize_t)(entries * 4)); | ||
| 275 | |||
| 276 | drain(sv[1], got, entries * 4); | ||
| 277 | CHECK_MEM(got, sent, entries * 4); | ||
| 278 | |||
| 279 | batch_reset(&batch); | ||
| 280 | close(sv[0]); | ||
| 281 | close(sv[1]); | ||
| 282 | } | ||
| 283 | |||
| 284 | /* Appending while earlier entries are still on their way is what a chunked | ||
| 285 | transfer does. Rather than depend on when a kernel decides to block, this | ||
| 286 | takes entries off the front the way a completed send would and checks that | ||
| 287 | the array reuses the space instead of growing. */ | ||
| 288 | static void case_append_while_sending(void) { | ||
| 289 | const size_t per_round = 12, each = 8; | ||
| 290 | ot_batch batch; | ||
| 291 | int sv[2]; | ||
| 292 | char expected[16], got[17]; | ||
| 293 | size_t i, round, refused = 0, space_after_first = 0; | ||
| 294 | |||
| 295 | memset(&batch, 0, sizeof(batch)); | ||
| 296 | CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0); | ||
| 297 | |||
| 298 | test_case("an array whose front has gone out is reused, not grown"); | ||
| 299 | for (round = 0; round < 8; ++round) { | ||
| 300 | for (i = 0; i < per_round; ++i) { | ||
| 301 | /* Distinct per entry, so a move from the wrong index shows up */ | ||
| 302 | char *piece = malloc(each); | ||
| 303 | |||
| 304 | memset(piece, (int)('a' + (round * per_round + i) % 26), each); | ||
| 305 | piece[0] = (char)('0' + i); | ||
| 306 | piece[1] = (char)('0' + round); | ||
| 307 | if (batch_add(&batch, piece, each)) | ||
| 308 | ++refused; | ||
| 309 | } | ||
| 310 | |||
| 311 | /* All but the last entry have left and been freed */ | ||
| 312 | while (batch.count - batch.first > 1) | ||
| 313 | free(batch.entries[batch.first++].iov_base); | ||
| 314 | |||
| 315 | if (!round) | ||
| 316 | space_after_first = batch.space; | ||
| 317 | } | ||
| 318 | CHECK_INT(refused, 0); | ||
| 319 | CHECK_INT(batch.space, space_after_first); | ||
| 320 | |||
| 321 | test_case("and what is still queued survives the move, in order"); | ||
| 322 | CHECK_INT(batch.count - batch.first, 1); | ||
| 323 | CHECK_INT(batch_add(&batch, memset(malloc(each), 'Z', each), each), 0); | ||
| 324 | CHECK_INT(batch_send(sv[0], &batch), (ssize_t)(2 * each)); | ||
| 325 | |||
| 326 | memset(expected, (int)('a' + (7 * per_round + per_round - 1) % 26), each); | ||
| 327 | expected[0] = (char)('0' + per_round - 1); | ||
| 328 | expected[1] = '7'; | ||
| 329 | memset(expected + each, 'Z', each); | ||
| 330 | drain(sv[1], got, 2 * each); | ||
| 331 | CHECK_MEM(got, expected, 2 * each); | ||
| 332 | |||
| 333 | batch_reset(&batch); | ||
| 334 | close(sv[0]); | ||
| 335 | close(sv[1]); | ||
| 336 | } | ||
| 337 | |||
| 338 | void test_batch(void) { | ||
| 339 | case_queue_accounting(); | ||
| 340 | case_send_in_one_go(); | ||
| 341 | case_vector_is_taken_whole(); | ||
| 342 | case_growing_one_entry_at_a_time(); | ||
| 343 | case_append_while_sending(); | ||
| 344 | case_partial_write_resumes(); | ||
| 345 | case_more_entries_than_one_writev(); | ||
| 346 | case_reset_frees_the_unsent(); | ||
| 347 | case_closed_socket_is_fatal(); | ||
| 348 | } | ||
diff --git a/tests/t_mutex.c b/tests/t_mutex.c new file mode 100644 index 0000000..3307d70 --- /dev/null +++ b/tests/t_mutex.c | |||
| @@ -0,0 +1,107 @@ | |||
| 1 | /* This software was written by Dirk Engling <erdgeist@erdgeist.org> | ||
| 2 | It is considered beerware. Prost. Skol. Cheers or whatever. | ||
| 3 | |||
| 4 | $id$ */ | ||
| 5 | |||
| 6 | /* System */ | ||
| 7 | #include <stdlib.h> | ||
| 8 | #include <string.h> | ||
| 9 | |||
| 10 | /* Opentracker */ | ||
| 11 | #include "ot_mutex.h" | ||
| 12 | #include "test.h" | ||
| 13 | |||
| 14 | /* A worker asks for a class of task and blocks until one turns up, so a task | ||
| 15 | whose class matches no worker is never claimed and whoever is waiting for | ||
| 16 | its answer waits forever. These cases pin the matching. */ | ||
| 17 | |||
| 18 | static void case_task_goes_to_its_own_class(void) { | ||
| 19 | ot_tasktype asked; | ||
| 20 | ot_taskid taskid; | ||
| 21 | |||
| 22 | test_case("a worker is handed the task of the class it asked for"); | ||
| 23 | CHECK_INT(mutex_workqueue_pushtask(41, TASK_FULLSCRAPE_TPB_ASCII), 0); | ||
| 24 | asked = TASK_FULLSCRAPE; | ||
| 25 | taskid = mutex_workqueue_poptask(&asked); | ||
| 26 | CHECK(taskid != 0); | ||
| 27 | |||
| 28 | test_case("and the full task type, not just its class"); | ||
| 29 | CHECK_INT(asked, TASK_FULLSCRAPE_TPB_ASCII); | ||
| 30 | mutex_workqueue_canceltask(41); | ||
| 31 | } | ||
| 32 | |||
| 33 | static void case_flags_survive_the_queue(void) { | ||
| 34 | ot_tasktype asked; | ||
| 35 | |||
| 36 | test_case("flags outside the class mask are carried through"); | ||
| 37 | CHECK_INT(mutex_workqueue_pushtask(42, TASK_FULLSCRAPE_TPB_URLENCODED | TASK_FLAG_GZIP), 0); | ||
| 38 | asked = TASK_FULLSCRAPE; | ||
| 39 | CHECK(mutex_workqueue_poptask(&asked) != 0); | ||
| 40 | CHECK_INT(asked, TASK_FULLSCRAPE_TPB_URLENCODED | TASK_FLAG_GZIP); | ||
| 41 | mutex_workqueue_canceltask(42); | ||
| 42 | } | ||
| 43 | |||
| 44 | static void case_classes_do_not_cross(void) { | ||
| 45 | ot_tasktype asked; | ||
| 46 | ot_taskid taskid; | ||
| 47 | |||
| 48 | test_case("a stats worker passes over a fullscrape task"); | ||
| 49 | CHECK_INT(mutex_workqueue_pushtask(43, TASK_FULLSCRAPE_TPB_BINARY), 0); | ||
| 50 | CHECK_INT(mutex_workqueue_pushtask(44, TASK_STATS_PEERS), 0); | ||
| 51 | |||
| 52 | asked = TASK_STATS; | ||
| 53 | taskid = mutex_workqueue_poptask(&asked); | ||
| 54 | CHECK(taskid != 0); | ||
| 55 | CHECK_INT(asked, TASK_STATS_PEERS); | ||
| 56 | |||
| 57 | test_case("and leaves it for the one that wants it"); | ||
| 58 | asked = TASK_FULLSCRAPE; | ||
| 59 | taskid = mutex_workqueue_poptask(&asked); | ||
| 60 | CHECK(taskid != 0); | ||
| 61 | CHECK_INT(asked, TASK_FULLSCRAPE_TPB_BINARY); | ||
| 62 | |||
| 63 | mutex_workqueue_canceltask(43); | ||
| 64 | mutex_workqueue_canceltask(44); | ||
| 65 | } | ||
| 66 | |||
| 67 | static void case_every_task_gets_its_own_id(void) { | ||
| 68 | ot_tasktype asked; | ||
| 69 | ot_taskid first, second; | ||
| 70 | |||
| 71 | test_case("two queued tasks are handed out under different ids"); | ||
| 72 | CHECK_INT(mutex_workqueue_pushtask(45, TASK_FULLSCRAPE), 0); | ||
| 73 | CHECK_INT(mutex_workqueue_pushtask(46, TASK_FULLSCRAPE), 0); | ||
| 74 | |||
| 75 | asked = TASK_FULLSCRAPE; | ||
| 76 | first = mutex_workqueue_poptask(&asked); | ||
| 77 | asked = TASK_FULLSCRAPE; | ||
| 78 | second = mutex_workqueue_poptask(&asked); | ||
| 79 | CHECK(first != 0); | ||
| 80 | CHECK(second != 0); | ||
| 81 | CHECK(first != second); | ||
| 82 | |||
| 83 | mutex_workqueue_canceltask(45); | ||
| 84 | mutex_workqueue_canceltask(46); | ||
| 85 | } | ||
| 86 | |||
| 87 | static void case_cancelled_task_is_not_handed_out(void) { | ||
| 88 | ot_tasktype asked; | ||
| 89 | |||
| 90 | test_case("a cancelled task is gone before any worker sees it"); | ||
| 91 | CHECK_INT(mutex_workqueue_pushtask(47, TASK_FULLSCRAPE_TPB_ASCII_PLUS), 0); | ||
| 92 | CHECK_INT(mutex_workqueue_pushtask(48, TASK_FULLSCRAPE), 0); | ||
| 93 | mutex_workqueue_canceltask(47); | ||
| 94 | |||
| 95 | asked = TASK_FULLSCRAPE; | ||
| 96 | CHECK(mutex_workqueue_poptask(&asked) != 0); | ||
| 97 | CHECK_INT(asked, TASK_FULLSCRAPE); | ||
| 98 | mutex_workqueue_canceltask(48); | ||
| 99 | } | ||
| 100 | |||
| 101 | void test_mutex(void) { | ||
| 102 | case_task_goes_to_its_own_class(); | ||
| 103 | case_flags_survive_the_queue(); | ||
| 104 | case_classes_do_not_cross(); | ||
| 105 | case_every_task_gets_its_own_id(); | ||
| 106 | case_cancelled_task_is_not_handed_out(); | ||
| 107 | } | ||
diff --git a/tests/test.h b/tests/test.h index d79c4d2..fd574c2 100644 --- a/tests/test.h +++ b/tests/test.h | |||
| @@ -86,5 +86,7 @@ void test_scan(void); | |||
| 86 | void test_buf(void); | 86 | void test_buf(void); |
| 87 | void test_ip(void); | 87 | void test_ip(void); |
| 88 | void test_socket(void); | 88 | void test_socket(void); |
| 89 | void test_mutex(void); | ||
| 90 | void test_batch(void); | ||
| 89 | 91 | ||
| 90 | #endif | 92 | #endif |
diff --git a/trackerlogic.c b/trackerlogic.c index 52c8c18..74fea6d 100644 --- a/trackerlogic.c +++ b/trackerlogic.c | |||
| @@ -10,9 +10,6 @@ | |||
| 10 | #include <stdlib.h> | 10 | #include <stdlib.h> |
| 11 | #include <string.h> | 11 | #include <string.h> |
| 12 | 12 | ||
| 13 | /* Libowfat */ | ||
| 14 | #include "io.h" | ||
| 15 | #include "iob.h" | ||
| 16 | 13 | ||
| 17 | /* Opentracker */ | 14 | /* Opentracker */ |
| 18 | #include "ot_accesslist.h" | 15 | #include "ot_accesslist.h" |
| @@ -550,7 +547,7 @@ void trackerlogic_add_random_torrents(size_t amount) { | |||
| 550 | } | 547 | } |
| 551 | #endif | 548 | #endif |
| 552 | 549 | ||
| 553 | void exerr(char *message) { | 550 | void exerr(const char *message) { |
| 554 | fprintf(stderr, "%s\n", message); | 551 | fprintf(stderr, "%s\n", message); |
| 555 | exit(111); | 552 | exit(111); |
| 556 | } | 553 | } |
diff --git a/trackerlogic.h b/trackerlogic.h index aa10a54..42018c3 100644 --- a/trackerlogic.h +++ b/trackerlogic.h | |||
| @@ -181,7 +181,7 @@ struct ot_workstruct { | |||
| 181 | 181 | ||
| 182 | void trackerlogic_init(void); | 182 | void trackerlogic_init(void); |
| 183 | void trackerlogic_deinit(void); | 183 | void trackerlogic_deinit(void); |
| 184 | void exerr(char *message); | 184 | void exerr(const char *message); |
| 185 | 185 | ||
| 186 | /* add_peer_to_torrent does only release the torrent bucket if from_sync is set, | 186 | /* add_peer_to_torrent does only release the torrent bucket if from_sync is set, |
| 187 | otherwise it is released in return_peers_for_torrent */ | 187 | otherwise it is released in return_peers_for_torrent */ |
