summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile12
-rw-r--r--opentracker.c40
-rw-r--r--ot_accesslist.c8
-rw-r--r--ot_accesslist.h6
-rw-r--r--ot_batch.c158
-rw-r--r--ot_batch.h48
-rw-r--r--ot_config.c12
-rw-r--r--ot_config.h8
-rw-r--r--ot_fullscrape.c2
-rw-r--r--ot_http.c85
-rw-r--r--ot_http.h9
-rw-r--r--ot_stats.c2
-rw-r--r--ot_timeout.h39
-rw-r--r--proxy.c16
-rw-r--r--scan_urlencoded_query.h4
-rw-r--r--tests/runner.c2
-rw-r--r--tests/stubs.c9
-rw-r--r--tests/t_batch.c348
-rw-r--r--tests/t_mutex.c107
-rw-r--r--tests/test.h2
-rw-r--r--trackerlogic.c5
-rw-r--r--trackerlogic.h2
22 files changed, 789 insertions, 135 deletions
diff --git a/Makefile b/Makefile
index 55be5ba..c7105d4 100644
--- a/Makefile
+++ b/Makefile
@@ -64,13 +64,13 @@ GIT_VERSION=`sh -c 'command -v git >/dev/null && test -d .git && git rev-parse H
64OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage 64OPTS_debug=-D_DEBUG -g -ggdb # -pg -fprofile-arcs -ftest-coverage
65OPTS_production=-O3 65OPTS_production=-O3
66 66
67CFLAGS+=-I$(LIBOWFAT_HEADERS) -DGIT_VERSION=$(GIT_VERSION) -Wall -pipe -pthread -Wextra #-ansi -pedantic 67CFLAGS+=-I$(LIBOWFAT_HEADERS) -DGIT_VERSION=$(GIT_VERSION) -Wall -pipe -pthread -Wextra -Wwrite-strings #-ansi -pedantic
68LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread 68LDFLAGS+=-L$(LIBOWFAT_LIBRARY) -lowfat -pthread
69 69
70BINARY =opentracker 70BINARY =opentracker
71HEADERS=trackerlogic.h scan_urlencoded_query.h ot_buf.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h 71HEADERS=trackerlogic.h scan_urlencoded_query.h ot_batch.h ot_buf.h ot_hex.h ot_ip.h ot_config.h ot_scan.h ot_socket.h ot_mutex.h ot_random.h ot_stats.h ot_vector.h ot_clean.h ot_udp.h ot_iovec.h ot_fullscrape.h ot_accesslist.h ot_http.h ot_livesync.h ot_rijndael.h
72SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c ot_buf.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c 72SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c ot_batch.c ot_buf.c ot_hex.c ot_ip.c ot_config.c ot_scan.c ot_socket.c ot_mutex.c ot_random.c ot_stats.c ot_vector.c ot_clean.c ot_udp.c ot_iovec.c ot_fullscrape.c ot_accesslist.c ot_http.c ot_livesync.c ot_rijndael.c
73SOURCES_proxy=proxy.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c 73SOURCES_proxy=proxy.c ot_batch.c ot_vector.c ot_mutex.c ot_random.c ot_scan.c ot_ip.c ot_socket.c
74 74
75OBJECTS = $(SOURCES:%.c=%.o) 75OBJECTS = $(SOURCES:%.c=%.o)
76OBJECTS_debug = $(SOURCES:%.c=%.debug.o) 76OBJECTS_debug = $(SOURCES:%.c=%.debug.o)
@@ -119,8 +119,8 @@ $(OBJECTS_proxy_debug): $(HEADERS)
119# has to. tests/stubs.c supplies the few symbols those modules reference but 119# has to. tests/stubs.c supplies the few symbols those modules reference but
120# the tests never reach. 120# the tests never reach.
121TESTBIN=tests/runner 121TESTBIN=tests/runner
122TEST_SOURCES=tests/runner.c tests/test.c tests/stubs.c tests/t_scan_urlencoded_query.c tests/t_iovec.c tests/t_vector.c tests/t_random.c tests/t_hex.c tests/t_accesslist.c tests/t_scan.c tests/t_buf.c tests/t_ip.c tests/t_socket.c 122TEST_SOURCES=tests/runner.c tests/test.c tests/stubs.c tests/t_scan_urlencoded_query.c tests/t_iovec.c tests/t_vector.c tests/t_random.c tests/t_hex.c tests/t_accesslist.c tests/t_scan.c tests/t_buf.c tests/t_ip.c tests/t_socket.c tests/t_mutex.c tests/t_batch.c
123TEST_UNITS=scan_urlencoded_query.c ot_buf.c ot_ip.c ot_socket.c ot_iovec.c ot_vector.c ot_random.c ot_hex.c ot_scan.c ot_accesslist.c 123TEST_UNITS=scan_urlencoded_query.c ot_batch.c ot_buf.c ot_ip.c ot_socket.c ot_iovec.c ot_vector.c ot_random.c ot_hex.c ot_scan.c ot_accesslist.c ot_mutex.c
124TEST_HEADERS=tests/test.h $(HEADERS) 124TEST_HEADERS=tests/test.h $(HEADERS)
125# Tests reach code that sits behind feature flags, so the units under test are 125# Tests reach code that sits behind feature flags, so the units under test are
126# built with those features enabled. A test binary is not a production binary. 126# built with those features enabled. A test binary is not a production binary.
diff --git a/opentracker.c b/opentracker.c
index 0af0d17..60bdceb 100644
--- a/opentracker.c
+++ b/opentracker.c
@@ -23,7 +23,6 @@
23 23
24/* Libowfat */ 24/* Libowfat */
25#include "io.h" 25#include "io.h"
26#include "iob.h"
27 26
28/* Opentracker */ 27/* Opentracker */
29#include "ot_accesslist.h" 28#include "ot_accesslist.h"
@@ -36,6 +35,7 @@
36#include "ot_random.h" 35#include "ot_random.h"
37#include "ot_scan.h" 36#include "ot_scan.h"
38#include "ot_socket.h" 37#include "ot_socket.h"
38#include "ot_timeout.h"
39#include "ot_stats.h" 39#include "ot_stats.h"
40#include "ot_udp.h" 40#include "ot_udp.h"
41#include "trackerlogic.h" 41#include "trackerlogic.h"
@@ -120,9 +120,8 @@ static ssize_t header_complete(char *request, ssize_t byte_count) {
120static void handle_dead(const int64_t sock) { 120static void handle_dead(const int64_t sock) {
121 struct http_data *cookie = io_getcookie(sock); 121 struct http_data *cookie = io_getcookie(sock);
122 if (cookie) { 122 if (cookie) {
123 size_t i; 123 if (cookie->batch)
124 for (i = 0; i < cookie->batches; ++i) 124 batch_reset(cookie->batch);
125 iob_reset(cookie->batch + i);
126 free(cookie->batch); 125 free(cookie->batch);
127 buf_reset(&cookie->request); 126 buf_reset(&cookie->request);
128 if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) 127 if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER))
@@ -178,33 +177,23 @@ static void handle_read(const int64_t sock, struct ot_workstruct *ws) {
178 177
179static void handle_write(const int64_t sock) { 178static void handle_write(const int64_t sock) {
180 struct http_data *cookie = io_getcookie(sock); 179 struct http_data *cookie = io_getcookie(sock);
181 size_t i;
182 int chunked = 0; 180 int chunked = 0;
183 181
184 /* Look for the first io_batch still containing bytes to write */
185 if (cookie) { 182 if (cookie) {
186 if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) 183 if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)
187 chunked = 1; 184 chunked = 1;
188 185
189 for (i = 0; i < cookie->batches; ++i) { 186 if (cookie->batch) {
190 if (cookie->batch[i].bytesleft) { 187 if (batch_send(sock, cookie->batch) == BATCH_FAILED) {
191 int64_t res = iob_send(sock, cookie->batch + i); 188 handle_dead(sock);
192 189 return;
193 if (res == -3) {
194 handle_dead(sock);
195 return;
196 }
197
198 if (!cookie->batch[i].bytesleft)
199 continue;
200
201 if (res == -1 || res > 0 || i < cookie->batches - 1)
202 return;
203 } 190 }
191 if (batch_pending(cookie->batch))
192 return;
204 } 193 }
205 } 194 }
206 195
207 /* In a chunked transfer after all batches accumulated have been sent, wait for the next one */ 196 /* In a chunked transfer, wait for the next piece once everything queued has gone out */
208 if (chunked) 197 if (chunked)
209 io_dontwantwrite(sock); 198 io_dontwantwrite(sock);
210 else 199 else
@@ -216,7 +205,6 @@ static void handle_accept(const int64_t serversocket) {
216 int64_t sock; 205 int64_t sock;
217 ot_ip6 ip; 206 ot_ip6 ip;
218 uint16_t port; 207 uint16_t port;
219 tai6464 t;
220 208
221 while ((sock = socket_accept(serversocket, ip, &port)) != -1) { 209 while ((sock = socket_accept(serversocket, ip, &port)) != -1) {
222 210
@@ -235,11 +223,7 @@ static void handle_accept(const int64_t serversocket) {
235 223
236 stats_issue_event(EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip); 224 stats_issue_event(EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip);
237 225
238 /* That breaks taia encapsulation. But there is no way to take system 226 timeout_in(sock, OT_CLIENT_TIMEOUT);
239 time this often in FreeBSD and libowfat does not allow to set unix time */
240 taia_uint(&t, 0); /* Clear t */
241 tai_unix(&(t.sec), (g_now_seconds + OT_CLIENT_TIMEOUT));
242 io_timeout(sock, t);
243 } 227 }
244 io_eagain(serversocket); 228 io_eagain(serversocket);
245} 229}
@@ -305,7 +289,7 @@ static void listen_bind_one(ot_listen *entry) {
305 289
306#ifdef _DEBUG 290#ifdef _DEBUG
307 { 291 {
308 char *protos[] = {"TCP", "UDP", "UDP mcast"}; 292 const char *protos[] = {"TCP", "UDP", "UDP mcast"};
309 char _debug[512]; 293 char _debug[512];
310 int off = snprintf(_debug, sizeof(_debug), "Binding socket type %s to address [", protos[entry->proto]); 294 int off = snprintf(_debug, sizeof(_debug), "Binding socket type %s to address [", protos[entry->proto]);
311 off += ip_format(_debug + off, entry->ip); 295 off += ip_format(_debug + off, entry->ip);
diff --git a/ot_accesslist.c b/ot_accesslist.c
index 2465ada..d7e88ee 100644
--- a/ot_accesslist.c
+++ b/ot_accesslist.c
@@ -32,10 +32,10 @@
32 32
33/* GLOBAL VARIABLES */ 33/* GLOBAL VARIABLES */
34#ifdef WANT_ACCESSLIST 34#ifdef WANT_ACCESSLIST
35char *g_accesslist_filename = NULL; 35const char *g_accesslist_filename = NULL;
36#ifdef WANT_DYNAMIC_ACCESSLIST 36#ifdef WANT_DYNAMIC_ACCESSLIST
37char *g_accesslist_pipe_add = NULL; 37const char *g_accesslist_pipe_add = NULL;
38char *g_accesslist_pipe_delete = NULL; 38const char *g_accesslist_pipe_delete = NULL;
39#endif 39#endif
40static pthread_mutex_t g_accesslist_mutex; 40static pthread_mutex_t g_accesslist_mutex;
41 41
@@ -272,7 +272,7 @@ static void *accesslist_worker(void *args) {
272 272
273#ifdef WANT_DYNAMIC_ACCESSLIST 273#ifdef WANT_DYNAMIC_ACCESSLIST
274static pthread_t thread_adder_id, thread_deleter_id; 274static pthread_t thread_adder_id, thread_deleter_id;
275static void *accesslist_adddel_worker(char *fifoname, ot_accesslist *_Atomic *adding_to, ot_accesslist *_Atomic *removing_from) { 275static void *accesslist_adddel_worker(const char *fifoname, ot_accesslist *_Atomic *adding_to, ot_accesslist *_Atomic *removing_from) {
276 struct stat st; 276 struct stat st;
277 277
278 if (!stat(fifoname, &st)) { 278 if (!stat(fifoname, &st)) {
diff --git a/ot_accesslist.h b/ot_accesslist.h
index 30aaebd..9dfdd95 100644
--- a/ot_accesslist.h
+++ b/ot_accesslist.h
@@ -23,10 +23,10 @@ void accesslist_cleanup(void);
23 line, and return how many were written to dest */ 23 line, and return how many were written to dest */
24size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len); 24size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len);
25 25
26extern char *g_accesslist_filename; 26extern const char *g_accesslist_filename;
27#ifdef WANT_DYNAMIC_ACCESSLIST 27#ifdef WANT_DYNAMIC_ACCESSLIST
28extern char *g_accesslist_pipe_add; 28extern const char *g_accesslist_pipe_add;
29extern char *g_accesslist_pipe_delete; 29extern const char *g_accesslist_pipe_delete;
30#endif 30#endif
31 31
32#else 32#else
diff --git a/ot_batch.c b/ot_batch.c
new file mode 100644
index 0000000..84eb4a8
--- /dev/null
+++ b/ot_batch.c
@@ -0,0 +1,158 @@
1/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
2 It is considered beerware. Prost. Skol. Cheers or whatever.
3
4 $id$ */
5
6/* System */
7#include <errno.h>
8#include <limits.h>
9#include <stdlib.h>
10#include <string.h>
11#include <sys/socket.h>
12#include <sys/uio.h>
13#include <unistd.h>
14
15/* Opentracker */
16#include "ot_batch.h"
17
18#ifndef IOV_MAX
19#define IOV_MAX 1024
20#endif
21
22#define OT_BATCH_MIN_ENTRIES 16
23
24static int batch_room_for(ot_batch *batch, size_t wanted) {
25 size_t space = batch->space ? batch->space : OT_BATCH_MIN_ENTRIES;
26 struct iovec *grown;
27
28 if (batch->count + wanted <= batch->space)
29 return 0;
30
31 /* During a chunked transfer entries are appended while earlier ones are
32 still going out, so the front of the array frees up as we go. Slide the
33 rest down before asking for more. */
34 if (batch->first && batch->count - batch->first + wanted <= batch->space) {
35 memmove(batch->entries, batch->entries + batch->first, (batch->count - batch->first) * sizeof(struct iovec));
36 batch->count -= batch->first;
37 batch->first = 0;
38 return 0;
39 }
40
41 while (space < batch->count + wanted)
42 space *= 2;
43
44 if (!(grown = realloc(batch->entries, space * sizeof(struct iovec))))
45 return -1;
46
47 batch->entries = grown;
48 batch->space = space;
49 return 0;
50}
51
52int batch_add(ot_batch *batch, void *buf, size_t len) {
53 if (batch_room_for(batch, 1)) {
54 free(buf);
55 return -1;
56 }
57
58 batch->entries[batch->count].iov_base = buf;
59 batch->entries[batch->count].iov_len = len;
60 ++batch->count;
61 return 0;
62}
63
64int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count) {
65 if (batch_room_for(batch, count)) {
66 while (count--)
67 free(entries[count].iov_base);
68 return -1;
69 }
70
71 memcpy(batch->entries + batch->count, entries, count * sizeof(struct iovec));
72 batch->count += count;
73 return 0;
74}
75
76/* Drop everything the kernel accepted, freeing each buffer as it goes */
77static void batch_consume(ot_batch *batch, size_t written) {
78 while (written) {
79 struct iovec *entry = batch->entries + batch->first;
80 size_t left = entry->iov_len - batch->offset;
81
82 if (written < left) {
83 batch->offset += written;
84 return;
85 }
86
87 free(entry->iov_base);
88 written -= left;
89 batch->offset = 0;
90 ++batch->first;
91 }
92
93 /* Nothing is outstanding, so the array can start over from the front */
94 if (batch->first == batch->count)
95 batch->first = batch->count = 0;
96}
97
98/* A peer that has gone away must come back as an error rather than as a
99 signal that kills the tracker, so ask the kernel not to raise one where it
100 can be asked. macOS cannot, and relies on SIGPIPE being blocked. */
101static ssize_t batch_writev(int s, struct iovec *entries, int count) {
102#ifdef MSG_NOSIGNAL
103 struct msghdr msg;
104
105 memset(&msg, 0, sizeof(msg));
106 msg.msg_iov = entries;
107 msg.msg_iovlen = count;
108 return sendmsg(s, &msg, MSG_NOSIGNAL);
109#else
110 return writev(s, entries, count);
111#endif
112}
113
114ssize_t batch_send(int s, ot_batch *batch) {
115 size_t entries = batch->count - batch->first;
116 ssize_t written;
117
118 if (!batch_pending(batch))
119 return BATCH_DONE;
120
121 if (entries > IOV_MAX)
122 entries = IOV_MAX;
123
124 /* The first entry may be half way out already */
125 batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base + batch->offset;
126 batch->entries[batch->first].iov_len -= batch->offset;
127 written = batch_writev(s, batch->entries + batch->first, (int)entries);
128 batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base - batch->offset;
129 batch->entries[batch->first].iov_len += batch->offset;
130
131 if (written < 0)
132 return (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) ? BATCH_AGAIN : BATCH_FAILED;
133 if (!written)
134 return BATCH_FAILED;
135
136 batch_consume(batch, (size_t)written);
137 return written;
138}
139
140void batch_reset(ot_batch *batch) {
141 while (batch->first < batch->count)
142 free(batch->entries[batch->first++].iov_base);
143
144 free(batch->entries);
145 batch->entries = NULL;
146 batch->count = batch->first = batch->offset = batch->space = 0;
147}
148
149/* For the few constants the chunked encoding needs, which the batch frees
150 along with everything else */
151int batch_add_copy(ot_batch *batch, const void *buf, size_t len) {
152 void *copy = malloc(len);
153
154 if (!copy)
155 return -1;
156 memcpy(copy, buf, len);
157 return batch_add(batch, copy, len);
158}
diff --git a/ot_batch.h b/ot_batch.h
new file mode 100644
index 0000000..c855671
--- /dev/null
+++ b/ot_batch.h
@@ -0,0 +1,48 @@
1/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
2 It is considered beerware. Prost. Skol. Cheers or whatever.
3
4 $id$ */
5
6#ifndef OT_BATCH_H__
7#define OT_BATCH_H__
8
9#include <stddef.h>
10#include <sys/types.h>
11#include <sys/uio.h>
12
13/* A queue of buffers on their way to one socket. The batch owns every buffer
14 handed to it and frees each one as soon as it has left, so a client that
15 reads slowly costs memory only for what it has not received yet.
16
17 A zeroed ot_batch is ready to use. */
18typedef struct {
19 struct iovec *entries;
20 size_t space;
21 size_t count;
22 size_t first; /* entries before this one are sent and freed */
23 size_t offset; /* bytes of entries[first] already sent */
24} ot_batch;
25
26static inline int batch_pending(const ot_batch *batch) { return batch->first != batch->count; }
27
28/* What batch_send reports instead of a byte count */
29enum { BATCH_DONE = 0, BATCH_AGAIN = -1, BATCH_FAILED = -2 };
30
31/* Queue buf and take ownership of it. Returns 0, or -1 having freed buf. */
32int batch_add(ot_batch *batch, void *buf, size_t len);
33
34/* Write as much as the socket takes. Returns the number of bytes written, or
35 BATCH_DONE, BATCH_AGAIN or BATCH_FAILED. */
36ssize_t batch_send(int s, ot_batch *batch);
37
38/* Queue every entry of a vector, taking ownership of each buffer but not of
39 the vector itself. Returns 0, or -1 having freed all of them. */
40int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count);
41
42/* Queue a copy of buf, which the batch then owns. Returns 0, or -1. */
43int batch_add_copy(ot_batch *batch, const void *buf, size_t len);
44
45/* Free whatever has not left yet, along with the batch's own array */
46void batch_reset(ot_batch *batch);
47
48#endif
diff --git a/ot_config.c b/ot_config.c
index 43c07c5..e9666f9 100644
--- a/ot_config.c
+++ b/ot_config.c
@@ -22,10 +22,10 @@
22#include "ot_stats.h" 22#include "ot_stats.h"
23#include "trackerlogic.h" 23#include "trackerlogic.h"
24 24
25char *g_redirecturl; 25const char *g_redirecturl;
26char *g_serverdir; 26const char *g_serverdir;
27char *g_serveruser; 27const char *g_serveruser;
28char *g_statefile; 28const char *g_statefile;
29 29
30#define OT_LISTEN_INHERIT (-1) 30#define OT_LISTEN_INHERIT (-1)
31 31
@@ -122,13 +122,13 @@ static int listen_scan_options(char *p, int *shared, int *workers) {
122} 122}
123 123
124 124
125char *set_config_option(char **option, char *value) { 125static const char *set_config_option(const char **option, char *value) {
126#ifdef _DEBUG 126#ifdef _DEBUG
127 fprintf(stderr, "Setting config option: %s\n", value); 127 fprintf(stderr, "Setting config option: %s\n", value);
128#endif 128#endif
129 while (isspace(*value)) 129 while (isspace(*value))
130 ++value; 130 ++value;
131 free(*option); 131 free((char *)*option);
132 return *option = strdup(value); 132 return *option = strdup(value);
133} 133}
134 134
diff --git a/ot_config.h b/ot_config.h
index ab25e2c..059573a 100644
--- a/ot_config.h
+++ b/ot_config.h
@@ -22,10 +22,10 @@ typedef struct {
22 int64_t sock; 22 int64_t sock;
23} ot_listen; 23} ot_listen;
24 24
25extern char *g_redirecturl; 25extern const char *g_redirecturl;
26extern char *g_serverdir; 26extern const char *g_serverdir;
27extern char *g_serveruser; 27extern const char *g_serveruser;
28extern char *g_statefile; 28extern const char *g_statefile;
29 29
30/* Reads the command line and every config file it names. Exits on error. */ 30/* Reads the command line and every config file it names. Exits on error. */
31void config_parse(int argc, char **argv); 31void config_parse(int argc, char **argv);
diff --git a/ot_fullscrape.c b/ot_fullscrape.c
index 9dcbe68..eade79d 100644
--- a/ot_fullscrape.c
+++ b/ot_fullscrape.c
@@ -83,7 +83,7 @@ void fullscrape_deinit(void) {
83} 83}
84 84
85void fullscrape_deliver( int64_t sock, ot_tasktype tasktype ) { 85void fullscrape_deliver( int64_t sock, ot_tasktype tasktype ) {
86 mutex_workqueue_pushtask( sock, tasktype ); 86 mutex_workqueue_pushtask( sock, TASK_FULLSCRAPE | tasktype );
87} 87}
88 88
89static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_torrent *torrent, ot_hash *hash ) { 89static char * fullscrape_write_one( ot_tasktype mode, char *r, ot_torrent *torrent, ot_hash *hash ) {
diff --git a/ot_http.c b/ot_http.c
index 46ca5a9..331a457 100644
--- a/ot_http.c
+++ b/ot_http.c
@@ -15,7 +15,7 @@
15#include <unistd.h> 15#include <unistd.h>
16 16
17/* Libowfat */ 17/* Libowfat */
18#include "iob.h" 18#include "io.h"
19 19
20/* Opentracker */ 20/* Opentracker */
21#include "ot_accesslist.h" 21#include "ot_accesslist.h"
@@ -26,20 +26,14 @@
26#include "ot_mutex.h" 26#include "ot_mutex.h"
27#include "ot_scan.h" 27#include "ot_scan.h"
28#include "ot_stats.h" 28#include "ot_stats.h"
29#include "ot_timeout.h"
29#include "scan_urlencoded_query.h" 30#include "scan_urlencoded_query.h"
30#include "trackerlogic.h" 31#include "trackerlogic.h"
31 32
32#ifdef WANT_NO_AUTO_FREE
33#define OT_IOB_INIT(B) (bzero(B, sizeof(io_batch)), 0)
34#else
35#define OT_IOB_INIT(B) iob_init_autofree(B, 0)
36#endif
37
38#define OT_MAXMULTISCRAPE_COUNT 64 33#define OT_MAXMULTISCRAPE_COUNT 64
39#define OT_BATCH_LIMIT (1024 * 1024 * 16) 34extern const char *g_redirecturl;
40extern char *g_redirecturl;
41 35
42char *g_stats_path; 36const char *g_stats_path;
43ssize_t g_stats_path_len; 37ssize_t g_stats_path_len;
44 38
45enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 }; 39enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 };
@@ -79,7 +73,6 @@ static void http_senddata(const int64_t sock, struct ot_workstruct *ws) {
79 73
80 if (written_size < ws->reply_size) { 74 if (written_size < ws->reply_size) {
81 char *outbuf; 75 char *outbuf;
82 tai6464 t;
83 76
84 if (!(outbuf = malloc(ws->reply_size - written_size))) { 77 if (!(outbuf = malloc(ws->reply_size - written_size))) {
85 buf_reset(&cookie->request); 78 buf_reset(&cookie->request);
@@ -90,25 +83,21 @@ static void http_senddata(const int64_t sock, struct ot_workstruct *ws) {
90 83
91 memcpy(outbuf, ws->reply + written_size, ws->reply_size - written_size); 84 memcpy(outbuf, ws->reply + written_size, ws->reply_size - written_size);
92 if (!cookie->batch) { 85 if (!cookie->batch) {
93 cookie->batch = malloc(sizeof(io_batch)); 86 cookie->batch = calloc(1, sizeof(ot_batch));
94 if (!cookie->batch || OT_IOB_INIT(cookie->batch) == -1) { 87 if (!cookie->batch) {
95 free(cookie->batch);
96 free(outbuf); 88 free(outbuf);
97 buf_reset(&cookie->request); 89 buf_reset(&cookie->request);
98 free(cookie); 90 free(cookie);
99 io_close(sock); 91 io_close(sock);
100 return; 92 return;
101 } 93 }
102
103 cookie->batches = 1;
104 } 94 }
105 95
106 iob_addbuf_free(cookie->batch, outbuf, ws->reply_size - written_size); 96 batch_add(cookie->batch, outbuf, ws->reply_size - written_size);
107 97
108 /* writeable short data sockets just have a tcp timeout */ 98 /* writeable short data sockets just have a tcp timeout */
109 if (!ws->keep_alive) { 99 if (!ws->keep_alive) {
110 taia_uint(&t, 0); 100 timeout_never(sock);
111 io_timeout(sock, t);
112 io_dontwantread(sock); 101 io_dontwantread(sock);
113 } 102 }
114 io_wantwrite(sock); 103 io_wantwrite(sock);
@@ -125,9 +114,9 @@ static void http_senddata(const int64_t sock, struct ot_workstruct *ws) {
125#define HTTPERROR_404 return http_issue_error(sock, ws, CODE_HTTPERROR_404) 114#define HTTPERROR_404 return http_issue_error(sock, ws, CODE_HTTPERROR_404)
126#define HTTPERROR_500 return http_issue_error(sock, ws, CODE_HTTPERROR_500) 115#define HTTPERROR_500 return http_issue_error(sock, ws, CODE_HTTPERROR_500)
127ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code) { 116ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code) {
128 char *error_code[] = {"302 Found", "400 Invalid Request", "400 Invalid Request", "400 Invalid Request", "402 Payment Required", 117 const char *error_code[] = {"302 Found", "400 Invalid Request", "400 Invalid Request", "400 Invalid Request", "402 Payment Required",
129 "403 Not Modest", "403 Access Denied", "404 Not Found", "500 Internal Server Error"}; 118 "403 Not Modest", "403 Access Denied", "404 Not Found", "500 Internal Server Error"};
130 char *title = error_code[code]; 119 const char *title = error_code[code];
131 120
132 ws->reply = ws->outbuf; 121 ws->reply = ws->outbuf;
133 if (code == CODE_HTTPERROR_302) 122 if (code == CODE_HTTPERROR_302)
@@ -146,11 +135,9 @@ ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code)
146 135
147ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial) { 136ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial) {
148 struct http_data *cookie = io_getcookie(sock); 137 struct http_data *cookie = io_getcookie(sock);
149 io_batch *current;
150 char *header; 138 char *header;
151 const char *encoding = ""; 139 const char *encoding = "";
152 size_t i, header_size, size = iovec_length(&iovec_entries, (const struct iovec **)&iovector); 140 size_t header_size, size = iovec_length(&iovec_entries, (const struct iovec **)&iovector);
153 tai6464 t;
154 141
155 /* No cookie? Bad socket. Leave. */ 142 /* No cookie? Bad socket. Leave. */
156 if (!cookie) { 143 if (!cookie) {
@@ -199,46 +186,28 @@ ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t
199 } 186 }
200 187
201 if (!cookie->batch) { 188 if (!cookie->batch) {
202 cookie->batch = malloc(sizeof(io_batch)); 189 cookie->batch = calloc(1, sizeof(ot_batch));
203 if (!cookie->batch || OT_IOB_INIT(cookie->batch) == -1) { 190 if (!cookie->batch) {
204 free(cookie->batch);
205 free(header); 191 free(header);
206 iovec_free(&iovec_entries, &iovector); 192 iovec_free(&iovec_entries, &iovector);
207 HTTPERROR_500; 193 HTTPERROR_500;
208 } 194 }
209 cookie->batches = 1;
210 } 195 }
211 current = cookie->batch + cookie->batches - 1;
212 iob_addbuf_free(current, header, header_size);
213 196
214 /* Split huge iovectors into separate io_batches */ 197 batch_add(cookie->batch, header, header_size);
215 for (i = 0; i < (size_t)iovec_entries; ++i) { 198 batch_add_vector(cookie->batch, iovector, iovec_entries);
216 /* If the current batch's limit is reached, try to reallocate a new batch to work on */
217 if (current->bytesleft > OT_BATCH_LIMIT) {
218 io_batch *new_batch = realloc(cookie->batch, (cookie->batches + 1) * sizeof(io_batch));
219 if (new_batch) {
220 cookie->batch = new_batch;
221 if (OT_IOB_INIT(current) != -1)
222 current = cookie->batch + cookie->batches++;
223 }
224 }
225 iob_addbuf_free(current, iovector[i].iov_base, iovector[i].iov_len);
226 }
227 free(iovector); 199 free(iovector);
200
228 if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) 201 if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)
229 iob_addbuf(current, "\r\n", 2); 202 batch_add_copy(cookie->batch, "\r\n", 2);
230 } 203 }
231 204
232 if ((cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) && cookie->batch && !is_partial) { 205 if ((cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) && cookie->batch && !is_partial) {
233 current = cookie->batch + cookie->batches - 1; 206 batch_add_copy(cookie->batch, "0\r\n\r\n", 5);
234 iob_addbuf(current, "0\r\n\r\n", 5);
235 cookie->flag &= ~STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; 207 cookie->flag &= ~STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER;
236 } 208 }
237 209
238 /* writeable sockets timeout after 10 minutes */ 210 timeout_in(sock, OT_CLIENT_TIMEOUT_SEND);
239 taia_now(&t);
240 taia_addsec(&t, &t, OT_CLIENT_TIMEOUT_SEND);
241 io_timeout(sock, t);
242 io_wantwrite(sock); 211 io_wantwrite(sock);
243 return 0; 212 return 0;
244} 213}
@@ -317,7 +286,6 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c
317 } 286 }
318 287
319 if (mode == TASK_STATS_TPB) { 288 if (mode == TASK_STATS_TPB) {
320 tai6464 t;
321#ifdef WANT_COMPRESSION_GZIP 289#ifdef WANT_COMPRESSION_GZIP
322 ws->request[ws->request_size - 1] = 0; 290 ws->request[ws->request_size - 1] = 0;
323#ifndef WANT_COMPRESSION_GZIP_ALWAYS 291#ifndef WANT_COMPRESSION_GZIP_ALWAYS
@@ -333,8 +301,7 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c
333 cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; 301 cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED;
334 302
335 /* Clients waiting for us should not easily timeout */ 303 /* Clients waiting for us should not easily timeout */
336 taia_uint(&t, 0); 304 timeout_never(sock);
337 io_timeout(sock, t);
338 fullscrape_deliver(sock, format); 305 fullscrape_deliver(sock, format);
339 io_dontwantread(sock); 306 io_dontwantread(sock);
340 return ws->reply_size = -2; 307 return ws->reply_size = -2;
@@ -343,13 +310,11 @@ static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, c
343 310
344 /* default format for now */ 311 /* default format for now */
345 if ((mode & TASK_CLASS_MASK) == TASK_STATS) { 312 if ((mode & TASK_CLASS_MASK) == TASK_STATS) {
346 tai6464 t;
347 313
348 cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK; 314 cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK;
349 315
350 /* Complex stats also include expensive memory debugging tools */ 316 /* Complex stats also include expensive memory debugging tools */
351 taia_uint(&t, 0); 317 timeout_never(sock);
352 io_timeout(sock, t);
353 stats_deliver(sock, mode); 318 stats_deliver(sock, mode);
354 return ws->reply_size = -2; 319 return ws->reply_size = -2;
355 } 320 }
@@ -371,7 +336,6 @@ typedef struct {
371static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *ws) { 336static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *ws) {
372 struct http_data *cookie = io_getcookie(sock); 337 struct http_data *cookie = io_getcookie(sock);
373 int format = 0; 338 int format = 0;
374 tai6464 t;
375 339
376#ifdef WANT_MODEST_FULLSCRAPES 340#ifdef WANT_MODEST_FULLSCRAPES
377 { 341 {
@@ -430,9 +394,8 @@ static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *
430 /* Pass this task to the worker thread */ 394 /* Pass this task to the worker thread */
431 cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; 395 cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED;
432 /* Clients waiting for us should not easily timeout */ 396 /* Clients waiting for us should not easily timeout */
433 taia_uint(&t, 0); 397 timeout_never(sock);
434 io_timeout(sock, t); 398 fullscrape_deliver(sock, format);
435 fullscrape_deliver(sock, TASK_FULLSCRAPE | format);
436 io_dontwantread(sock); 399 io_dontwantread(sock);
437 return ws->reply_size = -2; 400 return ws->reply_size = -2;
438} 401}
@@ -491,7 +454,7 @@ unsigned long long numwants[201];
491#endif 454#endif
492 455
493#if defined(WANT_KEEPALIVE) || defined(WANT_IP_FROM_PROXY) 456#if defined(WANT_KEEPALIVE) || defined(WANT_IP_FROM_PROXY)
494static char *http_header(char *data, size_t byte_count, char *header) { 457static char *http_header(char *data, size_t byte_count, const char *header) {
495 size_t i; 458 size_t i;
496 long sl = strlen(header); 459 long sl = strlen(header);
497 for (i = 0; i + sl + 2 < byte_count; ++i) { 460 for (i = 0; i + sl + 2 < byte_count; ++i) {
diff --git a/ot_http.h b/ot_http.h
index 965820f..003b545 100644
--- a/ot_http.h
+++ b/ot_http.h
@@ -10,9 +10,7 @@
10#include <sys/types.h> 10#include <sys/types.h>
11#include <sys/uio.h> 11#include <sys/uio.h>
12 12
13/* Libowfat */ 13#include "ot_batch.h"
14#include "iob.h"
15
16#include "ot_buf.h" 14#include "ot_buf.h"
17#include "trackerlogic.h" 15#include "trackerlogic.h"
18 16
@@ -27,8 +25,7 @@ typedef enum {
27 25
28struct http_data { 26struct http_data {
29 ot_buf request; 27 ot_buf request;
30 io_batch *batch; 28 ot_batch *batch;
31 size_t batches;
32 ot_ip6 ip; 29 ot_ip6 ip;
33 STRUCT_HTTP_FLAG flag; 30 STRUCT_HTTP_FLAG flag;
34}; 31};
@@ -37,7 +34,7 @@ ssize_t http_handle_request(const int64_t s, struct ot_workstruct *ws);
37ssize_t http_sendiovecdata(const int64_t s, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial); 34ssize_t http_sendiovecdata(const int64_t s, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial);
38ssize_t http_issue_error(const int64_t s, struct ot_workstruct *ws, int code); 35ssize_t http_issue_error(const int64_t s, struct ot_workstruct *ws, int code);
39 36
40extern char *g_stats_path; 37extern const char *g_stats_path;
41extern ssize_t g_stats_path_len; 38extern ssize_t g_stats_path_len;
42 39
43#endif 40#endif
diff --git a/ot_stats.c b/ot_stats.c
index f938e91..6589aef 100644
--- a/ot_stats.c
+++ b/ot_stats.c
@@ -53,7 +53,7 @@ static unsigned long long ot_full_scrape_count;
53static unsigned long long ot_full_scrape_request_count; 53static unsigned long long ot_full_scrape_request_count;
54static unsigned long long ot_full_scrape_size; 54static unsigned long long ot_full_scrape_size;
55static unsigned long long ot_failed_request_counts[CODE_HTTPERROR_COUNT]; 55static unsigned long long ot_failed_request_counts[CODE_HTTPERROR_COUNT];
56static char *ot_failed_request_names[] = { 56static const char *ot_failed_request_names[] = {
57 "302 Redirect", "400 Parse Error", "400 Invalid Parameter", "400 Invalid Parameter (compact=0)", "400 Not Modest", 57 "302 Redirect", "400 Parse Error", "400 Invalid Parameter", "400 Invalid Parameter (compact=0)", "400 Not Modest",
58 "402 Payment Required", "403 Access Denied", "404 Not found", "500 Internal Server Error"}; 58 "402 Payment Required", "403 Access Denied", "404 Not found", "500 Internal Server Error"};
59static unsigned long long ot_renewed[OT_PEER_TIMEOUT]; 59static unsigned long long ot_renewed[OT_PEER_TIMEOUT];
diff --git a/ot_timeout.h b/ot_timeout.h
new file mode 100644
index 0000000..1027c2e
--- /dev/null
+++ b/ot_timeout.h
@@ -0,0 +1,39 @@
1/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
2 It is considered beerware. Prost. Skol. Cheers or whatever.
3
4 $id$ */
5
6#ifndef OT_TIMEOUT_H__
7#define OT_TIMEOUT_H__
8
9#include <stdint.h>
10#include <string.h>
11
12/* Libowfat */
13#include "io.h"
14
15#include "trackerlogic.h"
16
17/* io_timeout wants an absolute TAI64 deadline and io_timeouted skips any
18 socket whose deadline is zero, so that is how a socket is told to wait
19 indefinitely.
20
21 The deadline is taken from the cached clock rather than from
22 gettimeofday, which is too expensive to call at this rate on FreeBSD. */
23
24static inline void timeout_never(int64_t sock) {
25 tai6464 t;
26
27 memset(&t, 0, sizeof(t));
28 io_timeout(sock, t);
29}
30
31static inline void timeout_in(int64_t sock, unsigned int seconds) {
32 tai6464 t;
33
34 memset(&t, 0, sizeof(t));
35 tai_unix(&t.sec, g_now_seconds + seconds);
36 io_timeout(sock, t);
37}
38
39#endif
diff --git a/proxy.c b/proxy.c
index a870c63..63b4c4f 100644
--- a/proxy.c
+++ b/proxy.c
@@ -16,9 +16,9 @@
16 16
17/* Libowfat */ 17/* Libowfat */
18#include "io.h" 18#include "io.h"
19#include "iob.h"
20 19
21/* Opentracker */ 20/* Opentracker */
21#include "ot_batch.h"
22#include "ot_ip.h" 22#include "ot_ip.h"
23#include "ot_mutex.h" 23#include "ot_mutex.h"
24#include "ot_random.h" 24#include "ot_random.h"
@@ -72,7 +72,7 @@ static void *livesync_worker(void *args);
72static void *streamsync_worker(void *args); 72static void *streamsync_worker(void *args);
73static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer); 73static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer);
74 74
75void exerr(char *message) { 75void exerr(const char *message) {
76 fprintf(stderr, "%s\n", message); 76 fprintf(stderr, "%s\n", message);
77 exit(111); 77 exit(111);
78} 78}
@@ -228,7 +228,7 @@ typedef struct {
228 size_t indata_length; /* Length of unprocessed data */ 228 size_t indata_length; /* Length of unprocessed data */
229 uint32_t tracker_id; /* How the other end greeted */ 229 uint32_t tracker_id; /* How the other end greeted */
230 int64_t fd; /* A file handle, if connected, <= 0 is disconnected (0 initially, -1 else) */ 230 int64_t fd; /* A file handle, if connected, <= 0 is disconnected (0 initially, -1 else) */
231 io_batch outdata; /* The iobatch containing our sync data */ 231 ot_batch outdata; /* The batch containing our sync data */
232 232
233 size_t packet_tcount; /* Number of unprocessed torrents in packet we currently receive */ 233 size_t packet_tcount; /* Number of unprocessed torrents in packet we currently receive */
234 uint8_t packet_tprefix; /* Prefix byte for all torrents in current packet */ 234 uint8_t packet_tprefix; /* Prefix byte for all torrents in current packet */
@@ -243,7 +243,7 @@ void reset_info_block(proxy_peer *peer) {
243 peer->tracker_id = 0; 243 peer->tracker_id = 0;
244 peer->fd = -1; 244 peer->fd = -1;
245 peer->packet_tcount = 0; 245 peer->packet_tcount = 0;
246 iob_reset(&peer->outdata); 246 batch_reset(&peer->outdata);
247 PROXYPEER_SETDISCONNECTED(peer->state); 247 PROXYPEER_SETDISCONNECTED(peer->state);
248} 248}
249 249
@@ -424,11 +424,11 @@ static void handle_write(int64_t peersocket) {
424 } 424 }
425 break; 425 break;
426 case FLAG_CONNECTED: 426 case FLAG_CONNECTED:
427 switch (iob_send(peersocket, &peer->outdata)) { 427 switch (batch_send(peersocket, &peer->outdata)) {
428 case 0: /* all data sent */ 428 case BATCH_DONE:
429 io_dontwantwrite(peersocket); 429 io_dontwantwrite(peersocket);
430 break; 430 break;
431 case -3: /* an error occured */ 431 case BATCH_FAILED:
432 io_close(peersocket); 432 io_close(peersocket);
433 reset_info_block(peer); 433 reset_info_block(peer);
434 break; 434 break;
@@ -728,7 +728,7 @@ static void *streamsync_worker(void *args) {
728 void *tmp = malloc(mem); 728 void *tmp = malloc(mem);
729 if (tmp) { 729 if (tmp) {
730 memcpy(tmp, ptr, mem); 730 memcpy(tmp, ptr, mem);
731 iob_addbuf_free(&g_connections[i].outdata, tmp, mem); 731 batch_add(&g_connections[i].outdata, tmp, mem);
732 io_wantwrite(g_connections[i].fd); 732 io_wantwrite(g_connections[i].fd);
733 } 733 }
734 } 734 }
diff --git a/scan_urlencoded_query.h b/scan_urlencoded_query.h
index d0929cf..f71752d 100644
--- a/scan_urlencoded_query.h
+++ b/scan_urlencoded_query.h
@@ -11,8 +11,8 @@
11#include <sys/types.h> 11#include <sys/types.h>
12 12
13typedef struct { 13typedef struct {
14 char *key; 14 const char *key;
15 int value; 15 int value;
16} ot_keywords; 16} ot_keywords;
17 17
18typedef enum { 18typedef enum {
diff --git a/tests/runner.c b/tests/runner.c
index db62246..148b928 100644
--- a/tests/runner.c
+++ b/tests/runner.c
@@ -23,6 +23,8 @@ static const struct module modules[] = {{"scan_urlencoded_query", test_scan_urle
23 {"ot_buf", test_buf}, 23 {"ot_buf", test_buf},
24 {"ot_ip", test_ip}, 24 {"ot_ip", test_ip},
25 {"ot_socket", test_socket}, 25 {"ot_socket", test_socket},
26 {"ot_mutex", test_mutex},
27 {"ot_batch", test_batch},
26 {"ot_accesslist", test_accesslist}, 28 {"ot_accesslist", test_accesslist},
27 {NULL, NULL}}; 29 {NULL, NULL}};
28 30
diff --git a/tests/stubs.c b/tests/stubs.c
index ce93071..09a7312 100644
--- a/tests/stubs.c
+++ b/tests/stubs.c
@@ -18,3 +18,12 @@ void free_peerlist(ot_peerlist *peer_list) { (void)peer_list; }
18 neither of which any test drives. */ 18 neither of which any test drives. */
19time_t g_now_seconds; 19time_t g_now_seconds;
20volatile int g_opentracker_running; 20volatile int g_opentracker_running;
21
22/* ot_mutex.c wakes the main loop through the self pipe when a result is
23 ready. The queue tests never get that far. */
24int g_self_pipe[2] = {-1, -1};
25long io_trywrite(long d, const char *buf, long len) {
26 (void)d;
27 (void)buf;
28 return len;
29}
diff --git a/tests/t_batch.c b/tests/t_batch.c
new file mode 100644
index 0000000..02743f1
--- /dev/null
+++ b/tests/t_batch.c
@@ -0,0 +1,348 @@
1/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
2 It is considered beerware. Prost. Skol. Cheers or whatever.
3
4 $id$ */
5
6/* System */
7#include <fcntl.h>
8#include <poll.h>
9#include <stdlib.h>
10#include <string.h>
11#include <sys/socket.h>
12#include <unistd.h>
13
14/* Opentracker */
15#include "ot_batch.h"
16#include "test.h"
17
18/* The batch writes to a real socket pair, because what is worth pinning down
19 is what it does when the kernel takes only part of what it was offered. */
20
21static char *dup_of(const char *text) {
22 size_t len = strlen(text);
23 char *copy = malloc(len);
24
25 memcpy(copy, text, len);
26 return copy;
27}
28
29static void drain(int s, char *into, size_t len) {
30 size_t got = 0;
31
32 while (got < len) {
33 ssize_t r = read(s, into + got, len - got);
34
35 if (r <= 0)
36 break;
37 got += (size_t)r;
38 }
39 into[got] = 0;
40}
41
42static void case_queue_accounting(void) {
43 ot_batch batch;
44
45 memset(&batch, 0, sizeof(batch));
46
47 test_case("a zeroed batch is empty and sends nothing");
48 CHECK_INT(batch_pending(&batch), 0);
49 CHECK_INT(batch_send(-1, &batch), BATCH_DONE);
50
51 test_case("anything queued makes it pending");
52 CHECK_INT(batch_add(&batch, dup_of("hello "), 6), 0);
53 CHECK_INT(batch_add_copy(&batch, "world", 5), 0);
54 CHECK_INT(batch_pending(&batch), 1);
55 CHECK_INT(batch.count, 2);
56
57 batch_reset(&batch);
58 test_case("and a reset empties it");
59 CHECK_INT(batch_pending(&batch), 0);
60 CHECK_PTR(batch.entries, NULL);
61}
62
63static void case_send_in_one_go(void) {
64 ot_batch batch;
65 int sv[2];
66 char got[32];
67
68 memset(&batch, 0, sizeof(batch));
69 CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
70
71 test_case("several buffers arrive as one stream, in order");
72 CHECK_INT(batch_add(&batch, dup_of("opentracker"), 11), 0);
73 CHECK_INT(batch_add_copy(&batch, " says ", 6), 0);
74 CHECK_INT(batch_add(&batch, dup_of("hello"), 5), 0);
75 CHECK_INT(batch_send(sv[0], &batch), 22);
76 CHECK_INT(batch_pending(&batch), 0);
77
78 drain(sv[1], got, 22);
79 CHECK_STR(got, "opentracker says hello");
80
81 test_case("and an empty batch reports it has nothing left");
82 CHECK_INT(batch_send(sv[0], &batch), BATCH_DONE);
83
84 batch_reset(&batch);
85 close(sv[0]);
86 close(sv[1]);
87}
88
89/* Send everything the batch holds, reading the other end as we go so the
90 socket never stays full, and return what arrived. */
91static size_t pump(int writer, int reader, ot_batch *batch, char *into, size_t room) {
92 size_t got = 0;
93
94 while (batch_pending(batch) || got < room) {
95 ssize_t r;
96
97 if (batch_pending(batch)) {
98 ssize_t res = batch_send(writer, batch);
99
100 if (res == BATCH_FAILED)
101 break;
102 if (res > 0)
103 continue;
104 }
105
106 r = read(reader, into + got, room - got);
107 if (r <= 0)
108 break;
109 got += (size_t)r;
110 }
111 return got;
112}
113
114static void case_partial_write_resumes(void) {
115 const size_t size = 200000;
116 ot_batch batch;
117 int sv[2], sndbuf = 4096;
118 char *sent, *got;
119 size_t i, arrived;
120
121 memset(&batch, 0, sizeof(batch));
122 CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
123 setsockopt(sv[0], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf));
124 fcntl(sv[0], F_SETFL, fcntl(sv[0], F_GETFL, 0) | O_NONBLOCK);
125
126 sent = malloc(size);
127 got = malloc(size);
128 for (i = 0; i < size; ++i)
129 sent[i] = (char)(i % 251);
130
131 test_case("a buffer the kernel takes in pieces arrives whole and in order");
132 CHECK_INT(batch_add(&batch, memcpy(malloc(size), sent, size), size), 0);
133 arrived = pump(sv[0], sv[1], &batch, got, size);
134 CHECK_INT(arrived, size);
135 CHECK_INT(batch_pending(&batch), 0);
136 CHECK_MEM(got, sent, size);
137
138 free(sent);
139 free(got);
140 batch_reset(&batch);
141 close(sv[0]);
142 close(sv[1]);
143}
144
145/* writev refuses more than IOV_MAX entries at once, so a long batch has to be
146 sent across several calls. */
147static void case_more_entries_than_one_writev(void) {
148 const size_t entries = 3000, each = 4;
149 ot_batch batch;
150 int sv[2], sndbuf = 4096;
151 char *sent, *got;
152 size_t i, arrived, refused = 0;
153
154 memset(&batch, 0, sizeof(batch));
155 CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
156 setsockopt(sv[0], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf));
157 fcntl(sv[0], F_SETFL, fcntl(sv[0], F_GETFL, 0) | O_NONBLOCK);
158
159 sent = malloc(entries * each);
160 got = malloc(entries * each);
161
162 test_case("a batch of more entries than writev accepts still arrives whole");
163 for (i = 0; i < entries; ++i) {
164 char piece[4];
165
166 piece[0] = (char)(i), piece[1] = (char)(i >> 8), piece[2] = 'x', piece[3] = 'y';
167 memcpy(sent + i * each, piece, each);
168 if (batch_add(&batch, memcpy(malloc(each), piece, each), each))
169 ++refused;
170 }
171 CHECK_INT(refused, 0);
172 CHECK_INT(batch.count, entries);
173
174 arrived = pump(sv[0], sv[1], &batch, got, entries * each);
175 CHECK_INT(arrived, entries * each);
176 CHECK_MEM(got, sent, entries * each);
177
178 free(sent);
179 free(got);
180 batch_reset(&batch);
181 close(sv[0]);
182 close(sv[1]);
183}
184
185static void case_reset_frees_the_unsent(void) {
186 ot_batch batch;
187 int sv[2];
188
189 memset(&batch, 0, sizeof(batch));
190 CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
191
192 test_case("a batch torn down mid flight keeps no bytes outstanding");
193 CHECK_INT(batch_add(&batch, dup_of("first"), 5), 0);
194 CHECK_INT(batch_add(&batch, dup_of("second"), 6), 0);
195 CHECK_INT(batch_send(sv[0], &batch), 11);
196 CHECK_INT(batch_add(&batch, dup_of("never sent"), 10), 0);
197 CHECK_INT(batch_pending(&batch), 1);
198
199 /* Whether the unsent buffer is freed is for the sanitizer to say */
200 batch_reset(&batch);
201 CHECK_INT(batch_pending(&batch), 0);
202 CHECK_INT(batch.count, 0);
203
204 close(sv[0]);
205 close(sv[1]);
206}
207
208static void case_closed_socket_is_fatal(void) {
209 ot_batch batch;
210 int sv[2];
211
212 memset(&batch, 0, sizeof(batch));
213 CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
214 close(sv[1]);
215
216 test_case("writing to a socket whose peer is gone reports a dead connection");
217 CHECK_INT(batch_add(&batch, dup_of("nobody is listening"), 19), 0);
218 CHECK_INT(batch_send(sv[0], &batch), BATCH_FAILED);
219
220 batch_reset(&batch);
221 close(sv[0]);
222}
223
224static void case_vector_is_taken_whole(void) {
225 ot_batch batch;
226 struct iovec *vector = malloc(3 * sizeof(struct iovec));
227 int sv[2];
228 char got[32];
229
230 memset(&batch, 0, sizeof(batch));
231 CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
232
233 test_case("a whole vector is queued in one go, in order");
234 vector[0].iov_base = dup_of("one "), vector[0].iov_len = 4;
235 vector[1].iov_base = dup_of("two "), vector[1].iov_len = 4;
236 vector[2].iov_base = dup_of("three"), vector[2].iov_len = 5;
237 CHECK_INT(batch_add_vector(&batch, vector, 3), 0);
238 free(vector);
239 CHECK_INT(batch.count, 3);
240 CHECK_INT(batch_send(sv[0], &batch), 13);
241
242 drain(sv[1], got, 13);
243 CHECK_STR(got, "one two three");
244
245 batch_reset(&batch);
246 close(sv[0]);
247 close(sv[1]);
248}
249
250/* The entry array starts small and doubles, so crossing that boundary one
251 buffer at a time is worth its own case. */
252static void case_growing_one_entry_at_a_time(void) {
253 const size_t entries = 100;
254 ot_batch batch;
255 int sv[2];
256 char sent[400], got[401];
257 size_t i, refused = 0;
258
259 memset(&batch, 0, sizeof(batch));
260 CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
261
262 test_case("a batch grown one buffer at a time keeps them all, in order");
263 for (i = 0; i < entries; ++i) {
264 char piece[4];
265
266 piece[0] = (char)('a' + i % 26), piece[1] = (char)('0' + i % 10);
267 piece[2] = (char)('A' + i % 26), piece[3] = '.';
268 memcpy(sent + i * 4, piece, 4);
269 if (batch_add(&batch, memcpy(malloc(4), piece, 4), 4))
270 ++refused;
271 }
272 CHECK_INT(refused, 0);
273 CHECK_INT(batch.count, entries);
274 CHECK_INT(batch_send(sv[0], &batch), (ssize_t)(entries * 4));
275
276 drain(sv[1], got, entries * 4);
277 CHECK_MEM(got, sent, entries * 4);
278
279 batch_reset(&batch);
280 close(sv[0]);
281 close(sv[1]);
282}
283
284/* Appending while earlier entries are still on their way is what a chunked
285 transfer does. Rather than depend on when a kernel decides to block, this
286 takes entries off the front the way a completed send would and checks that
287 the array reuses the space instead of growing. */
288static void case_append_while_sending(void) {
289 const size_t per_round = 12, each = 8;
290 ot_batch batch;
291 int sv[2];
292 char expected[16], got[17];
293 size_t i, round, refused = 0, space_after_first = 0;
294
295 memset(&batch, 0, sizeof(batch));
296 CHECK_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv), 0);
297
298 test_case("an array whose front has gone out is reused, not grown");
299 for (round = 0; round < 8; ++round) {
300 for (i = 0; i < per_round; ++i) {
301 /* Distinct per entry, so a move from the wrong index shows up */
302 char *piece = malloc(each);
303
304 memset(piece, (int)('a' + (round * per_round + i) % 26), each);
305 piece[0] = (char)('0' + i);
306 piece[1] = (char)('0' + round);
307 if (batch_add(&batch, piece, each))
308 ++refused;
309 }
310
311 /* All but the last entry have left and been freed */
312 while (batch.count - batch.first > 1)
313 free(batch.entries[batch.first++].iov_base);
314
315 if (!round)
316 space_after_first = batch.space;
317 }
318 CHECK_INT(refused, 0);
319 CHECK_INT(batch.space, space_after_first);
320
321 test_case("and what is still queued survives the move, in order");
322 CHECK_INT(batch.count - batch.first, 1);
323 CHECK_INT(batch_add(&batch, memset(malloc(each), 'Z', each), each), 0);
324 CHECK_INT(batch_send(sv[0], &batch), (ssize_t)(2 * each));
325
326 memset(expected, (int)('a' + (7 * per_round + per_round - 1) % 26), each);
327 expected[0] = (char)('0' + per_round - 1);
328 expected[1] = '7';
329 memset(expected + each, 'Z', each);
330 drain(sv[1], got, 2 * each);
331 CHECK_MEM(got, expected, 2 * each);
332
333 batch_reset(&batch);
334 close(sv[0]);
335 close(sv[1]);
336}
337
338void test_batch(void) {
339 case_queue_accounting();
340 case_send_in_one_go();
341 case_vector_is_taken_whole();
342 case_growing_one_entry_at_a_time();
343 case_append_while_sending();
344 case_partial_write_resumes();
345 case_more_entries_than_one_writev();
346 case_reset_frees_the_unsent();
347 case_closed_socket_is_fatal();
348}
diff --git a/tests/t_mutex.c b/tests/t_mutex.c
new file mode 100644
index 0000000..3307d70
--- /dev/null
+++ b/tests/t_mutex.c
@@ -0,0 +1,107 @@
1/* This software was written by Dirk Engling <erdgeist@erdgeist.org>
2 It is considered beerware. Prost. Skol. Cheers or whatever.
3
4 $id$ */
5
6/* System */
7#include <stdlib.h>
8#include <string.h>
9
10/* Opentracker */
11#include "ot_mutex.h"
12#include "test.h"
13
14/* A worker asks for a class of task and blocks until one turns up, so a task
15 whose class matches no worker is never claimed and whoever is waiting for
16 its answer waits forever. These cases pin the matching. */
17
18static void case_task_goes_to_its_own_class(void) {
19 ot_tasktype asked;
20 ot_taskid taskid;
21
22 test_case("a worker is handed the task of the class it asked for");
23 CHECK_INT(mutex_workqueue_pushtask(41, TASK_FULLSCRAPE_TPB_ASCII), 0);
24 asked = TASK_FULLSCRAPE;
25 taskid = mutex_workqueue_poptask(&asked);
26 CHECK(taskid != 0);
27
28 test_case("and the full task type, not just its class");
29 CHECK_INT(asked, TASK_FULLSCRAPE_TPB_ASCII);
30 mutex_workqueue_canceltask(41);
31}
32
33static void case_flags_survive_the_queue(void) {
34 ot_tasktype asked;
35
36 test_case("flags outside the class mask are carried through");
37 CHECK_INT(mutex_workqueue_pushtask(42, TASK_FULLSCRAPE_TPB_URLENCODED | TASK_FLAG_GZIP), 0);
38 asked = TASK_FULLSCRAPE;
39 CHECK(mutex_workqueue_poptask(&asked) != 0);
40 CHECK_INT(asked, TASK_FULLSCRAPE_TPB_URLENCODED | TASK_FLAG_GZIP);
41 mutex_workqueue_canceltask(42);
42}
43
44static void case_classes_do_not_cross(void) {
45 ot_tasktype asked;
46 ot_taskid taskid;
47
48 test_case("a stats worker passes over a fullscrape task");
49 CHECK_INT(mutex_workqueue_pushtask(43, TASK_FULLSCRAPE_TPB_BINARY), 0);
50 CHECK_INT(mutex_workqueue_pushtask(44, TASK_STATS_PEERS), 0);
51
52 asked = TASK_STATS;
53 taskid = mutex_workqueue_poptask(&asked);
54 CHECK(taskid != 0);
55 CHECK_INT(asked, TASK_STATS_PEERS);
56
57 test_case("and leaves it for the one that wants it");
58 asked = TASK_FULLSCRAPE;
59 taskid = mutex_workqueue_poptask(&asked);
60 CHECK(taskid != 0);
61 CHECK_INT(asked, TASK_FULLSCRAPE_TPB_BINARY);
62
63 mutex_workqueue_canceltask(43);
64 mutex_workqueue_canceltask(44);
65}
66
67static void case_every_task_gets_its_own_id(void) {
68 ot_tasktype asked;
69 ot_taskid first, second;
70
71 test_case("two queued tasks are handed out under different ids");
72 CHECK_INT(mutex_workqueue_pushtask(45, TASK_FULLSCRAPE), 0);
73 CHECK_INT(mutex_workqueue_pushtask(46, TASK_FULLSCRAPE), 0);
74
75 asked = TASK_FULLSCRAPE;
76 first = mutex_workqueue_poptask(&asked);
77 asked = TASK_FULLSCRAPE;
78 second = mutex_workqueue_poptask(&asked);
79 CHECK(first != 0);
80 CHECK(second != 0);
81 CHECK(first != second);
82
83 mutex_workqueue_canceltask(45);
84 mutex_workqueue_canceltask(46);
85}
86
87static void case_cancelled_task_is_not_handed_out(void) {
88 ot_tasktype asked;
89
90 test_case("a cancelled task is gone before any worker sees it");
91 CHECK_INT(mutex_workqueue_pushtask(47, TASK_FULLSCRAPE_TPB_ASCII_PLUS), 0);
92 CHECK_INT(mutex_workqueue_pushtask(48, TASK_FULLSCRAPE), 0);
93 mutex_workqueue_canceltask(47);
94
95 asked = TASK_FULLSCRAPE;
96 CHECK(mutex_workqueue_poptask(&asked) != 0);
97 CHECK_INT(asked, TASK_FULLSCRAPE);
98 mutex_workqueue_canceltask(48);
99}
100
101void test_mutex(void) {
102 case_task_goes_to_its_own_class();
103 case_flags_survive_the_queue();
104 case_classes_do_not_cross();
105 case_every_task_gets_its_own_id();
106 case_cancelled_task_is_not_handed_out();
107}
diff --git a/tests/test.h b/tests/test.h
index d79c4d2..fd574c2 100644
--- a/tests/test.h
+++ b/tests/test.h
@@ -86,5 +86,7 @@ void test_scan(void);
86void test_buf(void); 86void test_buf(void);
87void test_ip(void); 87void test_ip(void);
88void test_socket(void); 88void test_socket(void);
89void test_mutex(void);
90void test_batch(void);
89 91
90#endif 92#endif
diff --git a/trackerlogic.c b/trackerlogic.c
index 52c8c18..74fea6d 100644
--- a/trackerlogic.c
+++ b/trackerlogic.c
@@ -10,9 +10,6 @@
10#include <stdlib.h> 10#include <stdlib.h>
11#include <string.h> 11#include <string.h>
12 12
13/* Libowfat */
14#include "io.h"
15#include "iob.h"
16 13
17/* Opentracker */ 14/* Opentracker */
18#include "ot_accesslist.h" 15#include "ot_accesslist.h"
@@ -550,7 +547,7 @@ void trackerlogic_add_random_torrents(size_t amount) {
550} 547}
551#endif 548#endif
552 549
553void exerr(char *message) { 550void exerr(const char *message) {
554 fprintf(stderr, "%s\n", message); 551 fprintf(stderr, "%s\n", message);
555 exit(111); 552 exit(111);
556} 553}
diff --git a/trackerlogic.h b/trackerlogic.h
index aa10a54..42018c3 100644
--- a/trackerlogic.h
+++ b/trackerlogic.h
@@ -181,7 +181,7 @@ struct ot_workstruct {
181 181
182void trackerlogic_init(void); 182void trackerlogic_init(void);
183void trackerlogic_deinit(void); 183void trackerlogic_deinit(void);
184void exerr(char *message); 184void exerr(const char *message);
185 185
186/* add_peer_to_torrent does only release the torrent bucket if from_sync is set, 186/* add_peer_to_torrent does only release the torrent bucket if from_sync is set,
187 otherwise it is released in return_peers_for_torrent */ 187 otherwise it is released in return_peers_for_torrent */