From ad541c67761b8aa6da1566802c347cac4018d0c2 Mon Sep 17 00:00:00 2001 From: Dirk Engling Date: Mon, 2 Nov 2026 20:50:24 +0100 Subject: Move the build sources into src/ --- src/opentracker.c | 443 ++++++++++++++++++++++++++ src/ot_accesslist.c | 601 ++++++++++++++++++++++++++++++++++++ src/ot_accesslist.h | 77 +++++ src/ot_batch.c | 156 ++++++++++ src/ot_batch.h | 46 +++ src/ot_buf.c | 39 +++ src/ot_buf.h | 25 ++ src/ot_clean.c | 176 +++++++++++ src/ot_clean.h | 28 ++ src/ot_config.c | 586 +++++++++++++++++++++++++++++++++++ src/ot_config.h | 33 ++ src/ot_event.c | 357 +++++++++++++++++++++ src/ot_event.h | 47 +++ src/ot_fdstate.c | 202 ++++++++++++ src/ot_fdstate.h | 62 ++++ src/ot_fmt.c | 40 +++ src/ot_fmt.h | 19 ++ src/ot_fullscrape.c | 455 +++++++++++++++++++++++++++ src/ot_fullscrape.h | 23 ++ src/ot_hex.c | 56 ++++ src/ot_hex.h | 27 ++ src/ot_http.c | 810 ++++++++++++++++++++++++++++++++++++++++++++++++ src/ot_http.h | 53 ++++ src/ot_iovec.c | 87 ++++++ src/ot_iovec.h | 18 ++ src/ot_ip.c | 199 ++++++++++++ src/ot_ip.h | 31 ++ src/ot_livesync.c | 241 +++++++++++++++ src/ot_livesync.h | 75 +++++ src/ot_mutex.c | 70 +++++ src/ot_mutex.h | 21 ++ src/ot_random.c | 51 +++ src/ot_random.h | 17 + src/ot_rijndael.c | 488 +++++++++++++++++++++++++++++ src/ot_rijndael.h | 20 ++ src/ot_scan.c | 65 ++++ src/ot_scan.h | 25 ++ src/ot_socket.c | 310 +++++++++++++++++++ src/ot_socket.h | 56 ++++ src/ot_stats.c | 866 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/ot_stats.h | 40 +++ src/ot_tcp.c | 180 +++++++++++ src/ot_tcp.h | 24 ++ src/ot_udp.c | 226 ++++++++++++++ src/ot_udp.h | 17 + src/ot_urlencoded.c | 168 ++++++++++ src/ot_urlencoded.h | 53 ++++ src/ot_vector.c | 287 +++++++++++++++++ src/ot_vector.h | 39 +++ src/ot_worker.c | 73 +++++ src/ot_worker.h | 37 +++ src/ot_workqueue.c | 241 +++++++++++++++ src/ot_workqueue.h | 74 +++++ src/proxy.c | 871 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/trackerlogic.c | 628 +++++++++++++++++++++++++++++++++++++ src/trackerlogic.h | 283 +++++++++++++++++ 56 files changed, 10242 insertions(+) create mode 100644 src/opentracker.c create mode 100644 src/ot_accesslist.c create mode 100644 src/ot_accesslist.h create mode 100644 src/ot_batch.c create mode 100644 src/ot_batch.h create mode 100644 src/ot_buf.c create mode 100644 src/ot_buf.h create mode 100644 src/ot_clean.c create mode 100644 src/ot_clean.h create mode 100644 src/ot_config.c create mode 100644 src/ot_config.h create mode 100644 src/ot_event.c create mode 100644 src/ot_event.h create mode 100644 src/ot_fdstate.c create mode 100644 src/ot_fdstate.h create mode 100644 src/ot_fmt.c create mode 100644 src/ot_fmt.h create mode 100644 src/ot_fullscrape.c create mode 100644 src/ot_fullscrape.h create mode 100644 src/ot_hex.c create mode 100644 src/ot_hex.h create mode 100644 src/ot_http.c create mode 100644 src/ot_http.h create mode 100644 src/ot_iovec.c create mode 100644 src/ot_iovec.h create mode 100644 src/ot_ip.c create mode 100644 src/ot_ip.h create mode 100644 src/ot_livesync.c create mode 100644 src/ot_livesync.h create mode 100644 src/ot_mutex.c create mode 100644 src/ot_mutex.h create mode 100644 src/ot_random.c create mode 100644 src/ot_random.h create mode 100644 src/ot_rijndael.c create mode 100644 src/ot_rijndael.h create mode 100644 src/ot_scan.c create mode 100644 src/ot_scan.h create mode 100644 src/ot_socket.c create mode 100644 src/ot_socket.h create mode 100644 src/ot_stats.c create mode 100644 src/ot_stats.h create mode 100644 src/ot_tcp.c create mode 100644 src/ot_tcp.h create mode 100644 src/ot_udp.c create mode 100644 src/ot_udp.h create mode 100644 src/ot_urlencoded.c create mode 100644 src/ot_urlencoded.h create mode 100644 src/ot_vector.c create mode 100644 src/ot_vector.h create mode 100644 src/ot_worker.c create mode 100644 src/ot_worker.h create mode 100644 src/ot_workqueue.c create mode 100644 src/ot_workqueue.h create mode 100644 src/proxy.c create mode 100644 src/trackerlogic.c create mode 100644 src/trackerlogic.h (limited to 'src') diff --git a/src/opentracker.c b/src/opentracker.c new file mode 100644 index 0000000..14b817e --- /dev/null +++ b/src/opentracker.c @@ -0,0 +1,443 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. + Some of the stuff below is stolen from Fefes example libowfat httpd. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef WANT_SYSLOGS +#include +#endif + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_config.h" +#include "ot_hex.h" +#include "ot_http.h" +#include "ot_ip.h" +#include "ot_livesync.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_random.h" +#include "ot_scan.h" +#include "ot_event.h" +#include "ot_fdstate.h" +#include "ot_socket.h" +#include "ot_stats.h" +#include "ot_tcp.h" +#include "ot_udp.h" +#include "ot_worker.h" +#include "trackerlogic.h" + +/* Globals */ +uint32_t g_tracker_id; +_Atomic int g_opentracker_running = 1; +int g_self_pipe[2]; + +static void panic(const char *routine) __attribute__((noreturn)); +static void panic(const char *routine) { + fprintf(stderr, "%s: %s\n", routine, strerror(errno)); + exit(111); +} + +/* Say we are stopping, wake the loop, and let main do the work on its own + stack. Everything other than SIGINT falls through and does nothing, which + is what OT_SIGNAL_WAKE wants: returning from the read is all it is for. */ +static void signal_handler(int s) { + if (s == SIGINT) { + const char byte = 0; + int saved = errno; + + /* Any new interrupt signal quits the application */ + signal(SIGINT, SIG_DFL); + + /* Tell all other threads to not acquire any new lock on a bucket + but cancel their operations and return */ + g_opentracker_running = 0; + + (void)!write(g_self_pipe[1], &byte, 1); + errno = saved; + } +} + +static void defaul_signal_handlers(void) { + sigset_t signal_mask; + sigemptyset(&signal_mask); + sigaddset(&signal_mask, SIGPIPE); + sigaddset(&signal_mask, SIGHUP); + sigaddset(&signal_mask, SIGINT); + sigaddset(&signal_mask, SIGALRM); + pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); +} + +static void install_signal_handlers(void) { + struct sigaction sa; + sigset_t signal_mask; + sigemptyset(&signal_mask); + + sa.sa_handler = signal_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + if ((sigaction(SIGINT, &sa, NULL) == -1) || (sigaction(SIGALRM, &sa, NULL) == -1) || + (sigaction(OT_SIGNAL_WAKE, &sa, NULL) == -1)) + panic("install_signal_handlers"); + + sigaddset(&signal_mask, SIGINT); + pthread_sigmask(SIG_UNBLOCK, &signal_mask, NULL); +} + + +/* How many ready descriptors one wait hands back at a time */ +#define OT_EVENT_BATCH 512 + +static void *server_mainloop(void *args) { + struct ot_workstruct ws; + time_t next_timeout_check = time_future_seconds(OT_CLIENT_TIMEOUT_CHECKINTERVAL); + struct iovec *iovector; + size_t iovec_entries; + int is_partial; + + (void)args; + + if (workstruct_init(&ws) == -1) + panic("Initializing worker failed"); + + while (g_opentracker_running) { + ot_event ready[OT_EVENT_BATCH]; + int64_t sock; + int count, i; + + count = event_wait(ready, OT_EVENT_BATCH, OT_CLIENT_TIMEOUT_CHECKINTERVAL * 1000); + + for (i = 0; i < count; ++i) { + const void *cookie; + + sock = ready[i].fd; + /* A descriptor closed earlier in this same batch may already have been + handed out again by accept, so an event is only ours if the + generation it was registered under is still the one in place */ + if (fdstate_generation((int)sock) != ready[i].generation) + continue; + cookie = fdstate_cookie((int)sock); + + if ((intptr_t)cookie == FLAG_TCP) + tcp_accept(sock, &ws); + else if ((intptr_t)cookie == FLAG_UDP) + handle_udp6(sock, &ws); + else if ((intptr_t)cookie == FLAG_SELFPIPE) + while (read((int)sock, ws.inbuf, G_INBUF_SIZE) > 0) + ; + else if (fdstate_claim((int)sock, ready[i].generation)) { + if (ready[i].what & EVENT_IN) + tcp_read(sock, &ws); + /* The read may have closed it, so look again before writing */ + if ((ready[i].what & EVENT_OUT) && fdstate_generation((int)sock) == ready[i].generation) + tcp_write(sock); + if (fdstate_generation((int)sock) == ready[i].generation) + fdstate_release((int)sock); + } + } + + while ((sock = workqueue_popresult(&iovec_entries, &iovector, &is_partial)) != -1) + http_sendiovecdata(sock, &ws, iovec_entries, iovector, is_partial); + + if (time_now_seconds() > next_timeout_check) { + int cursor = 0; + + while ((sock = fdstate_next_expired(time_now_seconds(), &cursor)) != -1) + http_close(sock); + next_timeout_check = time_future_seconds(OT_CLIENT_TIMEOUT_CHECKINTERVAL); + } + + livesync_ticker(); + } + + /* This work struct is about to go out of scope, so hand its tally over + before anything can read it there */ + workstruct_deinit(&ws); + return 0; +} + + +/* A group of sockets sharing one address only spreads datagrams if every + member asked for it before binding, the first one included, so whether + this entry will have workers has to be known here. */ +static socket_sharing listen_sharing_for(const ot_listen *entry) { + if (entry->proto != FLAG_MCA && entry->workers > 1 && socket_can_balance()) + return SOCKET_BALANCED; + return entry->shared ? SOCKET_SHARED : SOCKET_EXCLUSIVE; +} + +static void listen_bind_one(ot_listen *entry) { + socket_sharing sharing = listen_sharing_for(entry); + +#ifdef _DEBUG + { + const char *protos[] = {"TCP", "UDP", "UDP mcast"}; + char _debug[512]; + int off = snprintf(_debug, sizeof(_debug), "Binding socket type %s to address [", protos[entry->proto]); + off += ip_format(_debug + off, entry->ip); + snprintf(_debug + off, sizeof(_debug) - off, "]:%d...", entry->port); + fputs(_debug, stderr); + } +#endif + +#ifdef WANT_SYNC_LIVE + if (entry->proto == FLAG_MCA) { + livesync_bind_mcast(entry->ip, entry->port); +#ifdef _DEBUG + fputs(" success.\n", stderr); +#endif + return; + } +#endif + + if (entry->proto == FLAG_TCP) { + if ((entry->sock = socket_tcp_listen(entry->ip, entry->port, sharing)) == -1) + panic("socket_tcp_listen"); + socket_defer_accept((int)entry->sock); + } else { + if ((entry->sock = socket_udp_bind(entry->ip, entry->port, sharing)) == -1) + panic("socket_udp_bind"); + } + + if (event_add((int)entry->sock, EVENT_IN, fdstate_open(entry->sock, (void *)entry->proto, 0)) == -1) + panic("event_add"); + +#ifdef _DEBUG + fputs(" success.\n", stderr); +#endif +} + +/* Bound in the order the entries were given. + Note: On macOS SO_REUSEPORT delivers everything to the socket that bound last. */ +static void listen_bind_all(void) { + size_t count, i; + ot_listen *entries = config_listen_entries(&count); + + for (i = 0; i < count; ++i) + listen_bind_one(entries + i); +} + +/* The socket the event loop was using is passed to the first worker. + If start up tests show that the kernel can spread connections over + a group bound to one address, every worker binds one of its own. + If not, all workers call accept on the first. */ +static void listen_start_tcp_workers(void) { + size_t count, i; + ot_listen *entries = config_listen_entries(&count); + + for (i = 0; i < count; ++i) { + ot_listen *entry = entries + i; + int balance = listen_sharing_for(entry) == SOCKET_BALANCED; + int worker; + + if (entry->proto != FLAG_TCP || entry->workers <= 0) + continue; + + event_del((int)entry->sock); + tcp_worker_install(entry->sock); + + for (worker = 1; worker < entry->workers; ++worker) { + int64_t sock = entry->sock; + + if (balance) { + sock = socket_tcp_listen(entry->ip, entry->port, SOCKET_BALANCED); + if (sock == -1) { + fprintf(stderr, "Warning: could not bind another socket for tcp worker %d: %s. Sharing the first instead.\n", worker, + strerror(errno)); + sock = entry->sock; + } + } + tcp_worker_install(sock); + } + } +} + +static void listen_start_udp_workers(void) { + size_t count, i; + ot_listen *entries = config_listen_entries(&count); + + udp_init(); + + for (i = 0; i < count; ++i) { + ot_listen *entry = entries + i; + int balance = listen_sharing_for(entry) == SOCKET_BALANCED; + int worker; + + if (entry->proto != FLAG_UDP || entry->workers <= 0) + continue; + + event_del((int)entry->sock); + udp_worker_install(entry->sock); + + for (worker = 1; worker < entry->workers; ++worker) { + int64_t sock = entry->sock; + + if (balance) { + sock = socket_udp_bind(entry->ip, entry->port, SOCKET_BALANCED); + if (sock == -1) { + fprintf(stderr, "Warning: could not bind another socket for udp worker %d: %s. Sharing the first instead.\n", worker, strerror(errno)); + sock = entry->sock; + } + } + udp_worker_install(sock); + } + } +} + +void load_state(const char *const state_filename) { + FILE *state_filehandle; + char inbuf[512]; + ot_hash infohash; + uint64_t base, downcount; + int consumed; + + state_filehandle = fopen(state_filename, "r"); + + if (state_filehandle == NULL) { + fprintf(stderr, "Warning: Can't open config file: %s.", state_filename); + return; + } + + /* We do ignore anything that is not of the form "^[:xdigit:]:\d+:\d+" */ + while (fgets(inbuf, sizeof(inbuf), state_filehandle)) { + size_t i; + + if (!hex_to_hash(infohash, inbuf)) + continue; + i = 2 * sizeof(ot_hash); + + if (inbuf[i++] != ':' || !(consumed = scan_uint(inbuf + i, &base, UINT64_MAX))) + continue; + i += consumed; + if (inbuf[i++] != ':' || !(consumed = scan_uint(inbuf + i, &downcount, UINT64_MAX))) + continue; + add_torrent_from_saved_state(infohash, base, downcount); + } + + fclose(state_filehandle); +} + +int drop_privileges(const char *const serveruser, const char *const serverdir) { + struct passwd *pws = NULL; + +#ifdef _DEBUG + if (!geteuid()) + fprintf(stderr, "Dropping to user %s.\n", serveruser); + if (serverdir) + fprintf(stderr, "ch%s'ing to directory %s.\n", geteuid() ? "dir" : "root", serverdir); +#endif + + /* Grab pws entry before chrooting */ + pws = getpwnam(serveruser); + endpwent(); + + if (geteuid() == 0) { + /* Running as root: chroot and drop privileges */ + if (serverdir && chroot(serverdir)) { + fprintf(stderr, "Could not chroot to %s, because: %s\n", serverdir, strerror(errno)); + return -1; + } + + if (chdir("/")) + panic("chdir() failed after chrooting: "); + + /* If we can't find server user, revert to nobody's default uid */ + if (!pws) { + fprintf(stderr, "Warning: Could not get password entry for %s. Reverting to uid -2.\n", serveruser); + if (setegid((gid_t)-2) || setgid((gid_t)-2) || setuid((uid_t)-2) || seteuid((uid_t)-2)) + panic("Could not set uid to value -2"); + } else { + if (setegid(pws->pw_gid) || setgid(pws->pw_gid) || setuid(pws->pw_uid) || seteuid(pws->pw_uid)) + panic("Could not set uid to specified value"); + } + + if (geteuid() == 0 || getegid() == 0) + panic("Still running with root privileges?!"); + } else { + /* Normal user, just chdir() */ + if (serverdir && chdir(serverdir)) { + fprintf(stderr, "Could not chroot to %s, because: %s\n", serverdir, strerror(errno)); + return -1; + } + } + + return 0; +} + +/* Maintain our copy of the clock. time() on BSDs is very expensive. */ +int main(int argc, char **argv) { + + random_init(); + if (fdstate_init() == -1) + panic("fdstate_init"); + if (event_init() == -1) + panic("event_init"); + config_parse(argc, argv); + + listen_bind_all(); + + defaul_signal_handlers(); + +#ifdef WANT_SYSLOGS + openlog("opentracker", 0, LOG_USER); + setlogmask(LOG_UPTO(LOG_INFO)); +#endif + + if (drop_privileges(g_serveruser ? g_serveruser : "nobody", g_serverdir) == -1) + panic("drop_privileges failed, exiting. Last error"); + + + /* Create our self pipe which allows us to interrupt mainloops + wait in case some data is available to send out */ + if (pipe(g_self_pipe) == -1) + panic("selfpipe failed: "); + /* Both ends have to be non blocking: the loop drains the read end until it + is empty, and a worker nudging a full pipe has already made its point */ + socket_set_blocking(g_self_pipe[0], 0); + socket_set_blocking(g_self_pipe[1], 0); + if (event_add(g_self_pipe[0], EVENT_IN, fdstate_open(g_self_pipe[0], (void *)FLAG_SELFPIPE, 0)) == -1) + panic("selfpipe event_add"); + + /* Init all sub systems. This call may fail with an exit() */ + trackerlogic_init(); + +#ifdef _DEBUG_RANDOMTORRENTS + fprintf(stderr, "DEBUG: Generating %d random peers on random torrents. This may take a while. (Setting RANDOMTORRENTS in trackerlogic.h)\n", RANDOMTORRENTS); + trackerlogic_add_random_torrents(RANDOMTORRENTS); + fprintf(stderr, "... done.\n"); +#endif + + if (g_statefile) + load_state(g_statefile); + + install_signal_handlers(); + + listen_start_tcp_workers(); + listen_start_udp_workers(); + + server_mainloop(0); + worker_stop_all(); + + /* No thread is serving any more, so the teardown can run where a debugger + and the sanitizers can follow it */ + trackerlogic_deinit(); +#ifdef WANT_SYSLOGS + closelog(); +#endif + + event_deinit(); + fdstate_deinit(); + return 0; +} diff --git a/src/ot_accesslist.c b/src/ot_accesslist.c new file mode 100644 index 0000000..86f9a7d --- /dev/null +++ b/src/ot_accesslist.c @@ -0,0 +1,601 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#if defined(WANT_ACCESSLIST_BLACK) || defined(WANT_ACCESSLIST_WHITE) +#include +#include +#include +#include +#include +#endif +#ifdef WANT_DYNAMIC_ACCESSLIST +#include +#endif + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_hex.h" +#include "ot_ip.h" +#include "ot_vector.h" +#include "trackerlogic.h" + +/* GLOBAL VARIABLES */ +#ifdef WANT_ACCESSLIST +const char *g_accesslist_filename = NULL; +#ifdef WANT_DYNAMIC_ACCESSLIST +const char *g_accesslist_pipe_add = NULL; +const char *g_accesslist_pipe_delete = NULL; +#endif +static pthread_mutex_t g_accesslist_mutex; + +/* Accesslists are lock free linked lists. We can not make them locking, because every announce + would try to acquire the mutex, making it the most contested mutex in the whole of opentracker, + basically creating a central performance choke point. + + The idea is that updating the list heads happens under the g_accesslist_mutex guard and is + done atomically, while consumers might potentially still hold pointers deeper inside the list. + + Consumers (for now only via accesslist_hashisvalid) will always fetch the list head pointer + that is guaranteed to live for at least five minutes. This should be many orders of magnitudes + more than how long it will be needed by the bsearch done on the list. */ +struct ot_accesslist; +typedef struct ot_accesslist ot_accesslist; +struct ot_accesslist { + ot_hash *list; + size_t size; + ot_time base; + ot_accesslist *next; +}; +static ot_accesslist *_Atomic g_accesslist = NULL; +#ifdef WANT_DYNAMIC_ACCESSLIST +static ot_accesslist *_Atomic g_accesslist_add = NULL; +static ot_accesslist *_Atomic g_accesslist_delete = NULL; +#endif + +/* Helpers to work on access lists */ +static int vector_compare_hash(const void *hash1, const void *hash2) { return memcmp(hash1, hash2, OT_HASH_COMPARE_SIZE); } + +static ot_accesslist *accesslist_free(ot_accesslist *accesslist) { + while (accesslist) { + ot_accesslist *this_accesslist = accesslist; + accesslist = this_accesslist->next; + free(this_accesslist->list); + free(this_accesslist); + } + return NULL; +} + +static ot_accesslist *accesslist_make(ot_accesslist *next, size_t size) { + ot_accesslist *accesslist_new = malloc(sizeof(ot_accesslist)); + if (accesslist_new) { + accesslist_new->list = size ? malloc(sizeof(ot_hash) * size) : NULL; + accesslist_new->size = size; + accesslist_new->base = 0; + accesslist_new->next = next; + if (size && !accesslist_new->list) { + free(accesslist_new); + accesslist_new = NULL; + } + } + return accesslist_new; +} + +/* This must be called with g_accesslist_mutex held. + This will never delete head, because that might still be in use. */ +static void accesslist_clean(ot_accesslist *accesslist) { + while (accesslist && accesslist->next) { + if (accesslist->next->base + 5 < time_now_minutes()) + accesslist->next = accesslist_free(accesslist->next); + accesslist = accesslist->next; + } +} + +/* A hash needs forty bytes, and all but a final one are followed by at least + one separator, so n hashes need at least 41n - 1 bytes. (len + 1) / 41 is + therefore an exact upper bound on what a buffer of len bytes can hold. */ +size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len) { + const char *end = buf + len; + const char *offs = buf; + size_t count = 0; + + while (count < capacity && end - offs >= 40) { + if (hex_to_hash(dest[count], offs)) { + offs += 40; + + if (offs == end || !hex_isdigit((unsigned char)*offs)) + ++count; + } + + /* Find start of next line */ + while (offs < end && *(offs++) != '\n') {} + } + + return count; +} + +/* Map a file read only. Returns 0 and sets map and maplen on success, -1 if + the file cannot be opened or mapped. */ +static int accesslist_map(const char *filename, const char **map, size_t *maplen) { + struct stat st; + void *mapped; + int fd = open(filename, O_RDONLY); + + if (fd < 0) + return -1; + + if (fstat(fd, &st) < 0 || st.st_size < 0 || (uintmax_t)st.st_size > (uintmax_t)SIZE_MAX) { + close(fd); + return -1; + } + + if (st.st_size == 0) { + close(fd); + *map = NULL; + *maplen = 0; + return 0; + } + + mapped = mmap(NULL, (size_t)st.st_size, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + + if (mapped == MAP_FAILED) + return -1; + + *map = mapped; + *maplen = (size_t)st.st_size; + return 0; +} + +static void accesslist_readfile(void) { + ot_accesslist *accesslist_new; + const char *map; + size_t maplen, capacity, found; + + if (!g_accesslist_filename) + return; + + if (accesslist_map(g_accesslist_filename, &map, &maplen) < 0) { + char *wd = getcwd(NULL, 0); + fprintf(stderr, "Warning: Can't open accesslist file: %s (but will try to create it later, if necessary and possible).\nPWD: %s\n", g_accesslist_filename, wd); + free(wd); + return; + } + + /* Make room for the most info_hashes this file could possibly hold */ + capacity = (maplen + 1) / 41; + accesslist_new = accesslist_make(g_accesslist, capacity); + if (!accesslist_new) { + fprintf(stderr, "Warning: Not enough memory to allocate %zd bytes for accesslist buffer. May succeed later.\n", capacity * sizeof(ot_hash)); + if (map) + munmap((void *)map, maplen); + return; + } + + found = maplen ? accesslist_parse_hashes(accesslist_new->list, capacity, map, maplen) : 0; +#ifdef _DEBUG + fprintf(stderr, "Added %zd info_hashes to accesslist\n", found); +#endif + + if (map) + munmap((void *)map, maplen); + + if (found) + qsort(accesslist_new->list, found, sizeof(ot_hash), vector_compare_hash); + accesslist_new->size = found; + + /* Now exchange the accesslist vector in the least race condition prone way */ + pthread_mutex_lock(&g_accesslist_mutex); + + /* Mark old accesslist as "needs to live another five minutes" */ + if (g_accesslist) + g_accesslist->base = time_now_minutes(); + accesslist_new->next = g_accesslist; + g_accesslist = accesslist_new; /* Only now set a new list */ + +#ifdef WANT_DYNAMIC_ACCESSLIST + /* If we have dynamic accesslists, reloading a new one will always void the add/delete lists. + Insert empty ones at the list head */ + if (g_accesslist_add && (accesslist_new = accesslist_make(g_accesslist_add, 0)) != NULL) { + g_accesslist_add->base = time_now_minutes(); + g_accesslist_add = accesslist_new; + } + if (g_accesslist_delete && (accesslist_new = accesslist_make(g_accesslist_delete, 0)) != NULL) { + g_accesslist_delete->base = time_now_minutes(); + g_accesslist_delete = accesslist_new; + } +#endif + + accesslist_clean(g_accesslist); + + pthread_mutex_unlock(&g_accesslist_mutex); +} + +int accesslist_hashisvalid(const ot_hash hash) { + /* Get working copy of current access list */ + ot_accesslist *accesslist = g_accesslist; +#ifdef WANT_DYNAMIC_ACCESSLIST + ot_accesslist *accesslist_add, *accesslist_delete; +#endif + void *exactmatch = NULL; + + if (accesslist && accesslist->size) + exactmatch = bsearch(hash, accesslist->list, accesslist->size, OT_HASH_COMPARE_SIZE, vector_compare_hash); + +#ifdef WANT_DYNAMIC_ACCESSLIST + /* If we had no match on the main list, scan the list of dynamically added hashes */ + accesslist_add = g_accesslist_add; + if ((exactmatch == NULL) && accesslist_add && accesslist_add->size) + exactmatch = bsearch(hash, accesslist_add->list, accesslist_add->size, OT_HASH_COMPARE_SIZE, vector_compare_hash); + + /* If we found a matching hash on the main list, scan the list of dynamically deleted hashes */ + accesslist_delete = g_accesslist_delete; + if ((exactmatch != NULL) && accesslist_delete && accesslist_delete->size && bsearch(hash, accesslist_delete->list, accesslist_delete->size, OT_HASH_COMPARE_SIZE, vector_compare_hash)) + exactmatch = NULL; +#endif + +#ifdef WANT_ACCESSLIST_BLACK + return exactmatch == NULL; +#else + return exactmatch != NULL; +#endif +} + +static void *accesslist_worker(void *args) { + int sig; + sigset_t signal_mask; + + sigemptyset(&signal_mask); + sigaddset(&signal_mask, SIGHUP); + + (void)args; + + while (1) { + if (!g_opentracker_running) + return NULL; + + /* Initial attempt to read accesslist */ + accesslist_readfile(); + + /* Wait for signals */ + while (sigwait(&signal_mask, &sig) != 0 && sig != SIGHUP) + ; + } + return NULL; +} + +#ifdef WANT_DYNAMIC_ACCESSLIST +static pthread_t thread_adder_id, thread_deleter_id; +static _Atomic int adder_left, deleter_left; +static void *accesslist_adddel_worker(const char *fifoname, ot_accesslist *_Atomic *adding_to, ot_accesslist *_Atomic *removing_from) { + struct stat st; + + if (!stat(fifoname, &st)) { + if (!S_ISFIFO(st.st_mode)) { + fprintf(stderr, "Error when starting dynamic accesslists: Found Non-FIFO file at %s.\nPlease remove it and restart opentracker.\n", fifoname); + return NULL; + } + } else { + int error = mkfifo(fifoname, 0755); + if (error && error != EEXIST) { + fprintf(stderr, "Error when starting dynamic accesslists: Couldn't create FIFO at %s, error: %s\n", fifoname, strerror(errno)); + return NULL; + } + } + + while (g_opentracker_running) { + FILE *fifo = fopen(fifoname, "r"); + char *line = NULL; + size_t linecap = 0; + ssize_t linelen; + + if (!fifo) { + /* Woken on purpose: the loop condition above decides what happens next */ + if (errno == EINTR) + continue; + fprintf(stderr, "Error when reading dynamic accesslists: Couldn't open FIFO at %s, error: %s\n", fifoname, strerror(errno)); + return NULL; + } + + while ((linelen = getline(&line, &linecap, fifo)) > 0) { + ot_hash info_hash; + + printf("Got line %*s", (int)linelen, line); + /* We do ignore anything that is not of the form "^[:xdigit:]{40}[^:xdigit:].*" + If there's not enough characters for an info_hash in the line, skip it. */ + if (linelen < 41) + continue; + + if (!hex_to_hash(info_hash, line)) + continue; + + /* From now on we modify g_accesslist_add and g_accesslist_delete, so prevent the + other worker threads from doing the same */ + pthread_mutex_lock(&g_accesslist_mutex); + + /* If the info hash is in the removing_from list, create a new head without that entry */ + if (*removing_from && (*removing_from)->list) { + ot_hash *exactmatch = bsearch(info_hash, (*removing_from)->list, (*removing_from)->size, OT_HASH_COMPARE_SIZE, vector_compare_hash); + if (exactmatch) { + ptrdiff_t off = exactmatch - (*removing_from)->list; + ot_accesslist *accesslist_new = accesslist_make(*removing_from, (*removing_from)->size - 1); + if (accesslist_new) { + size_t after = (*removing_from)->size - off - 1; + + /* Removing the only entry leaves a list with no room at all, and + memcpy may not be handed that even for nothing */ + if (off) + memcpy(accesslist_new->list, (*removing_from)->list, sizeof(ot_hash) * off); + if (after) + memcpy(accesslist_new->list + off, (*removing_from)->list + off + 1, sizeof(ot_hash) * after); + (*removing_from)->base = time_now_minutes(); + *removing_from = accesslist_new; + } + } + } + + /* Simple case: there's no adding_to list yet, create one with one member */ + if (!*adding_to) { + ot_accesslist *accesslist_new = accesslist_make(NULL, 1); + if (accesslist_new) { + memcpy(accesslist_new->list, info_hash, sizeof(ot_hash)); + *adding_to = accesslist_new; + } + } else { + int exactmatch = 0; + ot_hash *insert_point = vector_binary_search(info_hash, (*adding_to)->list, (*adding_to)->size, sizeof(ot_hash), OT_HASH_COMPARE_SIZE, &exactmatch); + + /* Only if the info hash is not in the adding_to list, create a new head with that entry */ + if (!exactmatch) { + ot_accesslist *accesslist_new = accesslist_make(*adding_to, (*adding_to)->size + 1); + ptrdiff_t off = insert_point - (*adding_to)->list; + if (accesslist_new) { + memcpy(accesslist_new->list, (*adding_to)->list, sizeof(ot_hash) * off); + memcpy(accesslist_new->list + off, info_hash, sizeof(info_hash)); + memcpy(accesslist_new->list + off + 1, (*adding_to)->list + off, sizeof(ot_hash) * ((*adding_to)->size - off)); + (*adding_to)->base = time_now_minutes(); + *adding_to = accesslist_new; + } + } + } + + pthread_mutex_unlock(&g_accesslist_mutex); + } + + free(line); + fclose(fifo); + } + return NULL; +} + +static void *accesslist_adder_worker(void *args) { + (void)args; + accesslist_adddel_worker(g_accesslist_pipe_add, &g_accesslist_add, &g_accesslist_delete); + atomic_store(&adder_left, 1); + return NULL; +} +static void *accesslist_deleter_worker(void *args) { + (void)args; + accesslist_adddel_worker(g_accesslist_pipe_delete, &g_accesslist_delete, &g_accesslist_add); + atomic_store(&deleter_left, 1); + return NULL; +} + +/* These sit in a blocking open or read on a FIFO, so they are woken the way + the other workers are, and the signal is repeated because it only + interrupts a call already under way. The pipe name is what says whether + the thread was ever started. */ +static void accesslist_stop_fifo_worker(pthread_t thread, _Atomic int *left, const char *pipe_name) { + if (!pipe_name) + return; + + while (!atomic_load(left)) { + pthread_kill(thread, OT_SIGNAL_WAKE); + usleep(1000); + } + pthread_join(thread, NULL); +} +#endif + +static pthread_t thread_id; +void accesslist_init(void) { + pthread_mutex_init(&g_accesslist_mutex, NULL); + pthread_create(&thread_id, NULL, accesslist_worker, NULL); +#ifdef WANT_DYNAMIC_ACCESSLIST + if (g_accesslist_pipe_add) + pthread_create(&thread_adder_id, NULL, accesslist_adder_worker, NULL); + if (g_accesslist_pipe_delete) + pthread_create(&thread_deleter_id, NULL, accesslist_deleter_worker, NULL); +#endif +} + +void accesslist_deinit(void) { + /* Wake the worker and wait for it to notice that the tracker is stopping. */ + pthread_kill(thread_id, SIGHUP); + pthread_join(thread_id, NULL); + +#ifdef WANT_DYNAMIC_ACCESSLIST + /* Before the lists they are reading are freed below */ + accesslist_stop_fifo_worker(thread_adder_id, &adder_left, g_accesslist_pipe_add); + accesslist_stop_fifo_worker(thread_deleter_id, &deleter_left, g_accesslist_pipe_delete); +#endif + + pthread_mutex_lock(&g_accesslist_mutex); + + g_accesslist = accesslist_free(g_accesslist); + +#ifdef WANT_DYNAMIC_ACCESSLIST + g_accesslist_add = accesslist_free(g_accesslist_add); + g_accesslist_delete = accesslist_free(g_accesslist_delete); +#endif + + pthread_mutex_unlock(&g_accesslist_mutex); + pthread_mutex_destroy(&g_accesslist_mutex); +} + +void accesslist_cleanup(void) { + pthread_mutex_lock(&g_accesslist_mutex); + + accesslist_clean(g_accesslist); +#if WANT_DYNAMIC_ACCESSLIST + accesslist_clean(g_accesslist_add); + accesslist_clean(g_accesslist_delete); +#endif + + pthread_mutex_unlock(&g_accesslist_mutex); +} +#endif + +int accesslist_address_in_net(const ot_ip6 address, const ot_net *net) { + int bits = net->bits, checkbits = (0x7f00 >> (bits & 7)); + int result = memcmp(address, &net->address, bits >> 3); + if (!result && (bits & 7)) + result = (checkbits & address[bits >> 3]) - (checkbits & net->address[bits >> 3]); + return result == 0; +} + +void *accesslist_set_value_for_net(const ot_net *net, ot_vector *vector, const void *value, const size_t member_size) { + size_t i; + int exactmatch; + + /* Caller must have a concept of ot_net in it's member */ + if (member_size < sizeof(ot_net)) + return 0; + + /* Check each net in vector for overlap */ + uint8_t *member = ((uint8_t *)vector->data); + for (i = 0; i < vector->size; ++i) { + if (accesslist_address_in_net(*(ot_ip6 *)member, net) || accesslist_address_in_net(net->address, (ot_net *)member)) + return 0; + member += member_size; + } + + member = vector_find_or_insert(vector, (void *)net, member_size, sizeof(ot_net), &exactmatch); + if (member) { + memcpy(member, net, sizeof(ot_net)); + memcpy(member + sizeof(ot_net), value, member_size - sizeof(ot_net)); + } + + return member; +} + +/* Takes a vector filled with { ot_net net, uint8_t[x] value }; + Returns value associated with the net, or NULL if not found */ +void *accesslist_get_value_for_net(const ot_ip6 address, const ot_vector *vector, const size_t member_size) { + int exactmatch; + /* This binary search will return a pointer to the first non-containing network... */ + ot_net *net = vector_binary_search(address, vector->data, vector->size, member_size, sizeof(ot_ip6), &exactmatch); + if (!net) + return NULL; + /* ... so we'll need to move back one step unless we've exactly hit the first address in network */ + if (!exactmatch && ((void *)net > vector->data)) + --net; + if (!accesslist_address_in_net(address, net)) + return NULL; + return (void *)net; +} + +#ifdef WANT_IP_FROM_PROXY +typedef struct { + ot_net *proxy; + ot_vector networks; +} ot_proxymap; + +static ot_vector g_proxies_list; +static pthread_mutex_t g_proxies_list_mutex = PTHREAD_MUTEX_INITIALIZER; + +int proxylist_add_network(const ot_net *proxy, const ot_net *net) { + ot_proxymap *map; + int exactmatch, result = 1; + pthread_mutex_lock(&g_proxies_list_mutex); + + /* If we have a direct hit, use and extend the vector there */ + map = vector_binary_search(proxy, g_proxies_list.data, g_proxies_list.size, sizeof(ot_proxymap), sizeof(ot_net), &exactmatch); + + if (!map || !exactmatch) { + /* else see, if we've got overlapping networks + and get a new empty vector if not */ + ot_vector empty; + memset(&empty, 0, sizeof(ot_vector)); + map = accesslist_set_value_for_net(proxy, &g_proxies_list, &empty, sizeof(ot_proxymap)); + } + + if (map && accesslist_set_value_for_net(net, &map->networks, NULL, sizeof(ot_net))) + result = 1; + + pthread_mutex_unlock(&g_proxies_list_mutex); + return result; +} + +int proxylist_check_proxy(const ot_ip6 proxy, const ot_ip6 address) { + int result = 0; + ot_proxymap *map; + + pthread_mutex_lock(&g_proxies_list_mutex); + + if ((map = accesslist_get_value_for_net(proxy, &g_proxies_list, sizeof(ot_proxymap)))) + if (!address || accesslist_get_value_for_net(address, &map->networks, sizeof(ot_net))) + result = 1; + + pthread_mutex_unlock(&g_proxies_list_mutex); + return result; +} + +#endif + +static ot_net g_admin_nets[OT_ADMINIP_MAX]; +static ot_permissions g_admin_nets_permissions[OT_ADMINIP_MAX]; +static size_t g_admin_nets_count = 0; + +int accesslist_bless_net(ot_net *net, ot_permissions permissions) { + if (g_admin_nets_count >= OT_ADMINIP_MAX) + return -1; + + memcpy(g_admin_nets + g_admin_nets_count, net, sizeof(ot_net)); + g_admin_nets_permissions[g_admin_nets_count++] = permissions; + +#ifdef _DEBUG + { + char _debug[512]; + int off = snprintf(_debug, sizeof(_debug), "Blessing ip net "); + off += ip_format(_debug + off, net->address); + if (net->bits < 128) { + _debug[off++] = '/'; + if (ip_is_v4mapped(net->address)) + off += snprintf(_debug + off, sizeof(_debug) - off, "%d", net->bits - 96); + else + off += snprintf(_debug + off, sizeof(_debug) - off, "%d", net->bits); + } + + if (permissions & OT_PERMISSION_MAY_STAT) + off += snprintf(_debug + off, 512 - off, " may_fetch_stats"); + if (permissions & OT_PERMISSION_MAY_LIVESYNC) + off += snprintf(_debug + off, 512 - off, " may_sync_live"); + if (permissions & OT_PERMISSION_MAY_FULLSCRAPE) + off += snprintf(_debug + off, 512 - off, " may_fetch_fullscrapes"); + if (permissions & OT_PERMISSION_MAY_PROXY) + off += snprintf(_debug + off, 512 - off, " may_proxy"); + if (!permissions) + off += snprintf(_debug + off, sizeof(_debug) - off, " nothing"); + _debug[off++] = '.'; + _debug[off++] = '\n'; + (void)write(2, _debug, off); + } +#endif + + return 0; +} + +int accesslist_is_blessed(ot_ip6 ip, ot_permissions permissions) { + size_t i; + for (i = 0; i < g_admin_nets_count; ++i) + if (accesslist_address_in_net(ip, g_admin_nets + i) && (g_admin_nets_permissions[i] & permissions)) + return 1; + return 0; +} diff --git a/src/ot_accesslist.h b/src/ot_accesslist.h new file mode 100644 index 0000000..68c88dc --- /dev/null +++ b/src/ot_accesslist.h @@ -0,0 +1,77 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_ACCESSLIST_H__ +#define OT_ACCESSLIST_H__ + +#include "trackerlogic.h" + +#if defined(WANT_ACCESSLIST_BLACK) && defined(WANT_ACCESSLIST_WHITE) +#error WANT_ACCESSLIST_BLACK and WANT_ACCESSLIST_WHITE are exclusive. +#endif + +#if defined(WANT_ACCESSLIST_BLACK) || defined(WANT_ACCESSLIST_WHITE) +#define WANT_ACCESSLIST +/* Starts a thread that rereads the list when sent SIGHUP, and is stopped by + being sent one, so the caller blocks SIGHUP process wide beforehand */ +void accesslist_init(void); +void accesslist_deinit(void); +int accesslist_hashisvalid(const ot_hash hash); +void accesslist_cleanup(void); + +/* Scan buf for up to capacity info_hashes in hexadecimal notation, one per + line, and return how many were written to dest */ +size_t accesslist_parse_hashes(ot_hash *dest, size_t capacity, const char *buf, size_t len); + +extern const char *g_accesslist_filename; +#ifdef WANT_DYNAMIC_ACCESSLIST +extern const char *g_accesslist_pipe_add; +extern const char *g_accesslist_pipe_delete; +#endif + +#else +#ifdef WANT_DYNAMIC_ACCESSLIST +#error WANT_DYNAMIC_ACCESSLIST needs either WANT_ACCESSLIST_BLACK or WANT_ACCESSLIST_WHITE +#endif + +#define accesslist_init(accesslist_filename) +#define accesslist_deinit() +#define accesslist_hashisvalid(hash) 1 +#endif + +/* Test if an address is subset of an ot_net, return value is considered a bool */ +int accesslist_address_in_net(const ot_ip6 address, const ot_net *net); + +/* Store a value into a vector of struct { ot_net net, uint8_t[x] value } member; + returns NULL + if member_size is too small, or + if one of the nets inside the vector are a subnet of _net_, or + if _net_ is a subnet of one of the nets inside the vector, or + if the vector could not be resized + returns pointer to new member in vector for success + member_size can be sizeof(ot_net) to reduce the lookup to a boolean mapping +*/ +void *accesslist_set_value_for_net(const ot_net *net, ot_vector *vector, const void *value, const size_t member_size); + +/* Takes a vector filled with struct { ot_net net, uint8_t[x] value } member; + Returns pointer to _member_ associated with the net, or NULL if not found + member_size can be sizeof(ot_net) to reduce the lookup to a boolean mapping +*/ +void *accesslist_get_value_for_net(const ot_ip6 address, const ot_vector *vector, const size_t member_size); + +#ifdef WANT_IP_FROM_PROXY +int proxylist_add_network(const ot_net *proxy, const ot_net *net); +int proxylist_check_network(const ot_ip6 *proxy, const ot_ip6 address /* can be NULL to only check proxy */); +#endif + +typedef enum { + OT_PERMISSION_MAY_FULLSCRAPE = 0x1, + OT_PERMISSION_MAY_STAT = 0x2, + OT_PERMISSION_MAY_LIVESYNC = 0x4, + OT_PERMISSION_MAY_PROXY = 0x8 +} ot_permissions; + +int accesslist_bless_net(ot_net *net, ot_permissions permissions); +int accesslist_is_blessed(ot_ip6 ip, ot_permissions permissions); + +#endif diff --git a/src/ot_batch.c b/src/ot_batch.c new file mode 100644 index 0000000..e4dda0b --- /dev/null +++ b/src/ot_batch.c @@ -0,0 +1,156 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_batch.h" + +#ifndef IOV_MAX +#define IOV_MAX 1024 +#endif + +#define OT_BATCH_MIN_ENTRIES 16 + +static int batch_room_for(ot_batch *batch, size_t wanted) { + size_t space = batch->space ? batch->space : OT_BATCH_MIN_ENTRIES; + struct iovec *grown; + + if (batch->count + wanted <= batch->space) + return 0; + + /* During a chunked transfer entries are appended while earlier ones are + still going out, so the front of the array frees up as we go. Slide the + rest down before asking for more. */ + if (batch->first && batch->count - batch->first + wanted <= batch->space) { + memmove(batch->entries, batch->entries + batch->first, (batch->count - batch->first) * sizeof(struct iovec)); + batch->count -= batch->first; + batch->first = 0; + return 0; + } + + while (space < batch->count + wanted) + space *= 2; + + if (!(grown = realloc(batch->entries, space * sizeof(struct iovec)))) + return -1; + + batch->entries = grown; + batch->space = space; + return 0; +} + +int batch_add(ot_batch *batch, void *buf, size_t len) { + if (batch_room_for(batch, 1)) { + free(buf); + return -1; + } + + batch->entries[batch->count].iov_base = buf; + batch->entries[batch->count].iov_len = len; + ++batch->count; + return 0; +} + +int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count) { + if (batch_room_for(batch, count)) { + while (count--) + free(entries[count].iov_base); + return -1; + } + + memcpy(batch->entries + batch->count, entries, count * sizeof(struct iovec)); + batch->count += count; + return 0; +} + +/* Drop everything the kernel accepted, freeing each buffer as it goes */ +static void batch_consume(ot_batch *batch, size_t written) { + while (written) { + struct iovec *entry = batch->entries + batch->first; + size_t left = entry->iov_len - batch->offset; + + if (written < left) { + batch->offset += written; + return; + } + + free(entry->iov_base); + written -= left; + batch->offset = 0; + ++batch->first; + } + + /* Nothing is outstanding, so the array can start over from the front */ + if (batch->first == batch->count) + batch->first = batch->count = 0; +} + +/* A peer that has gone away must come back as an error rather than as a + signal that kills the tracker, so ask the kernel not to raise one where it + can be asked. macOS cannot, and relies on SIGPIPE being blocked. */ +static ssize_t batch_writev(int s, struct iovec *entries, int count) { +#ifdef MSG_NOSIGNAL + struct msghdr msg; + + memset(&msg, 0, sizeof(msg)); + msg.msg_iov = entries; + msg.msg_iovlen = count; + return sendmsg(s, &msg, MSG_NOSIGNAL); +#else + return writev(s, entries, count); +#endif +} + +ssize_t batch_send(int s, ot_batch *batch) { + size_t entries = batch->count - batch->first; + ssize_t written; + + if (!batch_pending(batch)) + return BATCH_DONE; + + if (entries > IOV_MAX) + entries = IOV_MAX; + + /* The first entry may be half way out already */ + batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base + batch->offset; + batch->entries[batch->first].iov_len -= batch->offset; + written = batch_writev(s, batch->entries + batch->first, (int)entries); + batch->entries[batch->first].iov_base = (char *)batch->entries[batch->first].iov_base - batch->offset; + batch->entries[batch->first].iov_len += batch->offset; + + if (written < 0) + return (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) ? BATCH_AGAIN : BATCH_FAILED; + if (!written) + return BATCH_FAILED; + + batch_consume(batch, (size_t)written); + return written; +} + +void batch_reset(ot_batch *batch) { + while (batch->first < batch->count) + free(batch->entries[batch->first++].iov_base); + + free(batch->entries); + batch->entries = NULL; + batch->count = batch->first = batch->offset = batch->space = 0; +} + +/* For the few constants the chunked encoding needs, which the batch frees + along with everything else */ +int batch_add_copy(ot_batch *batch, const void *buf, size_t len) { + void *copy = malloc(len); + + if (!copy) + return -1; + memcpy(copy, buf, len); + return batch_add(batch, copy, len); +} diff --git a/src/ot_batch.h b/src/ot_batch.h new file mode 100644 index 0000000..910bea7 --- /dev/null +++ b/src/ot_batch.h @@ -0,0 +1,46 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_BATCH_H__ +#define OT_BATCH_H__ + +#include +#include +#include + +/* A queue of buffers on their way to one socket. The batch owns every buffer + handed to it and frees each one as soon as it has left, so a client that + reads slowly costs memory only for what it has not received yet. + + A zeroed ot_batch is ready to use. */ +typedef struct { + struct iovec *entries; + size_t space; + size_t count; + size_t first; /* entries before this one are sent and freed */ + size_t offset; /* bytes of entries[first] already sent */ +} ot_batch; + +static inline int batch_pending(const ot_batch *batch) { return batch->first != batch->count; } + +/* What batch_send reports instead of a byte count */ +enum { BATCH_DONE = 0, BATCH_AGAIN = -1, BATCH_FAILED = -2 }; + +/* Queue buf and take ownership of it. Returns 0, or -1 having freed buf. */ +int batch_add(ot_batch *batch, void *buf, size_t len); + +/* Write as much as the socket takes. Returns the number of bytes written, or + BATCH_DONE, BATCH_AGAIN or BATCH_FAILED. */ +ssize_t batch_send(int s, ot_batch *batch); + +/* Queue every entry of a vector, taking ownership of each buffer but not of + the vector itself. Returns 0, or -1 having freed all of them. */ +int batch_add_vector(ot_batch *batch, const struct iovec *entries, size_t count); + +/* Queue a copy of buf, which the batch then owns. Returns 0, or -1. */ +int batch_add_copy(ot_batch *batch, const void *buf, size_t len); + +/* Free whatever has not left yet, along with the batch's own array */ +void batch_reset(ot_batch *batch); + +#endif diff --git a/src/ot_buf.c b/src/ot_buf.c new file mode 100644 index 0000000..ecd3f8d --- /dev/null +++ b/src/ot_buf.c @@ -0,0 +1,39 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_buf.h" + +int buf_cat(ot_buf *buf, const void *src, size_t len) { + char *data; + + if (!len) + return 0; + + if (buf->size + len < buf->size) + return -1; + + if (!(data = realloc(buf->data, buf->size + len))) + return -1; + + buf->data = data; + memcpy(data + buf->size, src, len); + buf->size += len; + + return 0; +} + +void buf_truncate(ot_buf *buf, size_t len) { + if (len < buf->size) + buf->size = len; +} + +void buf_reset(ot_buf *buf) { + free(buf->data); + buf->data = NULL; + buf->size = 0; +} diff --git a/src/ot_buf.h b/src/ot_buf.h new file mode 100644 index 0000000..7fb944e --- /dev/null +++ b/src/ot_buf.h @@ -0,0 +1,25 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_BUF_H__ +#define OT_BUF_H__ + +#include + +/* A growable byte buffer, holding an HTTP request that did not arrive in a + single packet. Requests that arrive whole are served straight from the + worker's input buffer and never come near this. */ +typedef struct { + char *data; + size_t size; +} ot_buf; + +/* Append len bytes from src. Returns 0, or -1 if the memory could not be + had, in which case the buffer keeps whatever it already held. */ +int buf_cat(ot_buf *buf, const void *src, size_t len); + +void buf_truncate(ot_buf *buf, size_t len); + +void buf_reset(ot_buf *buf); + +#endif diff --git a/src/ot_clean.c b/src/ot_clean.c new file mode 100644 index 0000000..f181e83 --- /dev/null +++ b/src/ot_clean.c @@ -0,0 +1,176 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_clean.h" +#include "ot_mutex.h" +#include "ot_stats.h" +#include "ot_vector.h" +#include "trackerlogic.h" + +/* Returns amount of removed peers */ +static ssize_t clean_single_bucket(ot_peer *peers, size_t peer_count, size_t peer_size, time_t timedout, size_t *removed_seeders) { + ot_peer *last_peer = peers + peer_count * peer_size, *insert_point; + + /* Two scan modes: unless there is one peer removed, just increase ot_peertime */ + while (peers < last_peer) { + time_t timediff = timedout + OT_PEERTIME(peers, peer_size); + if (timediff >= OT_PEER_TIMEOUT) + break; + OT_PEERTIME(peers, peer_size) = timediff; + peers += peer_size; + } + + /* If we at least remove one peer, we have to copy */ + for (insert_point = peers; peers < last_peer; peers += peer_size) { + time_t timediff = timedout + OT_PEERTIME(peers, peer_size); + + if (timediff < OT_PEER_TIMEOUT) { + OT_PEERTIME(peers, peer_size) = timediff; + memcpy(insert_point, peers, peer_size); + insert_point += peer_size; + } else if (OT_PEERFLAG_D(peers, peer_size) & PEER_FLAG_SEEDING) + (*removed_seeders)++; + } + + return (peers - insert_point) / peer_size; +} + +int clean_single_peer_list(ot_peerlist *peer_list, size_t peer_size) { + ot_vector *peer_vector = &peer_list->peers; + time_t timedout = (time_t)(time_now_minutes() - peer_list->base); + size_t num_buckets = 1, removed_seeders = 0; + + /* No need to clean empty torrent */ + if (!timedout) + return 0; + + /* Torrent has idled out */ + if (timedout > OT_TORRENT_TIMEOUT) + return 1; + + /* Nothing to be cleaned here? Test if torrent is worth keeping */ + if (timedout > OT_PEER_TIMEOUT) { + if (!peer_list->peer_count) + return peer_list->down_count ? 0 : 1; + timedout = OT_PEER_TIMEOUT; + } + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets = peer_vector->size; + peer_vector = (ot_vector *)peer_vector->data; + } + + while (num_buckets--) { + size_t removed_peers = clean_single_bucket(peer_vector->data, peer_vector->size, peer_size, timedout, &removed_seeders); + peer_list->peer_count -= removed_peers; + peer_vector->size -= removed_peers; + if (removed_peers) + vector_fixup_peers(peer_vector, peer_size); + + /* Skip to next bucket, a vector containing peers */ + ++peer_vector; + } + + peer_list->seed_count -= removed_seeders; + + /* See if we need to convert a torrent from simple vector to bucket list */ + if ((peer_list->peer_count > OT_PEER_BUCKET_MINCOUNT) || OT_PEERLIST_HASBUCKETS(peer_list)) + vector_redistribute_buckets(peer_list, peer_size); + + if (peer_list->peer_count) + peer_list->base = time_now_minutes(); + else { + /* When we got here, the last time that torrent + has been touched is OT_PEER_TIMEOUT Minutes before */ + peer_list->base = time_now_minutes() - OT_PEER_TIMEOUT; + } + return 0; +} + +/* Clean a single torrent + return 1 if torrent timed out +*/ +int clean_single_torrent(ot_torrent *torrent) { + return clean_single_peer_list(torrent->peer_list6, OT_PEER_SIZE6) * clean_single_peer_list(torrent->peer_list4, OT_PEER_SIZE4); +} + +static pthread_mutex_t g_clean_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t g_clean_wakeup = PTHREAD_COND_INITIALIZER; + +static void clean_sleep(void) { + struct timespec deadline; + + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += OT_CLEAN_SLEEP / 1000000; + deadline.tv_nsec += (OT_CLEAN_SLEEP % 1000000) * 1000; + if (deadline.tv_nsec >= 1000000000) { + ++deadline.tv_sec; + deadline.tv_nsec -= 1000000000; + } + + /* Both sides hold the mutex, so the flag cannot be cleared and signalled + in the window between testing it and waiting on it */ + pthread_mutex_lock(&g_clean_mutex); + while (g_opentracker_running) + if (pthread_cond_timedwait(&g_clean_wakeup, &g_clean_mutex, &deadline) == ETIMEDOUT) + break; + pthread_mutex_unlock(&g_clean_mutex); +} + +/* Clean up all peers in one bucket, remove timedout pools and torrents */ +static void clean_bucket(size_t bucket) { + ot_vector *torrents_list = mutex_bucket_lock((int)bucket); + size_t toffs; + int delta_torrentcount = 0; + + for (toffs = 0; toffs < torrents_list->size; ++toffs) { + ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + toffs; + if (clean_single_torrent(torrent)) { + vector_remove_torrent(torrents_list, torrent); + --delta_torrentcount; + --toffs; + } + } + mutex_bucket_unlock((int)bucket, delta_torrentcount); +} + +static void *clean_worker(void *args) { + (void)args; + while (1) { + size_t bucket = OT_BUCKET_COUNT; + while (bucket) { + size_t todo = bucket < OT_CLEAN_BUCKETS_PER_WAKE ? bucket : OT_CLEAN_BUCKETS_PER_WAKE; + + /* One bucket is locked at a time, however many a wake gets through */ + while (todo--) + clean_bucket(--bucket); + + if (!g_opentracker_running) + return NULL; + clean_sleep(); + } + stats_cleanup(); +#ifdef WANT_ACCESSLIST + accesslist_cleanup(); +#endif + } + return NULL; +} + +static pthread_t thread_id; +void clean_init(void) { pthread_create(&thread_id, NULL, clean_worker, NULL); } + +void clean_deinit(void) { + pthread_mutex_lock(&g_clean_mutex); + pthread_cond_signal(&g_clean_wakeup); + pthread_mutex_unlock(&g_clean_mutex); + pthread_join(thread_id, NULL); +} diff --git a/src/ot_clean.h b/src/ot_clean.h new file mode 100644 index 0000000..0592791 --- /dev/null +++ b/src/ot_clean.h @@ -0,0 +1,28 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_CLEAN_H__ +#define OT_CLEAN_H__ + +#include "trackerlogic.h" + +/* The amount of time a clean cycle should take */ +#define OT_CLEAN_INTERVAL_MINUTES 2 + +/* How often the cleaner may wake. A full sweep takes OT_CLEAN_INTERVAL_MINUTES. + The clean operations per wake is goverend by OT_BUCKET_COUNT_BITS, + set by the Makefile. + At the default of 1024 buckets this works out at one bucket every + 117 ms. */ +#define OT_CLEAN_MAX_WAKES_PER_SECOND 10 + +#define OT_CLEAN_WAKE_BUDGET ((OT_CLEAN_INTERVAL_MINUTES) * 60 * (OT_CLEAN_MAX_WAKES_PER_SECOND)) +#define OT_CLEAN_BUCKETS_PER_WAKE (((OT_BUCKET_COUNT) + (OT_CLEAN_WAKE_BUDGET) - 1) / (OT_CLEAN_WAKE_BUDGET)) +#define OT_CLEAN_WAKES ((OT_BUCKET_COUNT) / (OT_CLEAN_BUCKETS_PER_WAKE)) +#define OT_CLEAN_SLEEP (((OT_CLEAN_INTERVAL_MINUTES) * 60 * 1000000) / (OT_CLEAN_WAKES)) + +void clean_init(void); +void clean_deinit(void); +int clean_single_torrent(ot_torrent *torrent); + +#endif diff --git a/src/ot_config.c b/src/ot_config.c new file mode 100644 index 0000000..95136b5 --- /dev/null +++ b/src/ot_config.c @@ -0,0 +1,586 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_event.h" +#include "ot_config.h" +#include "ot_http.h" +#include "ot_ip.h" +#include "ot_livesync.h" +#include "ot_scan.h" +#include "ot_stats.h" +#include "trackerlogic.h" + +const char *g_redirecturl; +const char *g_serverdir; +const char *g_serveruser; +const char *g_statefile; + +#define OT_LISTEN_INHERIT (-1) + +static ot_listen *g_listen; +static size_t g_listen_count; +static int g_listen_shared; +static int g_listen_workers_udp = 1; +static int g_listen_workers_tcp = 0; + +static void panic(const char *routine) __attribute__((noreturn)); +static void panic(const char *routine) { + fprintf(stderr, "%s: %s\n", routine, strerror(errno)); + exit(111); +} + +static void listen_append(const ot_ip6 ip, uint16_t port, PROTO_FLAG proto, int shared, int workers) { + ot_listen *grown = realloc(g_listen, (g_listen_count + 1) * sizeof(ot_listen)); + ot_listen *entry; + + if (!grown) + panic("listen_append"); + g_listen = grown; + + entry = g_listen + g_listen_count++; + memcpy(entry->ip, ip, sizeof(ot_ip6)); + entry->port = port; + entry->proto = proto; + entry->shared = shared; + entry->workers = workers; + entry->sock = -1; +} + +typedef enum { LISTEN_OPTION_UNKNOWN, LISTEN_OPTION_SHARED, LISTEN_OPTION_WORKERS } listen_option_kind; + +typedef struct { + const char *name; + size_t len; + listen_option_kind kind; + int takes_value; +} listen_option_key; + +#define LISTEN_OPTION_KEY(s, k, v) {s, sizeof(s) - 1, k, v} + +static const listen_option_key listen_option_keys[] = { + LISTEN_OPTION_KEY("shared", LISTEN_OPTION_SHARED, 0), + LISTEN_OPTION_KEY("workers", LISTEN_OPTION_WORKERS, 1), +}; +static const size_t listen_option_count = sizeof(listen_option_keys) / sizeof(listen_option_keys[0]); + +static listen_option_kind parse_listen_option(char **p) { + size_t i; + + for (i = 0; i < listen_option_count; ++i) { + const listen_option_key *key = &listen_option_keys[i]; + char delimiter = (*p)[key->len]; + + if (strncmp(*p, key->name, key->len)) + continue; + if (key->takes_value ? delimiter != '=' : (delimiter && !isspace((unsigned char)delimiter))) + continue; + + *p += key->len + (key->takes_value ? 1 : 0); + return key->kind; + } + return LISTEN_OPTION_UNKNOWN; +} + +/* Options trailing an address in a listen statement, in any order */ +static int listen_scan_options(char *p, int *shared, int *workers) { + uint64_t value; + + for (;;) { + while (isspace((unsigned char)*p)) + ++p; + if (!*p) + return 1; + + switch (parse_listen_option(&p)) { + case LISTEN_OPTION_SHARED: + *shared = 1; + break; + case LISTEN_OPTION_WORKERS: { + size_t off = scan_uint(p, &value, UINT_MAX); + + if (!off) + return 0; + *workers = (int)value; + p += off; + break; + } + default: + return 0; + } + } +} + + +static const char *set_config_option(const char **option, char *value) { +#ifdef _DEBUG + fprintf(stderr, "Setting config option: %s\n", value); +#endif + while (isspace(*value)) + ++value; + free((char *)*option); + return *option = strdup(value); +} + +static int scan_ip6_port(const char *src, ot_ip6 ip, uint16_t *port) { + const char *s = src; + uint64_t value; + int off, bracket = 0; + while (isspace(*s)) + ++s; + if (*s == '[') + ++s, ++bracket; /* for v6 style notation */ + if (!(off = ip_parse(s, ip))) + return 0; + s += off; + if (bracket && *s == ']') + ++s; + if (*s == 0 || isspace(*s)) + return s - src; + if (!ip_is_v4mapped(ip)) { + if (*s != ':' && *s != '.') + return 0; + if (!bracket && *(s) == ':') + return 0; + s++; + } else { + if (*(s++) != ':') + return 0; + } + if (!(off = scan_uint(s, &value, 65535))) + return 0; + *port = (uint16_t)value; + return off + s - src; +} + +static int scan_ip6_net(const char *src, ot_net *net) { + const char *s = src; + uint64_t value; + int off; + while (isspace(*s)) + ++s; + if (!(off = ip_parse(s, net->address))) + return 0; + s += off; + if (*s != '/') + net->bits = 128; + else { + s++; + if (!(off = scan_uint(s, &value, 128))) + return 0; + net->bits = (int)value; + if (ip_is_v4mapped(net->address)) + net->bits += 96; + if (net->bits > 128) + return 0; + s += off; + } + return off + s - src; +} + +typedef enum { + CONFIG_UNKNOWN, + CONFIG_TRACKER_ROOTDIR, + CONFIG_TRACKER_USER, + CONFIG_LISTEN_TCP_UDP, + CONFIG_LISTEN_TCP, + CONFIG_LISTEN_UDP, + CONFIG_LISTEN_UDP_WORKERS, + CONFIG_LISTEN_TCP_WORKERS, + CONFIG_LISTEN_SHARED, + CONFIG_ACCESS_WHITELIST, + CONFIG_ACCESS_BLACKLIST, + CONFIG_ACCESS_FIFO_ADD, + CONFIG_ACCESS_FIFO_DELETE, + CONFIG_ACCESS_STATS, + CONFIG_ACCESS_STATS_PATH, + CONFIG_ACCESS_PROXY, + CONFIG_TRACKER_REDIRECT_URL, + CONFIG_LIVESYNC_CLUSTER_NODE_IP, + CONFIG_LIVESYNC_CLUSTER_LISTEN, +} config_kind; + +typedef struct { + const char *name; + size_t len; + config_kind kind; +} config_key; + +#define CONFIG_KEY(s, k) { s, sizeof(s) - 1, k } + +static const config_key config_keys[] = { + CONFIG_KEY("tracker.rootdir", CONFIG_TRACKER_ROOTDIR), + CONFIG_KEY("tracker.user", CONFIG_TRACKER_USER), + CONFIG_KEY("listen.tcp_udp", CONFIG_LISTEN_TCP_UDP), + CONFIG_KEY("listen.tcp", CONFIG_LISTEN_TCP), + CONFIG_KEY("listen.udp", CONFIG_LISTEN_UDP), + CONFIG_KEY("listen.udp.workers", CONFIG_LISTEN_UDP_WORKERS), + CONFIG_KEY("listen.tcp.workers", CONFIG_LISTEN_TCP_WORKERS), + CONFIG_KEY("listen.shared", CONFIG_LISTEN_SHARED), +#ifdef WANT_ACCESSLIST_WHITE + CONFIG_KEY("access.whitelist", CONFIG_ACCESS_WHITELIST), +#elif defined(WANT_ACCESSLIST_BLACK) + CONFIG_KEY("access.blacklist", CONFIG_ACCESS_BLACKLIST), +#endif +#ifdef WANT_DYNAMIC_ACCESSLIST + CONFIG_KEY("access.fifo_add", CONFIG_ACCESS_FIFO_ADD), + CONFIG_KEY("access.fifo_delete", CONFIG_ACCESS_FIFO_DELETE), +#endif +#ifdef WANT_RESTRICT_STATS + CONFIG_KEY("access.stats", CONFIG_ACCESS_STATS), +#endif + CONFIG_KEY("access.stats_path", CONFIG_ACCESS_STATS_PATH), +#ifdef WANT_IP_FROM_PROXY + CONFIG_KEY("access.proxy", CONFIG_ACCESS_PROXY), +#endif + CONFIG_KEY("tracker.redirect_url", CONFIG_TRACKER_REDIRECT_URL), +#ifdef WANT_SYNC_LIVE + CONFIG_KEY("livesync.cluster.node_ip", CONFIG_LIVESYNC_CLUSTER_NODE_IP), + CONFIG_KEY("livesync.cluster.listen", CONFIG_LIVESYNC_CLUSTER_LISTEN), +#endif +}; +static const size_t config_entry_count = sizeof(config_keys) / sizeof(config_keys[0]); + +static config_kind parse_config_key(char **p) { + size_t i; + for (i = 0; i < config_entry_count; ++i) { + const config_key *key = &config_keys[i]; + if (strncmp(*p, key->name, key->len) == 0 && isspace((unsigned char)(*p)[key->len])) { + *p += key->len + 1; + /* Skip white spaces */ + while (isspace((unsigned char)**p)) + ++(*p); + return key->kind; + } + } + return CONFIG_UNKNOWN; +} + +static int parse_configfile(char *config_filename) { + FILE *config_file_handle; + char inbuf[512]; + ot_ip6 tmpip; +#if defined(WANT_RESTRICT_STATS) || defined(WANT_IP_FROM_PROXY) || defined(WANT_SYNC_LIVE) + ot_net tmpnet; +#endif + int bound = 0; + + config_file_handle = fopen(config_filename, "r"); + + if (config_file_handle == NULL) { + fprintf(stderr, "Warning: Can't open config file: %s.", config_filename); + return 0; + } + + while (fgets(inbuf, sizeof(inbuf), config_file_handle)) { + char *p = inbuf, *end; + uint64_t value; + uint16_t tmpport = 6969; + config_kind kind; + + /* Skip white spaces */ + while (isspace((unsigned char)*p)) + ++p; + + /* Ignore comments and empty lines */ + if ((*p == '#') || (*p == '\n') || (*p == 0)) + continue; + + /* consume trailing new lines and spaces */ + end = p + strlen(p); + while (end > p && isspace((unsigned char)end[-1])) + *--end = 0; + + /* Scan for commands */ + kind = parse_config_key(&p); + switch (kind) { + case CONFIG_TRACKER_ROOTDIR: + set_config_option(&g_serverdir, p); + break; + case CONFIG_TRACKER_USER: + set_config_option(&g_serveruser, p); + break; + case CONFIG_LISTEN_TCP_UDP: + case CONFIG_LISTEN_TCP: + case CONFIG_LISTEN_UDP: { + int shared = OT_LISTEN_INHERIT; + int workers = OT_LISTEN_INHERIT; + int off = scan_ip6_port(p, tmpip, &tmpport); + + if (!off || !listen_scan_options(p + off, &shared, &workers)) + goto parse_error; + if (kind != CONFIG_LISTEN_UDP) + listen_append(tmpip, tmpport, FLAG_TCP, shared, workers), ++bound; + if (kind != CONFIG_LISTEN_TCP) + listen_append(tmpip, tmpport, FLAG_UDP, shared, workers), ++bound; + break; + } + case CONFIG_LISTEN_UDP_WORKERS: + if (!scan_uint(p, &value, UINT_MAX)) + goto parse_error; + g_listen_workers_udp = (int)value; + break; + case CONFIG_LISTEN_TCP_WORKERS: + if (!scan_uint(p, &value, UINT_MAX)) + goto parse_error; + g_listen_workers_tcp = (int)value; + break; + case CONFIG_LISTEN_SHARED: + if (!scan_uint(p, &value, 1)) + goto parse_error; + g_listen_shared = (int)value; + break; +#ifdef WANT_ACCESSLIST_WHITE + case CONFIG_ACCESS_WHITELIST: + set_config_option(&g_accesslist_filename, p); + break; +#elif defined(WANT_ACCESSLIST_BLACK) + case CONFIG_ACCESS_BLACKLIST: + set_config_option(&g_accesslist_filename, p); + break; +#endif +#ifdef WANT_DYNAMIC_ACCESSLIST + case CONFIG_ACCESS_FIFO_ADD: + set_config_option(&g_accesslist_pipe_add, p); + break; + case CONFIG_ACCESS_FIFO_DELETE: + set_config_option(&g_accesslist_pipe_delete, p); + break; +#endif +#ifdef WANT_RESTRICT_STATS + case CONFIG_ACCESS_STATS: + if (!scan_ip6_net(p, &tmpnet)) + goto parse_error; + accesslist_bless_net(&tmpnet, OT_PERMISSION_MAY_STAT); + break; +#endif + case CONFIG_ACCESS_STATS_PATH: + set_config_option(&g_stats_path, p); + break; +#ifdef WANT_IP_FROM_PROXY + case CONFIG_ACCESS_PROXY: + if (!scan_ip6_net(p, &tmpnet)) + goto parse_error; + accesslist_bless_net(&tmpnet, OT_PERMISSION_MAY_PROXY); + break; +#endif + case CONFIG_TRACKER_REDIRECT_URL: + set_config_option(&g_redirecturl, p); + break; +#ifdef WANT_SYNC_LIVE + case CONFIG_LIVESYNC_CLUSTER_NODE_IP: + if (!scan_ip6_net(p, &tmpnet)) + goto parse_error; + accesslist_bless_net(&tmpnet, OT_PERMISSION_MAY_LIVESYNC); + break; + case CONFIG_LIVESYNC_CLUSTER_LISTEN: + tmpport = LIVESYNC_PORT; + { + int shared = OT_LISTEN_INHERIT, dummy = OT_LISTEN_INHERIT; + int off = scan_ip6_port(p, tmpip, &tmpport); + + if (!off || !listen_scan_options(p + off, &shared, &dummy)) + goto parse_error; + listen_append(tmpip, tmpport, FLAG_MCA, shared, OT_LISTEN_INHERIT); + } + break; +#endif + default: + fprintf(stderr, "Unhandled line in config file: %s\n", inbuf); + break; + parse_error: + fprintf(stderr, "Parse error in config file: %s\n", inbuf); + } + } + fclose(config_file_handle); + return bound; +} + +static void usage(char *name) { + fprintf(stderr, + "Usage: %s [-i ip] [-p port] [-P port] [-r redirect] [-d dir] [-u user] [-A ip[/bits]] [-f config] [-s livesyncport]" +#ifdef WANT_ACCESSLIST_BLACK + " [-b blacklistfile]" +#elif defined(WANT_ACCESSLIST_WHITE) + " [-w whitelistfile]" +#endif + "\n", + name); +} + +#define HELPLINE(opt, desc) fprintf(stderr, "\t%-10s%s\n", opt, desc) +static void help(char *name) { + usage(name); + + HELPLINE("-f config", "include and execute the config file"); + HELPLINE("-i ip", "specify ip to bind to with next -[pP] (default: any, overrides preceeding ones)"); + HELPLINE("-p port", "do bind to tcp port (default: 6969, you may specify more than one)"); + HELPLINE("-P port", "do bind to udp port (default: 6969, you may specify more than one)"); + HELPLINE("-r redirecturl", "specify url where / should be redirected to (default none)"); + HELPLINE("-d dir", "specify directory to try to chroot to (default: \".\")"); + HELPLINE("-u user", "specify user under whose privileges opentracker should run (default: \"nobody\")"); + HELPLINE("-A ip[/bits]", "bless an ip address or net as admin address (e.g. to allow syncs from this address)"); + HELPLINE("-v", "report version, event mechanism and the feature flags opentracker was built with"); +#ifdef WANT_ACCESSLIST_BLACK + HELPLINE("-b file", "specify blacklist file."); +#elif defined(WANT_ACCESSLIST_WHITE) + HELPLINE("-w file", "specify whitelist file."); +#endif + + fprintf(stderr, "\nExample: ./opentracker -i 127.0.0.1 -p 6969 -P 6969 -f ./opentracker.conf -i 10.1.1.23 -p 2710 -p 80\n"); + fprintf(stderr, " Here -i 127.0.0.1 selects the ip address for the next -p 6969 and -P 6969.\n"); + fprintf(stderr, " If no port is bound from config file or command line, the last address given\n"); + fprintf(stderr, " (or ::1 if none is set) will be used on port 6969.\n"); +} +#undef HELPLINE + + +static void listen_resolve_defaults(void) { + size_t i; + + for (i = 0; i < g_listen_count; ++i) { + ot_listen *entry = g_listen + i; + + if (entry->shared == OT_LISTEN_INHERIT) + entry->shared = g_listen_shared; + if (entry->workers == OT_LISTEN_INHERIT) + entry->workers = entry->proto == FLAG_TCP ? g_listen_workers_tcp : g_listen_workers_udp; + + /* A tcp worker hands a connection it cannot finish to the event loop, + which only one thread may touch where the set is ours rather than the + kernel's */ + if (entry->proto == FLAG_TCP && entry->workers > 1 && !event_can_share()) { + fprintf(stderr, "Warning: Multiple workers impossible on this platform with event mechanism %s chosen.\n", + event_mechanism()); + entry->workers = 1; + } + } +} + +ot_listen *config_listen_entries(size_t *count) { + *count = g_listen_count; + return g_listen; +} + +void config_parse(int argc, char **argv) { + ot_ip6 serverip; + ot_net tmpnet; + int bound = 0, scanon = 1; + uint64_t value; + + memset(serverip, 0, sizeof(ot_ip6)); +#ifdef WANT_V4_ONLY + serverip[10] = serverip[11] = -1; +#endif + + while (scanon) { + switch (getopt(argc, argv, + ":i:p:A:P:d:u:r:s:f:l:v" +#ifdef WANT_ACCESSLIST_BLACK + "b:" +#elif defined(WANT_ACCESSLIST_WHITE) + "w:" +#endif + "h")) { + case -1: + scanon = 0; + break; + case 'i': + if (!ip_parse(optarg, serverip)) { + usage(argv[0]); + exit(1); + } + break; +#ifdef WANT_ACCESSLIST_BLACK + case 'b': + set_config_option(&g_accesslist_filename, optarg); + break; +#elif defined(WANT_ACCESSLIST_WHITE) + case 'w': + set_config_option(&g_accesslist_filename, optarg); + break; +#endif + case 'p': + if (!scan_uint(optarg, &value, 65535)) { + usage(argv[0]); + exit(1); + } + listen_append(serverip, (uint16_t)value, FLAG_TCP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + bound++; + break; + case 'P': + if (!scan_uint(optarg, &value, 65535)) { + usage(argv[0]); + exit(1); + } + listen_append(serverip, (uint16_t)value, FLAG_UDP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + bound++; + break; +#ifdef WANT_SYNC_LIVE + case 's': + if (!scan_uint(optarg, &value, 65535)) { + usage(argv[0]); + exit(1); + } + listen_append(serverip, (uint16_t)value, FLAG_MCA, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + break; +#endif + case 'd': + set_config_option(&g_serverdir, optarg); + break; + case 'u': + set_config_option(&g_serveruser, optarg); + break; + case 'r': + set_config_option(&g_redirecturl, optarg); + break; + case 'l': + g_statefile = optarg; + break; + case 'A': + if (!scan_ip6_net(optarg, &tmpnet)) { + usage(argv[0]); + exit(1); + } + accesslist_bless_net(&tmpnet, 0xffff); /* Allow everything for now */ + break; + case 'f': + bound += parse_configfile(optarg); + break; + case 'h': + help(argv[0]); + exit(0); + case 'v': { + char buffer[8192]; + char *cursor = buffer; + + stats_return_tracker_version(&cursor, buffer + sizeof(buffer) - 1); + stats_return_tracker_features(&cursor, buffer + sizeof(buffer) - 1); + *cursor = 0; + fputs(buffer, stderr); + exit(0); + } + default: + case '?': + usage(argv[0]); + exit(1); + } + } + + if (!bound) { + listen_append(serverip, 6969, FLAG_TCP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + listen_append(serverip, 6969, FLAG_UDP, OT_LISTEN_INHERIT, OT_LISTEN_INHERIT); + } + + listen_resolve_defaults(); +} diff --git a/src/ot_config.h b/src/ot_config.h new file mode 100644 index 0000000..79187ee --- /dev/null +++ b/src/ot_config.h @@ -0,0 +1,33 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_CONFIG_H__ +#define OT_CONFIG_H__ + +#include + +#include "trackerlogic.h" + +/* An address opentracker was asked to listen on. Options the statement did + not name have been filled in from the file wide defaults by the time these + are handed out. sock is filled in by whoever binds them. */ +typedef struct { + ot_ip6 ip; + uint16_t port; + PROTO_FLAG proto; + int shared; + int workers; + int64_t sock; +} ot_listen; + +extern const char *g_redirecturl; +extern const char *g_serverdir; +extern const char *g_serveruser; +extern const char *g_statefile; + +/* Reads the command line and every config file it names. Exits on error. */ +void config_parse(int argc, char **argv); + +ot_listen *config_listen_entries(size_t *count); + +#endif diff --git a/src/ot_event.c b/src/ot_event.c new file mode 100644 index 0000000..6062f70 --- /dev/null +++ b/src/ot_event.c @@ -0,0 +1,357 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_event.h" + +/* One caller waits, so the scratch the mechanism needs is kept here rather + than allocated per call. */ + +/* poll works everywhere and is what the BSDs get until kqueue is written. + Defining OT_EVENT_FORCE_POLL picks it on a platform that would otherwise + use something better, which is how both are exercised by the tests. */ +#if defined(OT_EVENT_FORCE_POLL) +#define OT_EVENT_POLL +#elif defined(OT_EVENT_FORCE_KQUEUE) +#define OT_EVENT_KQUEUE +#elif defined(__linux__) +#define OT_EVENT_EPOLL +#elif defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || defined(__APPLE__) +#define OT_EVENT_KQUEUE +#else +#define OT_EVENT_POLL +#endif + +#ifdef OT_EVENT_EPOLL +/* ------------------------------------------------------------- epoll */ + +#include + +static int g_epoll = -1; +static struct epoll_event *g_ready; +static int g_ready_space; + +const char *event_mechanism(void) { return "epoll"; } +/* The set lives in the kernel, and epoll_ctl during another thread's + epoll_wait is what it is for */ +int event_can_share(void) { return 1; } + +int event_init(void) { + g_ready = NULL; + g_ready_space = 0; + return (g_epoll = epoll_create1(EPOLL_CLOEXEC)) == -1 ? -1 : 0; +} + +void event_deinit(void) { + if (g_epoll != -1) + close(g_epoll); + free(g_ready); + g_epoll = -1; + g_ready = NULL; + g_ready_space = 0; +} + +static int event_ctl(int op, int fd, int what, uint32_t generation) { + struct epoll_event ev; + + memset(&ev, 0, sizeof(ev)); + ev.events = (what & EVENT_IN ? (uint32_t)EPOLLIN : 0) | (what & EVENT_OUT ? (uint32_t)EPOLLOUT : 0); + /* The generation rides along inside the kernel, so an event says which + connection it belongs to without anybody looking it up */ + ev.data.u64 = ((uint64_t)generation << 32) | (uint32_t)fd; + return epoll_ctl(g_epoll, op, fd, &ev); +} + +int event_add(int fd, int what, uint32_t generation) { return event_ctl(EPOLL_CTL_ADD, fd, what, generation); } +int event_modify(int fd, int what, uint32_t generation) { return event_ctl(EPOLL_CTL_MOD, fd, what, generation); } +int event_del(int fd) { return epoll_ctl(g_epoll, EPOLL_CTL_DEL, fd, NULL); } + +/* A descriptor leaves the interest list when the last reference to its open + file description is closed, and the tracker never duplicates one */ +int event_close(int fd) { return close(fd); } + +int event_wait(ot_event *out, int max, int timeout_ms) { + int count, i; + + if (max > g_ready_space) { + struct epoll_event *grown = realloc(g_ready, (size_t)max * sizeof(struct epoll_event)); + + if (!grown) + return -1; + g_ready = grown; + g_ready_space = max; + } + + /* An interrupted wait is reported as no events rather than retried here, + so that the caller gets to look at whatever the signal was about */ + if ((count = epoll_wait(g_epoll, g_ready, max, timeout_ms)) == -1) + return errno == EINTR ? 0 : -1; + for (i = 0; i < count; ++i) { + uint32_t events = g_ready[i].events; + + out[i].fd = (int)(uint32_t)g_ready[i].data.u64; + out[i].generation = (uint32_t)(g_ready[i].data.u64 >> 32); + /* A hangup or an error is reported as both, so whichever side the caller + was waiting on gets to notice */ + out[i].what = (events & (EPOLLIN | EPOLLHUP | EPOLLERR) ? EVENT_IN : 0) | + (events & (EPOLLOUT | EPOLLHUP | EPOLLERR) ? EVENT_OUT : 0); + } + return count; +} + +#endif /* OT_EVENT_EPOLL */ + +#ifdef OT_EVENT_KQUEUE +/* ------------------------------------------------------------ kqueue */ + +#include +#include +#include + +static int g_kqueue = -1; +static struct kevent *g_ready; +static int g_ready_space; + +const char *event_mechanism(void) { return "kqueue"; } +/* The set lives in the kernel, and kevent may be called from any thread */ +int event_can_share(void) { return 1; } + +int event_init(void) { + g_ready = NULL; + g_ready_space = 0; + return (g_kqueue = kqueue()) == -1 ? -1 : 0; +} + +void event_deinit(void) { + if (g_kqueue != -1) + close(g_kqueue); + free(g_ready); + g_kqueue = -1; + g_ready = NULL; + g_ready_space = 0; +} + +/* kqueue keeps one registration per filter, so reading and writing are two + entries. A change of interest is an add of the one wanted and a delete + of the one not. The generation travels in udata, which the kernel passes + back untouched. */ +static int event_change(int fd, int what, uint32_t generation, int deleting) { + struct kevent change[2], result[2]; + void *udata = (void *)(uintptr_t)(((uint64_t)generation << 32) | (uint32_t)fd); + int succeeded = 0, count, i; + + EV_SET(&change[0], (uintptr_t)fd, EVFILT_READ, + EV_RECEIPT | ((deleting || !(what & EVENT_IN)) ? EV_DELETE : EV_ADD), 0, 0, udata); + EV_SET(&change[1], (uintptr_t)fd, EVFILT_WRITE, + EV_RECEIPT | ((deleting || !(what & EVENT_OUT)) ? EV_DELETE : EV_ADD), 0, 0, udata); + + if ((count = kevent(g_kqueue, change, 2, result, 2, NULL)) == -1) + return -1; + + for (i = 0; i < count && i < 2; ++i) { + if (!result[i].data) { + ++succeeded; + continue; + } + /* Dropping interest we never registered is not a failure */ + if ((change[i].flags & EV_DELETE) && result[i].data == ENOENT) + continue; + errno = (int)result[i].data; + return -1; + } + + /* Nothing at all was registered or removed, so the descriptor is unknown */ + if (!succeeded) { + errno = ENOENT; + return -1; + } + return 0; +} + +int event_add(int fd, int what, uint32_t generation) { return what ? event_change(fd, what, generation, 0) : 0; } +int event_modify(int fd, int what, uint32_t generation) { return event_change(fd, what, generation, 0); } +int event_del(int fd) { return event_change(fd, 0, 0, 1); } + +/* Closing a descriptor removes every kevent that references it */ +int event_close(int fd) { return close(fd); } + +int event_wait(ot_event *out, int max, int timeout_ms) { + struct timespec timeout; + struct timespec *deadline = NULL; + int count, i; + + if (max > g_ready_space) { + struct kevent *grown = realloc(g_ready, (size_t)max * sizeof(struct kevent)); + + if (!grown) + return -1; + g_ready = grown; + g_ready_space = max; + } + + if (timeout_ms >= 0) { + timeout.tv_sec = timeout_ms / 1000; + timeout.tv_nsec = (long)(timeout_ms % 1000) * 1000000; + deadline = &timeout; + } + + /* An interrupted wait is reported as no events rather than retried here, + so that the caller gets to look at whatever the signal was about */ + if ((count = kevent(g_kqueue, NULL, 0, g_ready, max, deadline)) == -1) + return errno == EINTR ? 0 : -1; + for (i = 0; i < count; ++i) { + uint64_t udata = (uint64_t)(uintptr_t)g_ready[i].udata; + int eof = (g_ready[i].flags & (EV_EOF | EV_ERROR)) != 0; + + out[i].fd = (int)(uint32_t)udata; + out[i].generation = (uint32_t)(udata >> 32); + out[i].what = ((g_ready[i].filter == EVFILT_READ || eof) ? EVENT_IN : 0) | + ((g_ready[i].filter == EVFILT_WRITE || eof) ? EVENT_OUT : 0); + } + return count; +} + +#endif /* OT_EVENT_KQUEUE */ + +#ifdef OT_EVENT_POLL +/* -------------------------------------------------------------- poll */ + +#include + +/* poll wants its descriptors packed, so the registered ones live in one + array and a second maps a descriptor number to its place in it. */ +static struct pollfd *g_pfds; +static uint32_t *g_generation; +static int *g_where; +static int g_count, g_space, g_where_space; + +const char *event_mechanism(void) { return "poll"; } +/* The arrays below are ours and nothing guards them */ +int event_can_share(void) { return 0; } + +int event_init(void) { + g_pfds = NULL, g_generation = NULL, g_where = NULL; + g_count = g_space = g_where_space = 0; + return 0; +} + +void event_deinit(void) { + free(g_pfds), free(g_generation), free(g_where); + g_pfds = NULL, g_generation = NULL, g_where = NULL; + g_count = g_space = g_where_space = 0; +} + +static short event_to_poll(int what) { + return (short)((what & EVENT_IN ? POLLIN : 0) | (what & EVENT_OUT ? POLLOUT : 0)); +} + +static int event_room_for(int fd) { + if (fd >= g_where_space) { + int space = g_where_space ? g_where_space : 64; + int *grown; + + while (space <= fd) + space *= 2; + if (!(grown = realloc(g_where, (size_t)space * sizeof(int)))) + return -1; + while (g_where_space < space) + grown[g_where_space++] = -1; + g_where = grown; + } + + if (g_count == g_space) { + int space = g_space ? g_space * 2 : 64; + struct pollfd *pfds = realloc(g_pfds, (size_t)space * sizeof(struct pollfd)); + uint32_t *gens; + + if (!pfds) + return -1; + g_pfds = pfds; + if (!(gens = realloc(g_generation, (size_t)space * sizeof(uint32_t)))) + return -1; + g_generation = gens; + g_space = space; + } + return 0; +} + +int event_add(int fd, int what, uint32_t generation) { + if (fd < 0 || event_room_for(fd) == -1) + return -1; + if (g_where[fd] != -1) { + errno = EEXIST; + return -1; + } + + g_pfds[g_count].fd = fd; + g_pfds[g_count].events = event_to_poll(what); + g_pfds[g_count].revents = 0; + g_generation[g_count] = generation; + g_where[fd] = g_count++; + return 0; +} + +int event_modify(int fd, int what, uint32_t generation) { + if (fd < 0 || fd >= g_where_space || g_where[fd] == -1) { + errno = ENOENT; + return -1; + } + g_pfds[g_where[fd]].events = event_to_poll(what); + g_generation[g_where[fd]] = generation; + return 0; +} + +int event_del(int fd) { + int at; + + if (fd < 0 || fd >= g_where_space || (at = g_where[fd]) == -1) { + errno = ENOENT; + return -1; + } + + /* The last one moves into the gap, so the array stays packed */ + g_pfds[at] = g_pfds[--g_count]; + g_generation[at] = g_generation[g_count]; + g_where[g_pfds[at].fd] = at; + g_where[fd] = -1; + return 0; +} + +/* Nothing here lives in the kernel, so the entry has to come out by hand + before the descriptor goes, or poll reports POLLNVAL for it forever */ +int event_close(int fd) { + event_del(fd); + return close(fd); +} + +int event_wait(ot_event *out, int max, int timeout_ms) { + int ready, found = 0, i; + + /* An interrupted wait is reported as no events rather than retried here, + so that the caller gets to look at whatever the signal was about */ + if ((ready = poll(g_pfds, (nfds_t)g_count, timeout_ms)) == -1) + return errno == EINTR ? 0 : -1; + if (ready <= 0) + return ready; + + for (i = 0; i < g_count && found < max; ++i) { + short revents = g_pfds[i].revents; + + if (!revents) + continue; + out[found].fd = g_pfds[i].fd; + out[found].generation = g_generation[i]; + out[found].what = (revents & (POLLIN | POLLHUP | POLLERR | POLLNVAL) ? EVENT_IN : 0) | + (revents & (POLLOUT | POLLHUP | POLLERR | POLLNVAL) ? EVENT_OUT : 0); + ++found; + } + return found; +} + +#endif /* OT_EVENT_POLL */ diff --git a/src/ot_event.h b/src/ot_event.h new file mode 100644 index 0000000..cb84ca8 --- /dev/null +++ b/src/ot_event.h @@ -0,0 +1,47 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_EVENT_H__ +#define OT_EVENT_H__ + +#include + +/* Which descriptors have become readable or writable, asked of whichever + mechanism the kernel offers. The generation a descriptor was registered + with is handed back with every event, so a caller can tell an event for + the connection it is holding from one for a connection that has since been + closed and had its number reused. */ + +enum { EVENT_IN = 1, EVENT_OUT = 2 }; + +typedef struct { + int fd; + uint32_t generation; + int what; +} ot_event; + +int event_init(void); +void event_deinit(void); + +int event_add(int fd, int what, uint32_t generation); +int event_modify(int fd, int what, uint32_t generation); +int event_del(int fd); + +/* Stop watching a descriptor and close it. Where the kernel drops a closed + descriptor from its set by itself, this is one system call where doing the + two separately would be two. */ +int event_close(int fd); + +/* Waits at most timeout_ms, or indefinitely when that is negative, and fills + out with up to max ready descriptors. Returns how many, or -1. */ +int event_wait(ot_event *out, int max, int timeout_ms); + +/* Whether the registered set may be changed from another thread while this + one is waiting on it. True where the kernel keeps the set, false where + this module keeps it in memory of its own. */ +int event_can_share(void); + +/* Which mechanism was compiled in, for the version string */ +const char *event_mechanism(void); + +#endif diff --git a/src/ot_fdstate.c b/src/ot_fdstate.c new file mode 100644 index 0000000..8e4bd89 --- /dev/null +++ b/src/ot_fdstate.c @@ -0,0 +1,202 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_fdstate.h" + +enum { FD_FREE, FD_IDLE, FD_BUSY, FD_CLOSING }; + +/* Generation and state share one word so that taking the state is also a + check that the generation has not moved */ +#define FD_GEN(w) ((uint32_t)((w) >> 32)) +#define FD_ST(w) ((uint32_t)(((w) >> 24) & 0xff)) +#define FD_WORD(g, s) (((uint64_t)(g) << 32) | ((uint64_t)(s) << 24)) +#define FD_SET_ST(w, s) (((w) & ~(uint64_t)0xff000000) | ((uint64_t)(s) << 24)) + +typedef struct { + _Atomic uint64_t state; + _Atomic(void *) cookie; + _Atomic ot_time expiry; +} ot_fdslot; + +/* A descriptor the kernel can hand out is below the limit it was handed out + under, but the limit can be raised after we sized the table, so an out of + range descriptor is rejected */ +#define FD_SLOTS_MAX (1u << 20) + +static ot_fdslot *g_slots; +static size_t g_slot_count; +/* Descriptors are handed out lowest first, so a sweep only has to walk as + far as the highest one ever seen rather than the whole table. */ +static _Atomic size_t g_slot_highest; + +int fdstate_init(void) { + struct rlimit limit; + size_t count = FD_SLOTS_MAX; + + if (getrlimit(RLIMIT_NOFILE, &limit) == 0 && limit.rlim_cur != RLIM_INFINITY && limit.rlim_cur < FD_SLOTS_MAX) + count = (size_t)limit.rlim_cur; + + g_slots = mmap(NULL, count * sizeof(ot_fdslot), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (g_slots == MAP_FAILED) { + g_slots = NULL; + return -1; + } + + g_slot_count = count; + atomic_store(&g_slot_highest, 0); + return 0; +} + +void fdstate_deinit(void) { + if (g_slots) + munmap(g_slots, g_slot_count * sizeof(ot_fdslot)); + g_slots = NULL; + g_slot_count = 0; + atomic_store(&g_slot_highest, 0); +} + +size_t fdstate_slots(void) { return g_slot_count; } + +static ot_fdslot *fdstate_slot(int fd) { + if (fd < 0 || (size_t)fd >= g_slot_count) + return NULL; + return g_slots + fd; +} + +uint32_t fdstate_open(int fd, void *cookie, ot_time expiry) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + + if (!slot) + return 0; + + { + size_t highest = atomic_load_explicit(&g_slot_highest, memory_order_relaxed); + + while ((size_t)fd > highest && + !atomic_compare_exchange_weak_explicit(&g_slot_highest, &highest, (size_t)fd, memory_order_relaxed, + memory_order_relaxed)) + ; + } + + word = atomic_load_explicit(&slot->state, memory_order_relaxed); + atomic_store_explicit(&slot->cookie, cookie, memory_order_release); + atomic_store_explicit(&slot->expiry, expiry, memory_order_relaxed); + /* Publishes the two above to whoever claims the slot next */ + atomic_store_explicit(&slot->state, FD_WORD(FD_GEN(word), FD_IDLE), memory_order_release); + + return FD_GEN(word); +} + +uint32_t fdstate_generation(int fd) { + ot_fdslot *slot = fdstate_slot(fd); + + return slot ? FD_GEN(atomic_load_explicit(&slot->state, memory_order_relaxed)) : 0; +} + +int fdstate_claim(int fd, uint32_t generation) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + + if (!slot) + return 0; + + word = atomic_load_explicit(&slot->state, memory_order_acquire); + for (;;) { + if (FD_GEN(word) != generation || FD_ST(word) != FD_IDLE) + return 0; + if (atomic_compare_exchange_weak_explicit(&slot->state, &word, FD_SET_ST(word, FD_BUSY), memory_order_acquire, + memory_order_relaxed)) + return 1; + /* word now holds what was really there, so the test above runs again */ + } +} + +void fdstate_release(int fd) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + + if (!slot) + return; + + word = atomic_load_explicit(&slot->state, memory_order_relaxed); + atomic_store_explicit(&slot->state, FD_SET_ST(word, FD_IDLE), memory_order_release); +} + +int fdstate_claim_expired(int fd, ot_time now) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + ot_time expiry; + + if (!slot) + return 0; + + /* Reading before asking for the slot keeps the common case, a descriptor + that has not expired, off the bus entirely */ + word = atomic_load_explicit(&slot->state, memory_order_relaxed); + if (FD_ST(word) != FD_IDLE) + return 0; + expiry = atomic_load_explicit(&slot->expiry, memory_order_relaxed); + if (!expiry || expiry > now) + return 0; + + /* Nothing can move the expiry while the slot sits at FD_IDLE, so winning + the slot is the last word */ + return atomic_compare_exchange_strong_explicit(&slot->state, &word, FD_SET_ST(word, FD_CLOSING), memory_order_acquire, + memory_order_relaxed); +} + +void fdstate_close(int fd) { + ot_fdslot *slot = fdstate_slot(fd); + uint64_t word; + + if (!slot) + return; + + word = atomic_load_explicit(&slot->state, memory_order_relaxed); + atomic_store_explicit(&slot->cookie, NULL, memory_order_relaxed); + atomic_store_explicit(&slot->expiry, 0, memory_order_relaxed); + atomic_store_explicit(&slot->state, FD_WORD(FD_GEN(word) + 1, FD_FREE), memory_order_release); +} + +int fdstate_next_expired(ot_time now, int *cursor) { + size_t highest = atomic_load_explicit(&g_slot_highest, memory_order_relaxed); + + for (; (size_t)*cursor <= highest; ++*cursor) + if (fdstate_claim_expired(*cursor, now)) + return (*cursor)++; + + return -1; +} + +void *fdstate_cookie(int fd) { + ot_fdslot *slot = fdstate_slot(fd); + + /* Pairs with the release in fdstate_set_cookie and fdstate_open */ + return slot ? atomic_load_explicit(&slot->cookie, memory_order_acquire) : NULL; +} + +void fdstate_set_cookie(int fd, void *cookie) { + ot_fdslot *slot = fdstate_slot(fd); + + /* Publishes what the cookie points at to whoever reads it next, which may + be another thread entirely */ + if (slot) + atomic_store_explicit(&slot->cookie, cookie, memory_order_release); +} + +void fdstate_set_expiry(int fd, ot_time expiry) { + ot_fdslot *slot = fdstate_slot(fd); + + if (slot) + atomic_store_explicit(&slot->expiry, expiry, memory_order_relaxed); +} diff --git a/src/ot_fdstate.h b/src/ot_fdstate.h new file mode 100644 index 0000000..60fe070 --- /dev/null +++ b/src/ot_fdstate.h @@ -0,0 +1,62 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_FDSTATE_H__ +#define OT_FDSTATE_H__ + +#include + +#include "trackerlogic.h" + +/* What the tracker knows about a file descriptor: who owns it, when it + should be given up on, and whether anyone is working on it right now. + + A descriptor number says nothing about identity, since the kernel hands + the same number out again as soon as it is closed. Every slot therefore + carries a generation that is bumped when the descriptor is given up, and + every caller that was told about a descriptor has to say which generation + it was told about. A caller holding a stale generation is turned away + instead of being handed somebody else's connection. + + The table is sized once from RLIMIT_NOFILE and never moves, so looking a + descriptor up needs no synchronisation at all. */ + +int fdstate_init(void); +void fdstate_deinit(void); +size_t fdstate_slots(void); + +/* Take a freshly accepted descriptor into the table and return the + generation to quote when asking for it back. expiry of 0 never expires. */ +uint32_t fdstate_open(int fd, void *cookie, ot_time expiry); + +/* Which generation a descriptor currently carries, to check an event that + was handed out earlier against the connection living there now. */ +uint32_t fdstate_generation(int fd); + +/* Ask for exclusive use. Returns 0 if the descriptor has been recycled since + generation was handed out, or if somebody else is using it. */ +int fdstate_claim(int fd, uint32_t generation); +void fdstate_release(int fd); + +/* The cleaner's side of the same thing: takes the descriptor only if nobody + is working on it and it really has expired. Never waits. */ +int fdstate_claim_expired(int fd, ot_time now); + +/* Give the descriptor up. The generation is bumped here, which has to happen + before close(2) so that the kernel cannot reissue the number while events + quoting the old generation are still in flight. */ +void fdstate_close(int fd); + +/* Walks the table for descriptors that have expired, handing back one at a + time until it returns -1. The cursor belongs to the caller, so several + sweepers can run without sharing anything. */ +int fdstate_next_expired(ot_time now, int *cursor); + +/* Both only valid while the caller holds the descriptor. */ +void *fdstate_cookie(int fd); + +/* Hand the slot a different cookie, for a connection whose state has moved */ +void fdstate_set_cookie(int fd, void *cookie); +void fdstate_set_expiry(int fd, ot_time expiry); + +#endif diff --git a/src/ot_fmt.c b/src/ot_fmt.c new file mode 100644 index 0000000..362c9b5 --- /dev/null +++ b/src/ot_fmt.c @@ -0,0 +1,40 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include + +/* Opentracker */ +#include "ot_fmt.h" + +void fmt_append(char **cursor, const char *end, const char *format, ...) { + va_list arguments; + size_t room; + int written; + + if (*cursor >= end) { + *cursor = (char *)end; + return; + } + + room = (size_t)(end - *cursor); + va_start(arguments, format); + written = vsnprintf(*cursor, room, format, arguments); + va_end(arguments); + + if (written < 0) + return; + *cursor = (size_t)written >= room ? (char *)end : *cursor + written; +} + +void fmt_bytes(char **cursor, const char *end, const void *bytes, size_t length) { + if (*cursor >= end || (size_t)(end - *cursor) < length) { + *cursor = (char *)end; + return; + } + + memcpy(*cursor, bytes, length); + *cursor += length; +} diff --git a/src/ot_fmt.h b/src/ot_fmt.h new file mode 100644 index 0000000..214cbc4 --- /dev/null +++ b/src/ot_fmt.h @@ -0,0 +1,19 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_FMT_H__ +#define OT_FMT_H__ + +#include + +/* Appending to a cursor that walks a fixed buffer. Both take the cursor by + reference and advance it, so a reply is written as a run of statements and + no caller does pointer arithmetic of its own. */ + +void fmt_append(char **cursor, const char *end, const char *format, ...) __attribute__((format(printf, 3, 4))); + +/* Append raw bytes for the binary stretches of a bencoded reply. + Either everything is appended or nothing at all */ +void fmt_bytes(char **cursor, const char *end, const void *bytes, size_t length); + +#endif diff --git a/src/ot_fullscrape.c b/src/ot_fullscrape.c new file mode 100644 index 0000000..9afad92 --- /dev/null +++ b/src/ot_fullscrape.c @@ -0,0 +1,455 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifdef WANT_FULLSCRAPE + +/* System */ +#include +#include +#include +#include +#ifdef WANT_COMPRESSION_GZIP +#include +#endif +#ifdef WANT_COMPRESSION_ZSTD +#include +#endif + +/* Opentracker */ +#include "ot_fmt.h" +#include "ot_hex.h" +#include "ot_fullscrape.h" +#include "ot_iovec.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_urlencoded.h" +#include "trackerlogic.h" + +/* Fetch full scrape info for all torrents + Full scrapes usually are huge and one does not want to + allocate more memory. So lets get them in 512k units +*/ +#define OT_SCRAPE_CHUNK_SIZE (1024 * 1024) + +/* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */ +#define OT_SCRAPE_MAXENTRYLEN 256 + +/* Forward declaration */ +static void fullscrape_make(int taskid, ot_tasktype mode); +#ifdef WANT_COMPRESSION_GZIP +static void fullscrape_make_gzip(int taskid, ot_tasktype mode); +#endif +#ifdef WANT_COMPRESSION_ZSTD +static void fullscrape_make_zstd(int taskid, ot_tasktype mode); +#endif + +/* Converter function from memory to human readable hex strings + XXX - Duplicated from ot_stats. Needs fix. */ +/* This is the entry point into this worker thread + It grabs tasks from mutex_tasklist and delivers results back +*/ +static void *fullscrape_worker(void *args) { + (void)args; + + while (g_opentracker_running) { + ot_tasktype tasktype = TASK_FULLSCRAPE; + ot_taskid taskid = workqueue_poptask(&tasktype); + if (!taskid) + break; +#ifdef WANT_COMPRESSION_ZSTD + if (tasktype & TASK_FLAG_ZSTD) + fullscrape_make_zstd(taskid, tasktype); + else +#endif +#ifdef WANT_COMPRESSION_GZIP + if (tasktype & TASK_FLAG_GZIP) + fullscrape_make_gzip(taskid, tasktype); + else +#endif + fullscrape_make(taskid, tasktype); + workqueue_pushchunked(taskid, NULL); + } + return NULL; +} + +static pthread_t thread_id; +void fullscrape_init(void) { + pthread_create( &thread_id, NULL, fullscrape_worker, NULL ); +} + +void fullscrape_deinit(void) { + workqueue_wakeall(); + pthread_join( thread_id, NULL ); +} + +void fullscrape_deliver( int64_t sock, ot_tasktype tasktype ) { + workqueue_pushtask( sock, TASK_FULLSCRAPE | tasktype ); +} + +static void fullscrape_write_one( ot_tasktype mode, char **cursor, ot_torrent *torrent, ot_hash *hash ) { + /* Callers guarantee this much room: the chunk writer keeps its low + watermark that far from the end, the compressors hand over a buffer of + exactly that size */ + char *r = *cursor; + const char *r_end = r + OT_SCRAPE_MAXENTRYLEN; + size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; + + switch (mode & TASK_TASK_MASK) { + case TASK_FULLSCRAPE: + default: + /* push hash as bencoded string */ + *r++ = '2'; + *r++ = '0'; + *r++ = ':'; + memcpy(r, hash, sizeof(ot_hash)); + r += sizeof(ot_hash); + /* push rest of the scrape string */ + fmt_append(&r, r_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", seed_count, down_count, peer_count - seed_count); + + break; + case TASK_FULLSCRAPE_TPB_ASCII: + hex_from_hash(r, *hash); + r += 2 * sizeof(ot_hash); + fmt_append(&r, r_end, ":%zd:%zd\n", seed_count, peer_count - seed_count); + break; + case TASK_FULLSCRAPE_TPB_ASCII_PLUS: + hex_from_hash(r, *hash); + r += 2 * sizeof(ot_hash); + fmt_append(&r, r_end, ":%zd:%zd:%zd\n", seed_count, peer_count - seed_count, down_count); + break; + case TASK_FULLSCRAPE_TPB_BINARY: + memcpy(r, *hash, sizeof(ot_hash)); + r += sizeof(ot_hash); + *(uint32_t *)(r + 0) = htonl((uint32_t)seed_count); + *(uint32_t *)(r + 4) = htonl((uint32_t)(peer_count - seed_count)); + r += 8; + break; + case TASK_FULLSCRAPE_TPB_URLENCODED: + r += urlencoded_format(r, *hash, sizeof(ot_hash)); + fmt_append(&r, r_end, ":%zd:%zd\n", seed_count, peer_count - seed_count); + break; + case TASK_FULLSCRAPE_TRACKERSTATE: + hex_from_hash(r, *hash); + r += 2 * sizeof(ot_hash); + fmt_append(&r, r_end, ":%zd:%zd\n", torrent->peer_list6->base, down_count); + break; + } + *cursor = r; +} + +static void fullscrape_make(int taskid, ot_tasktype mode) { + size_t bucket; + char *r, *re; + struct iovec iovector = {NULL, 0}; + + /* Setup return vector... */ + r = iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!r) + return; + + /* re points to low watermark */ + re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) + fmt_append(&r, (char *)iovector.iov_base + OT_SCRAPE_CHUNK_SIZE, "d5:filesd"); + + /* For each bucket... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + /* Get exclusive access to that bucket */ + ot_vector *torrents_list = mutex_bucket_lock(bucket); + ot_torrent *torrents = (ot_torrent *)(torrents_list->data); + size_t i; + + /* For each torrent in this bucket.. */ + for (i = 0; i < torrents_list->size; ++i) { + fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); + + if (r > re) { + iovector.iov_len = r - (char *)iovector.iov_base; + + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + mutex_bucket_unlock(bucket, 0); + return; + } + /* Allocate a fresh output buffer */ + r = iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!r) { + mutex_bucket_unlock(bucket, 0); + return; + } + + /* re points to low watermark */ + re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; + } + } + + /* All torrents done: release lock on current bucket */ + mutex_bucket_unlock(bucket, 0); + + /* Parent thread died? */ + if (!g_opentracker_running) + return; + } + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) + fmt_append(&r, (char *)iovector.iov_base + OT_SCRAPE_CHUNK_SIZE, "ee"); + + /* Send rest of data */ + iovector.iov_len = r - (char *)iovector.iov_base; + if (workqueue_pushchunked(taskid, &iovector)) + free(iovector.iov_base); +} + +#ifdef WANT_COMPRESSION_GZIP + +static void fullscrape_make_gzip(int taskid, ot_tasktype mode) { + size_t bucket; + char *r; + struct iovec iovector = {NULL, 0}; + int zres; + z_stream strm; + /* Setup return vector... */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!iovector.iov_base) + return; + + memset(&strm, 0, sizeof(strm)); + strm.next_out = (uint8_t *)iovector.iov_base; + strm.avail_out = OT_SCRAPE_CHUNK_SIZE; + if (deflateInit2(&strm, 7, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY) != Z_OK) + fprintf(stderr, "not ok.\n"); + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { + strm.next_in = (uint8_t *)"d5:filesd"; + strm.avail_in = strlen("d5:filesd"); + zres = deflate(&strm, Z_NO_FLUSH); + } + + /* For each bucket... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + /* Get exclusive access to that bucket */ + ot_vector *torrents_list = mutex_bucket_lock(bucket); + ot_torrent *torrents = (ot_torrent *)(torrents_list->data); + size_t i; + + /* For each torrent in this bucket.. */ + for (i = 0; i < torrents_list->size; ++i) { + char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; + r = compress_buffer; + fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); + strm.next_in = (uint8_t *)compress_buffer; + strm.avail_in = r - compress_buffer; + zres = deflate(&strm, Z_NO_FLUSH); + if ((zres < Z_OK) && (zres != Z_BUF_ERROR)) + fprintf(stderr, "deflate() failed while in fullscrape_make().\n"); + + /* Check if there still is enough buffer left */ + while (!strm.avail_out) { + iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; + + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + mutex_bucket_unlock(bucket, 0); + return; + } + /* Allocate a fresh output buffer */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!iovector.iov_base) { + fprintf(stderr, "Out of memory trying to claim ouput buffer\n"); + deflateEnd(&strm); + mutex_bucket_unlock(bucket, 0); + return; + } + strm.next_out = (uint8_t *)iovector.iov_base; + strm.avail_out = OT_SCRAPE_CHUNK_SIZE; + zres = deflate(&strm, Z_NO_FLUSH); + if ((zres < Z_OK) && (zres != Z_BUF_ERROR)) + fprintf(stderr, "deflate() failed while in fullscrape_make().\n"); + } + } + + /* All torrents done: release lock on current bucket */ + mutex_bucket_unlock(bucket, 0); + + /* Parent thread died? */ + if (!g_opentracker_running) { + deflateEnd(&strm); + return; + } + } + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { + strm.next_in = (uint8_t *)"ee"; + strm.avail_in = strlen("ee"); + } + + if (deflate(&strm, Z_FINISH) < Z_OK) + fprintf(stderr, "deflate() failed while in fullscrape_make()'s endgame.\n"); + + iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + deflateEnd(&strm); + return; + } + + /* Check if there's a last batch of data in the zlib buffer */ + if (!strm.avail_out) { + /* Allocate a fresh output buffer */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + + if (!iovector.iov_base) { + fprintf(stderr, "Problem with iovec_next_chunk\n"); + deflateEnd(&strm); + return; + } + strm.next_out = iovector.iov_base; + strm.avail_out = OT_SCRAPE_CHUNK_SIZE; + if (deflate(&strm, Z_FINISH) < Z_OK) + fprintf(stderr, "deflate() failed while in fullscrape_make()'s endgame.\n"); + + /* Only pass the new buffer if there actually was some data left in the buffer */ + iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base; + if (!iovector.iov_len || workqueue_pushchunked(taskid, &iovector)) + free(iovector.iov_base); + } + + deflateEnd(&strm); +} +/* WANT_COMPRESSION_GZIP */ +#endif + +#ifdef WANT_COMPRESSION_ZSTD + +static void fullscrape_make_zstd(int taskid, ot_tasktype mode) { + int bucket; + char *r; + struct iovec iovector = {NULL, 0}; + ZSTD_CCtx *zstream = ZSTD_createCCtx(); + ZSTD_inBuffer inbuf; + ZSTD_outBuffer outbuf; + size_t more_bytes; + + if (!zstream) + return; + + /* Setup return vector... */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!iovector.iov_base) { + ZSTD_freeCCtx(zstream); + return; + } + + /* Working with a compression level 6 is half as fast as level 3, but + seems to be the last reasonable bump that's worth extra cpu */ + ZSTD_CCtx_setParameter(zstream, ZSTD_c_compressionLevel, 6); + + outbuf.dst = iovector.iov_base; + outbuf.size = OT_SCRAPE_CHUNK_SIZE; + outbuf.pos = 0; + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { + inbuf.src = (const void *)"d5:filesd"; + inbuf.size = strlen("d5:filesd"); + inbuf.pos = 0; + ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue); + } + + /* For each bucket... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + /* Get exclusive access to that bucket */ + ot_vector *torrents_list = mutex_bucket_lock(bucket); + ot_torrent *torrents = (ot_torrent *)(torrents_list->data); + size_t i; + + /* For each torrent in this bucket.. */ + for (i = 0; i < torrents_list->size; ++i) { + char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; + r = compress_buffer; + fullscrape_write_one(mode, &r, torrents + i, &torrents[i].hash); + inbuf.src = compress_buffer; + inbuf.size = r - compress_buffer; + inbuf.pos = 0; + ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue); + + /* Check if there still is enough buffer left */ + while (outbuf.pos + OT_SCRAPE_MAXENTRYLEN > outbuf.size) { + iovector.iov_len = outbuf.size; + + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + ZSTD_freeCCtx(zstream); + return mutex_bucket_unlock(bucket, 0); + } + /* Allocate a fresh output buffer */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + if (!iovector.iov_base) { + fprintf(stderr, "Out of memory trying to claim ouput buffer\n"); + ZSTD_freeCCtx(zstream); + return mutex_bucket_unlock(bucket, 0); + } + + outbuf.dst = iovector.iov_base; + outbuf.size = OT_SCRAPE_CHUNK_SIZE; + outbuf.pos = 0; + + ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue); + } + } + + /* All torrents done: release lock on current bucket */ + mutex_bucket_unlock(bucket, 0); + + /* Parent thread died? */ + if (!g_opentracker_running) + return; + } + + if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) { + inbuf.src = (const void *)"ee"; + inbuf.size = strlen("ee"); + inbuf.pos = 0; + } + + more_bytes = ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end); + + iovector.iov_len = outbuf.pos; + if (workqueue_pushchunked(taskid, &iovector)) { + free(iovector.iov_base); + ZSTD_freeCCtx(zstream); + return; + } + + /* Check if there's a last batch of data in the zlib buffer */ + if (more_bytes) { + /* Allocate a fresh output buffer */ + iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE); + + if (!iovector.iov_base) { + fprintf(stderr, "Problem with iovec_next_chunk\n"); + ZSTD_freeCCtx(zstream); + return; + } + + outbuf.dst = iovector.iov_base; + outbuf.size = OT_SCRAPE_CHUNK_SIZE; + outbuf.pos = 0; + + ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end); + + /* Only pass the new buffer if there actually was some data left in the buffer */ + iovector.iov_len = outbuf.pos; + if (!iovector.iov_len || workqueue_pushchunked(taskid, &iovector)) + free(iovector.iov_base); + } + + ZSTD_freeCCtx(zstream); +} +/* WANT_COMPRESSION_ZSTD */ +#endif + +/* WANT_FULLSCRAPE */ +#endif diff --git a/src/ot_fullscrape.h b/src/ot_fullscrape.h new file mode 100644 index 0000000..8d11cd3 --- /dev/null +++ b/src/ot_fullscrape.h @@ -0,0 +1,23 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_FULLSCRAPE_H__ +#define OT_FULLSCRAPE_H__ + +#ifdef WANT_FULLSCRAPE + +#include "ot_mutex.h" +#include "ot_workqueue.h" + +void fullscrape_init(void); +void fullscrape_deinit(void); +void fullscrape_deliver(int64_t sock, ot_tasktype tasktype); + +#else + +#define fullscrape_init() +#define fullscrape_deinit() + +#endif + +#endif diff --git a/src/ot_hex.c b/src/ot_hex.c new file mode 100644 index 0000000..b537094 --- /dev/null +++ b/src/ot_hex.c @@ -0,0 +1,56 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_hex.h" + +int hex_value(int c) { + unsigned char u = (unsigned char)c; + + if (u >= '0' && u <= '9') + return u - '0'; + u |= 0x20; /* fold upper case onto lower, digits are already handled */ + if (u >= 'a' && u <= 'f') + return u - 'a' + 10; + return -1; +} + +int hex_isdigit(int c) { return hex_value(c) >= 0; } + +char *hex_from_hash(char *dest, const uint8_t *hash) { + static const char digits[] = "0123456789ABCDEF"; + char *d = dest; + size_t i; + + for (i = 0; i < sizeof(ot_hash); ++i) { + *d++ = digits[hash[i] >> 4]; + *d++ = digits[hash[i] & 15]; + } + *d = 0; + return dest; +} + +int hex_to_hash(ot_hash hash, const char *src) { + uint8_t decoded[sizeof(ot_hash)]; + size_t i; + + for (i = 0; i < sizeof(ot_hash); ++i) { + int high, low; + + /* Tested one character at a time so that a string ending early stops us + at its terminator rather than one byte past it */ + if ((high = hex_value((unsigned char)src[2 * i])) < 0) + return 0; + if ((low = hex_value((unsigned char)src[2 * i + 1])) < 0) + return 0; + + decoded[i] = (uint8_t)((high << 4) | low); + } + + memcpy(hash, decoded, sizeof(decoded)); + return 1; +} diff --git a/src/ot_hex.h b/src/ot_hex.h new file mode 100644 index 0000000..ddbbb44 --- /dev/null +++ b/src/ot_hex.h @@ -0,0 +1,27 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_HEX_H__ +#define OT_HEX_H__ + +#include + +#include "trackerlogic.h" + +/* Write hash as forty upper case hex digits followed by a terminating zero. + dest must have room for forty one bytes. Returns dest. */ +char *hex_from_hash(char *dest, const uint8_t *hash); + +/* Read forty hex digits from src into hash. Returns non-zero on success. + On failure hash is left untouched and no byte of src beyond the offending + character is read, so src need not be forty characters long. */ +int hex_to_hash(ot_hash hash, const char *src); + +/* Non-zero if c is a hexadecimal digit, in either case. */ +int hex_isdigit(int c); + +/* The value of a hexadecimal digit, or -1 if c is not one. Shared with the + address scanner so that both agree on what a digit is. */ +int hex_value(int c); + +#endif diff --git a/src/ot_http.c b/src/ot_http.c new file mode 100644 index 0000000..96b8f4e --- /dev/null +++ b/src/ot_http.c @@ -0,0 +1,810 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_fullscrape.h" +#include "ot_http.h" +#include "ot_iovec.h" +#include "ot_ip.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_scan.h" +#include "ot_event.h" +#include "ot_fdstate.h" +#include "ot_vector.h" +#include "ot_fmt.h" +#include "ot_stats.h" +#include "ot_urlencoded.h" +#include "trackerlogic.h" + +#define OT_MAXMULTISCRAPE_COUNT 64 +extern const char *g_redirecturl; + +const char *g_stats_path; +ssize_t g_stats_path_len; + +enum { SUCCESS_HTTP_HEADER_LENGTH = 80, SUCCESS_HTTP_SIZE_OFF = 17 }; + +struct http_data *http_adopt(const int64_t sock) { + struct http_data *cookie = fdstate_cookie(sock), *heap; + int interest; + + if (!cookie || !(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK)) + return cookie; + + if (!(heap = malloc(sizeof(*heap)))) + return NULL; + + *heap = *cookie; + heap->flag &= ~STRUCT_HTTP_FLAG_ONSTACK; + + /* Publishing the cookie is what lets another thread reach this connection, + so nothing may touch it afterwards, not even to read what it asked for */ + interest = heap->interest; + fdstate_set_cookie((int)sock, heap); + + if (event_add((int)sock, interest, fdstate_generation((int)sock)) == -1) { + fdstate_set_cookie((int)sock, cookie); + free(heap); + return NULL; + } + return heap; +} + +/* Give up a connection: the one place that knows everything hanging off it. */ +void http_close(const int64_t sock) { + struct http_data *cookie = fdstate_cookie(sock); + + fdstate_close((int)sock); + + if (cookie) { + if (cookie->batch) + batch_reset(cookie->batch); + free(cookie->batch); + buf_reset(&cookie->request); + if (cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) + workqueue_canceltask(sock); + if (!(cookie->flag & STRUCT_HTTP_FLAG_ONSTACK)) + free(cookie); + } + + event_close((int)sock); +} + +static void http_senddata(const int64_t sock, struct ot_workstruct *ws) { + struct http_data *cookie = fdstate_cookie(sock); + ssize_t written_size; + + if (!cookie) { + http_close(sock); + return; + } + + /* whoever sends data is not interested in its input buffer */ + if (ws->keep_alive && ws->header_size != ws->request_size) { + size_t rest = ws->request_size - ws->header_size; + if (cookie->request.data) { + memmove(cookie->request.data, ws->request + ws->header_size, rest); + buf_truncate(&cookie->request, rest); + } else if (buf_cat(&cookie->request, ws->request + ws->header_size, rest)) { + /* Without room for what the client pipelined behind this request, answer + this one and let the connection close, rather than keeping it open + around a request we have silently dropped */ + buf_reset(&cookie->request); + ws->keep_alive = 0; + } + } else + buf_reset(&cookie->request); + + written_size = write(sock, ws->reply, ws->reply_size); + /* A send buffer with no room in it is not a failure: nothing went out, so + the whole reply goes into the batch below and leaves when the socket + drains. Only a real error ends the connection here. */ + if (written_size < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) + written_size = 0; + + if ((written_size < 0) || ((written_size == ws->reply_size) && !ws->keep_alive)) { + http_close(sock); + return; + } + + if (written_size < ws->reply_size) { + char *outbuf; + + if (!(outbuf = malloc(ws->reply_size - written_size))) { + http_close(sock); + return; + } + + memcpy(outbuf, ws->reply + written_size, ws->reply_size - written_size); + if (!cookie->batch) { + cookie->batch = calloc(1, sizeof(ot_batch)); + if (!cookie->batch) { + free(outbuf); + http_close(sock); + return; + } + } + + batch_add(cookie->batch, outbuf, ws->reply_size - written_size); + + /* writeable short data sockets just have a tcp timeout */ + if (!ws->keep_alive) + fdstate_set_expiry(sock, 0); + http_interest_set(sock, cookie, EVENT_OUT); + } +} + +#define HTTPERROR_302 return http_issue_error(sock, ws, CODE_HTTPERROR_302) +#define HTTPERROR_400 return http_issue_error(sock, ws, CODE_HTTPERROR_400) +#define HTTPERROR_400_PARAM return http_issue_error(sock, ws, CODE_HTTPERROR_400_PARAM) +#define HTTPERROR_400_COMPACT return http_issue_error(sock, ws, CODE_HTTPERROR_400_COMPACT) +#define HTTPERROR_400_DOUBLEHASH return http_issue_error(sock, ws, CODE_HTTPERROR_400_PARAM) +#define HTTPERROR_402_NOTMODEST return http_issue_error(sock, ws, CODE_HTTPERROR_402_NOTMODEST) +#define HTTPERROR_403_IP return http_issue_error(sock, ws, CODE_HTTPERROR_403_IP) +#define HTTPERROR_404 return http_issue_error(sock, ws, CODE_HTTPERROR_404) +#define HTTPERROR_500 return http_issue_error(sock, ws, CODE_HTTPERROR_500) +/* A connection is either being read or being written, never both, so what it + waits for is set outright rather than added to and taken away again. The + event layer only hears about it when the answer changes. */ +void http_interest_set(const int64_t sock, struct http_data *cookie, int what) { + if (cookie->interest == what) + return; + cookie->interest = what; + /* Still on a stack, so not in the event loop yet: whoever adopts this + connection registers whatever it asked for last */ + if (cookie->flag & STRUCT_HTTP_FLAG_ONSTACK) + return; + event_modify((int)sock, what, fdstate_generation((int)sock)); +} + +ssize_t http_issue_error(const int64_t sock, struct ot_workstruct *ws, int code) { + const char *error_code[] = {"302 Found", "400 Invalid Request", "400 Invalid Request", "400 Invalid Request", "402 Payment Required", + "403 Not Modest", "403 Access Denied", "404 Not Found", "500 Internal Server Error"}; + const char *title = error_code[code]; + + ws->reply = ws->outbuf; + if (code == CODE_HTTPERROR_302) + ws->reply_size = snprintf(ws->reply, G_OUTBUF_SIZE, "HTTP/1.0 302 Found\r\nContent-Length: 0\r\nLocation: %s\r\n\r\n", g_redirecturl); + else + ws->reply_size = snprintf(ws->reply, G_OUTBUF_SIZE, "HTTP/1.0 %s\r\nContent-Type: text/html\r\nContent-Length: %zd\r\n\r\n%s\n", title, + strlen(title) + 16 - 4, title + 4); + +#ifdef _DEBUG_HTTPERROR + fprintf(stderr, "DEBUG: invalid request was: %s\n", ws->debugbuf); +#endif + stats_issue_event(ws, EVENT_FAILED, FLAG_TCP, code); + http_senddata(sock, ws); + return ws->reply_size = -2; +} + +ssize_t http_sendiovecdata(const int64_t sock, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial) { + struct http_data *cookie = fdstate_cookie(sock); + char *header; + const char *encoding = ""; + size_t header_size, size = iovec_length(&iovec_entries, (const struct iovec **)&iovector); + + /* No cookie? Bad socket. Leave. */ + if (!cookie) { + iovec_free(&iovec_entries, &iovector); + HTTPERROR_500; + } + + /* Descriptor numbers get recycled, so a result can only belong to this + connection if this connection is in fact waiting for one. */ + if (!(cookie->flag & (STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER))) { + iovec_free(&iovec_entries, &iovector); + return 0; + } + + /* If this socket collected request in a buffer, free it now */ + buf_reset(&cookie->request); + + /* If we came here, wait for the answer is over */ + if (cookie->flag & STRUCT_HTTP_FLAG_WAITINGFORTASK) { + http_interest_set(sock, cookie, 0); + cookie->flag &= ~STRUCT_HTTP_FLAG_WAITINGFORTASK; + } + + if (iovec_entries) { + + if (cookie->flag & STRUCT_HTTP_FLAG_ZSTD) + encoding = "Content-Encoding: zstd\r\n"; + else if (cookie->flag & STRUCT_HTTP_FLAG_GZIP) + encoding = "Content-Encoding: gzip\r\n"; + else if (cookie->flag & STRUCT_HTTP_FLAG_BZIP2) + encoding = "Content-Encoding: bzip2\r\n"; + + if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED)) + header_size = asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size); + else { + if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) { + header_size = + asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n%sTransfer-Encoding: chunked\r\n\r\n%zx\r\n", encoding, size); + cookie->flag |= STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; + } else + header_size = asprintf(&header, "%zx\r\n", size); + } + if (!header) { + iovec_free(&iovec_entries, &iovector); + HTTPERROR_500; + } + + if (!cookie->batch) { + cookie->batch = calloc(1, sizeof(ot_batch)); + if (!cookie->batch) { + free(header); + iovec_free(&iovec_entries, &iovector); + HTTPERROR_500; + } + } + + batch_add(cookie->batch, header, header_size); + batch_add_vector(cookie->batch, iovector, iovec_entries); + free(iovector); + + if (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) + batch_add_copy(cookie->batch, "\r\n", 2); + } + + if ((cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) && cookie->batch && !is_partial) { + batch_add_copy(cookie->batch, "0\r\n\r\n", 5); + cookie->flag &= ~STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER; + } + + fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT_SEND)); + http_interest_set(sock, cookie, EVENT_OUT); + return 0; +} + +static ssize_t http_handle_stats(const int64_t sock, struct ot_workstruct *ws, char *read_ptr) { + static const ot_keywords keywords_main[] = {{"mode", 1}, {"format", 2}, {"info_hash", 3}, {NULL, -3}}; + static const ot_keywords keywords_mode[] = {{"peer", TASK_STATS_PEERS}, + {"conn", TASK_STATS_CONNS}, + {"scrp", TASK_STATS_SCRAPE}, + {"udp4", TASK_STATS_UDP}, + {"tcp4", TASK_STATS_TCP}, + {"torr", TASK_STATS_TORRENTS}, + {"fscr", TASK_STATS_FULLSCRAPE}, + {"s24s", TASK_STATS_SLASH24S}, + {"tpbs", TASK_STATS_TPB}, + {"herr", TASK_STATS_HTTPERRORS}, + {"completed", TASK_STATS_COMPLETED}, + {"top100", TASK_STATS_TOP100}, + {"top10", TASK_STATS_TOP10}, + {"renew", TASK_STATS_RENEW}, + {"syncs", TASK_STATS_SYNCS}, + {"everything", TASK_STATS_EVERYTHING}, + {"statedump", TASK_FULLSCRAPE_TRACKERSTATE}, + {"woodpeckers", TASK_STATS_WOODPECKERS}, +#ifdef WANT_LOG_NUMWANT + {"numwants", TASK_STATS_NUMWANTS}, +#endif + {NULL, -3}}; + static const ot_keywords keywords_format[] = {{"bin", TASK_FULLSCRAPE_TPB_BINARY}, {"ben", TASK_FULLSCRAPE}, + {"url", TASK_FULLSCRAPE_TPB_URLENCODED}, {"txt", TASK_FULLSCRAPE_TPB_ASCII}, + {"txtp", TASK_FULLSCRAPE_TPB_ASCII_PLUS}, {NULL, -3}}; + + struct http_data *cookie = fdstate_cookie(sock); + char *reply; + int mode = TASK_STATS_PEERS, scanon = 1, format = 0; + + /* A socket without one is a dead socket, which is what http_sendiovecdata + already answers 500 to. */ + if (!cookie) + HTTPERROR_500; + +#ifdef WANT_RESTRICT_STATS + if (!accesslist_is_blessed(cookie->ip, OT_PERMISSION_MAY_STAT)) + HTTPERROR_403_IP; +#endif + + while (scanon) { + switch (urlencoded_match_keyword(keywords_main, &read_ptr, SCAN_SEARCHPATH_PARAM)) { + case -2: + scanon = 0; + break; /* TERMINATOR */ + case -1: + HTTPERROR_400_PARAM; /* PARSE ERROR */ + case -3: + urlencoded_skip_value(&read_ptr); + break; + case 1: /* matched "mode" */ + if ((mode = urlencoded_match_keyword(keywords_mode, &read_ptr, SCAN_SEARCHPATH_VALUE)) <= 0) + HTTPERROR_400_PARAM; + break; + case 2: /* matched "format" */ + if ((format = urlencoded_match_keyword(keywords_format, &read_ptr, SCAN_SEARCHPATH_VALUE)) <= 0) + HTTPERROR_400_PARAM; + break; + case 3: + HTTPERROR_400_PARAM; /* If the stats URL was mistakenly added as announce URL, return a 400 */ + } + } + +#ifdef WANT_FULLSCRAPE + if (mode == TASK_FULLSCRAPE_TRACKERSTATE) { + format = mode; + mode = TASK_STATS_TPB; + } + + if (mode == TASK_STATS_TPB) { +#ifdef WANT_COMPRESSION_GZIP + ws->request[ws->request_size - 1] = 0; +#ifndef WANT_COMPRESSION_GZIP_ALWAYS + if (strstr(read_ptr - 1, "gzip")) { +#endif + cookie->flag |= STRUCT_HTTP_FLAG_GZIP; + format |= TASK_FLAG_GZIP; +#ifndef WANT_COMPRESSION_GZIP_ALWAYS + } +#endif +#endif + cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; + + /* Clients waiting for us should not easily timeout */ + fdstate_set_expiry(sock, 0); + http_interest_set(sock, cookie, 0); + + /* Everything this connection needs has been written, so it can be given + a home of its own and only then be made reachable by a worker thread */ + if (!http_adopt(sock)) + HTTPERROR_500; + fullscrape_deliver(sock, format); + return ws->reply_size = -2; + } +#endif + + /* default format for now */ + if ((mode & TASK_CLASS_MASK) == TASK_STATS) { + + cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK; + + /* Complex stats also include expensive memory debugging tools */ + fdstate_set_expiry(sock, 0); + + if (!http_adopt(sock)) + HTTPERROR_500; + stats_deliver(sock, mode); + return ws->reply_size = -2; + } + + /* Simple stats can be answerred immediately */ + reply = ws->reply; + return_stats_for_tracker(&reply, ws->outbuf + G_OUTBUF_SIZE, mode, 0); + return ws->reply_size = reply - ws->reply; +} + +#ifdef WANT_MODEST_FULLSCRAPES +static pthread_mutex_t g_modest_fullscrape_mutex = PTHREAD_MUTEX_INITIALIZER; +static ot_vector g_modest_fullscrape_timeouts; +typedef struct { + ot_ip6 ip; + ot_time last_fullscrape; +} ot_scrape_log; +#endif + +#ifdef WANT_FULLSCRAPE +static ssize_t http_handle_fullscrape(const int64_t sock, struct ot_workstruct *ws) { + struct http_data *cookie = fdstate_cookie(sock); + int format = 0; + +#ifdef WANT_MODEST_FULLSCRAPES + { + ot_scrape_log this_peer, *new_peer; + int exactmatch; + memcpy(this_peer.ip, cookie->ip, sizeof(ot_ip6)); + this_peer.last_fullscrape = time_now_seconds(); + pthread_mutex_lock(&g_modest_fullscrape_mutex); + new_peer = vector_find_or_insert(&g_modest_fullscrape_timeouts, &this_peer, sizeof(ot_scrape_log), sizeof(ot_ip6), &exactmatch); + if (!new_peer) { + pthread_mutex_unlock(&g_modest_fullscrape_mutex); + HTTPERROR_500; + } + if (exactmatch && (this_peer.last_fullscrape - new_peer->last_fullscrape) < OT_MODEST_PEER_TIMEOUT) { + pthread_mutex_unlock(&g_modest_fullscrape_mutex); + HTTPERROR_402_NOTMODEST; + } + memcpy(new_peer, &this_peer, sizeof(ot_scrape_log)); + pthread_mutex_unlock(&g_modest_fullscrape_mutex); + } +#endif + + +#if defined(WANT_COMPRESSION_GZIP) || defined(WANT_COMPRESSION_ZSTD) + ws->request[ws->request_size - 1] = 0; +#ifdef WANT_COMPRESSION_GZIP + if (strstr(ws->request, "gzip")) { + cookie->flag |= STRUCT_HTTP_FLAG_GZIP; + format |= TASK_FLAG_GZIP; + } +#endif +#ifdef WANT_COMPRESSION_ZSTD + if (strstr(ws->request, "zstd")) { + cookie->flag |= STRUCT_HTTP_FLAG_ZSTD; + format |= TASK_FLAG_ZSTD; + } +#endif + +#if defined(WANT_COMPRESSION_ZSTD) && defined(WANT_COMPRESSION_ZSTD_ALWAYS) + cookie->flag |= STRUCT_HTTP_FLAG_ZSTD; + format |= TASK_FLAG_ZSTD; +#endif + +#if defined(WANT_COMPRESSION_GZIP) && defined(WANT_COMPRESSION_GZIP_ALWAYS) + cookie->flag |= STRUCT_HTTP_FLAG_GZIP; + format |= TASK_FLAG_GZIP; +#endif +#endif + + stats_issue_event(ws, EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip); + +#ifdef _DEBUG_HTTPERROR + fprintf(stderr, "%s", ws->debugbuf); +#endif + + /* Pass this task to the worker thread */ + cookie->flag |= STRUCT_HTTP_FLAG_WAITINGFORTASK | STRUCT_HTTP_FLAG_CHUNKED; + /* Clients waiting for us should not easily timeout */ + fdstate_set_expiry(sock, 0); + http_interest_set(sock, cookie, 0); + + if (!http_adopt(sock)) + HTTPERROR_500; + fullscrape_deliver(sock, format); + return ws->reply_size = -2; +} +#endif + +static int compare_hash(const void *left, const void *right) { return memcmp(left, right, sizeof(ot_hash)); } + +static ssize_t http_handle_scrape(const int64_t sock, struct ot_workstruct *ws, char *read_ptr) { + static const ot_keywords keywords_scrape[] = {{"info_hash", 1}, {NULL, -3}}; + + ot_hash *multiscrape_buf = (ot_hash *)ws->request; + char *reply = ws->reply; + int scanon = 1; + size_t i, unique, numwant = 0; + + /* This is to hack around stupid clients that send "scrape ?info_hash" */ + if (read_ptr[-1] != '?') { + while ((*read_ptr != '?') && (*read_ptr != '\n')) + ++read_ptr; + if (*read_ptr == '\n') + HTTPERROR_400_PARAM; + ++read_ptr; + } + + while (scanon) { + switch (urlencoded_match_keyword(keywords_scrape, &read_ptr, SCAN_SEARCHPATH_PARAM)) { + case -2: + scanon = 0; + break; /* TERMINATOR */ + default: + HTTPERROR_400_PARAM; /* PARSE ERROR */ + case -3: + urlencoded_skip_value(&read_ptr); + break; + case 1: /* matched "info_hash" */ + /* ignore this, when we have less than 20 bytes */ + if (urlencoded_parse(&read_ptr, (char *)(multiscrape_buf + numwant++), SCAN_SEARCHPATH_VALUE) != (ssize_t)sizeof(ot_hash)) + HTTPERROR_400_PARAM; + break; + } + } + + /* No info_hash found? Inform user */ + if (!numwant) + HTTPERROR_400_PARAM; + + /* Limit number of hashes to process */ + if (numwant > OT_MAXMULTISCRAPE_COUNT) + numwant = OT_MAXMULTISCRAPE_COUNT; + + /* A bencoded dictionary has its keys in order and names each one once, so + the answer cannot just follow the order the client asked in */ + qsort(multiscrape_buf, numwant, sizeof(ot_hash), compare_hash); + for (i = unique = 1; i < numwant; ++i) + if (memcmp(multiscrape_buf[i], multiscrape_buf[unique - 1], sizeof(ot_hash))) + memcpy(multiscrape_buf[unique++], multiscrape_buf[i], sizeof(ot_hash)); + numwant = unique; + + /* Enough for http header + whole scrape string */ + return_tcp_scrape_for_torrent((const ot_hash *)multiscrape_buf, numwant, &reply, ws->outbuf + G_OUTBUF_SIZE); + ws->reply_size = reply - ws->reply; + stats_issue_event(ws, EVENT_SCRAPE, FLAG_TCP, ws->reply_size); + return ws->reply_size; +} + +#ifdef WANT_LOG_NUMWANT +unsigned long long numwants[201]; +#endif + +#if defined(WANT_KEEPALIVE) || defined(WANT_IP_FROM_PROXY) +static char *http_header(char *data, size_t byte_count, const char *header) { + size_t i; + long sl = strlen(header); + for (i = 0; i + sl + 2 < byte_count; ++i) { + if (data[i] != '\n' || data[i + sl + 1] != ':') + continue; + if (strncasecmp(data + i + 1, header, sl)) + continue; + data += i + sl + 2; + while (*data == ' ' || *data == '\t') + ++data; + return data; + } + return 0; +} +#endif + +static ot_keywords keywords_announce[] = {{"port", 1}, {"left", 2}, {"event", 3}, {"numwant", 4}, {"compact", 5}, {"compact6", 5}, {"info_hash", 6}, +#ifdef WANT_IP_FROM_QUERY_STRING + {"ip", 7}, +#endif + {"peer_id", 8}, {NULL, -3}}; +static ot_keywords keywords_announce_event[] = {{"completed", 1}, {"stopped", 2}, {NULL, -3}}; + +/* For the replies that are one fixed string and nothing else, where the only + thing the caller wants back is how long it turned out */ +static ssize_t reply_literal(struct ot_workstruct *ws, const char *text) { + char *reply = ws->reply; + + fmt_bytes(&reply, ws->outbuf + G_OUTBUF_SIZE, text, strlen(text)); + return reply - ws->reply; +} + +static ssize_t http_handle_announce(const int64_t sock, struct ot_workstruct *ws, char *read_ptr) { + size_t numwant; + int64_t tmp; + int scanon; + unsigned short port = 0; + char *write_ptr; + ssize_t len; + struct http_data *cookie = fdstate_cookie(sock); + + /* This is to hack around stupid clients that send "announce ?info_hash" */ + if (read_ptr[-1] != '?') { + while ((*read_ptr != '?') && (*read_ptr != '\n')) + ++read_ptr; + if (*read_ptr == '\n') + HTTPERROR_400_PARAM; + ++read_ptr; + } + +#ifdef WANT_IP_FROM_PROXY + if (accesslist_is_blessed(cookie->ip, OT_PERMISSION_MAY_PROXY)) { + ot_ip6 proxied_ip; + char *fwd = http_header(ws->request, ws->header_size, "x-forwarded-for"); + if (fwd && ip_parse(fwd, proxied_ip)) { + OT_SETIP(ws->peer, proxied_ip); + } else + OT_SETIP(ws->peer, cookie->ip); + } else +#endif + OT_SETIP(ws->peer, cookie->ip); + + ws->peer_id = NULL; + ws->hash = NULL; + + OT_SETPORT(ws->peer, &port); + OT_PEERFLAG(ws->peer) = 0; + numwant = 50; + scanon = 1; + + while (scanon) { + switch (urlencoded_match_keyword(keywords_announce, &read_ptr, SCAN_SEARCHPATH_PARAM)) { + case -2: + scanon = 0; + break; /* TERMINATOR */ + case -1: + HTTPERROR_400_PARAM; /* PARSE ERROR */ + case -3: + urlencoded_skip_value(&read_ptr); + break; + case 1: /* matched "port" */ + len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE); + if ((len <= 0) || scan_int(write_ptr, len, &tmp) || (tmp > 0xffff)) + HTTPERROR_400_PARAM; + port = htons((uint16_t)tmp); + OT_SETPORT(&ws->peer, &port); + break; + case 2: /* matched "left" */ + if ((len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE)) <= 0) + HTTPERROR_400_PARAM; + if (scan_int(write_ptr, len, &tmp)) + tmp = 0; + if (!tmp) + OT_PEERFLAG(&ws->peer) |= PEER_FLAG_SEEDING; + break; + case 3: /* matched "event" */ + switch (urlencoded_match_keyword(keywords_announce_event, &read_ptr, SCAN_SEARCHPATH_VALUE)) { + case -1: + HTTPERROR_400_PARAM; + case 1: /* matched "completed" */ + OT_PEERFLAG(&ws->peer) |= PEER_FLAG_COMPLETED; + break; + case 2: /* matched "stopped" */ + OT_PEERFLAG(&ws->peer) |= PEER_FLAG_STOPPED; + break; + default: + break; + } + break; + case 4: /* matched "numwant" */ + len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE); + if ((len <= 0) || scan_int(write_ptr, len, &tmp)) + HTTPERROR_400_PARAM; + if (tmp < 0) + tmp = 50; + if (tmp > 200) + tmp = 200; + numwant = (size_t)tmp; + break; + case 5: /* matched "compact" */ + len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE); + if ((len <= 0) || scan_int(write_ptr, len, &tmp)) + HTTPERROR_400_PARAM; + if (!tmp) + HTTPERROR_400_COMPACT; + break; + case 6: /* matched "info_hash" */ + if (ws->hash) + HTTPERROR_400_DOUBLEHASH; + /* ignore this, when we have less than 20 bytes */ + if (urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE) != 20) + HTTPERROR_400_PARAM; + ws->hash = (ot_hash *)write_ptr; + break; +#ifdef WANT_IP_FROM_QUERY_STRING + case 7: /* matched "ip" */ + { + char *tmp_buf1 = ws->reply, *tmp_buf2 = ws->reply + 16; + len = urlencoded_parse(&read_ptr, tmp_buf2, SCAN_SEARCHPATH_VALUE); + tmp_buf2[len] = 0; + if ((len <= 0) || ip_parse(tmp_buf2, tmp_buf1) != (size_t)len) + HTTPERROR_400_PARAM; + OT_SETIP(&ws->peer, tmp_buf1); + } break; +#endif + case 8: /* matched "peer_id" */ + /* ignore this, when we have less than 20 bytes */ + if (urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_SEARCHPATH_VALUE) != 20) + HTTPERROR_400_PARAM; + ws->peer_id = write_ptr; + break; + } + } + +#ifdef WANT_LOG_NUMWANT + numwants[numwant]++; +#endif + + /* XXX DEBUG + stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ws->reply ); + */ + + /* Scanned whole query string */ + if (!ws->hash) + return ws->reply_size = reply_literal(ws, "d14:failure reason80:Your client forgot to send your torrent's info_hash. Please upgrade your client.e"); + + if (OT_PEERFLAG(&ws->peer) & PEER_FLAG_STOPPED) + ws->reply_size = remove_peer_from_torrent(FLAG_TCP, ws); + else + ws->reply_size = add_peer_to_torrent_and_return_peers(FLAG_TCP, ws, numwant); + + stats_issue_event(ws, EVENT_ANNOUNCE, FLAG_TCP, ws->reply_size); + return ws->reply_size; +} + +ssize_t http_handle_request(const int64_t sock, struct ot_workstruct *ws) { + ssize_t reply_off, len; + char *read_ptr = ws->request, *write_ptr; + +#ifdef _DEBUG_HTTPERROR + reply_off = ws->request_size; + if (ws->request_size >= G_DEBUGBUF_SIZE) + reply_off = G_DEBUGBUF_SIZE - 1; + memcpy(ws->debugbuf, ws->request, reply_off); + ws->debugbuf[reply_off] = 0; +#endif + + /* Tell subroutines where to put reply data */ + ws->reply = ws->outbuf + SUCCESS_HTTP_HEADER_LENGTH; + + /* Find out if the client wants to keep this connection alive, before any + subroutine gets to answer: an error reply finishes with the connection + on its way out, and ws outlives the connection it was serving */ + ws->keep_alive = 0; +#ifdef WANT_KEEPALIVE + if ((write_ptr = http_header(ws->request, ws->header_size, "connection")) && (*write_ptr == 'K' || *write_ptr == 'k')) + ws->keep_alive = 1; +#endif + { + struct http_data *connection = fdstate_cookie(sock); + if (connection) { + if (ws->keep_alive) + connection->flag |= STRUCT_HTTP_FLAG_KEEPALIVE; + else + connection->flag &= ~STRUCT_HTTP_FLAG_KEEPALIVE; + } + } + + /* This one implicitely tests strlen < 5, too -- remember, it is \n terminated */ + if (memcmp(read_ptr, "GET /", 5)) + HTTPERROR_400; + + /* Skip leading '/' */ + for (read_ptr += 4; *read_ptr == '/'; ++read_ptr) + ; + + /* Try to parse the request. + In reality we abandoned requiring the url to be correct. This now + only decodes url encoded characters, we check for announces and + scrapes by looking for "a*" or "sc" */ + len = urlencoded_parse(&read_ptr, write_ptr = read_ptr, SCAN_PATH); + + /* If parsing returned an error, leave with not found */ + if (g_redirecturl && (len == -2)) + HTTPERROR_302; + if (len <= 0) + HTTPERROR_404; + + /* This is the hardcore match for announce*/ + if ((*write_ptr == 'a') || (*write_ptr == '?')) + http_handle_announce(sock, ws, read_ptr); +#ifdef WANT_FULLSCRAPE + else if (!memcmp(write_ptr, "scrape HTTP/", 12)) + http_handle_fullscrape(sock, ws); +#endif + /* This is the hardcore match for scrape */ + else if (!memcmp(write_ptr, "sc", 2)) + http_handle_scrape(sock, ws, read_ptr); + /* All the rest is matched the standard way */ + else if (len == g_stats_path_len && !memcmp(write_ptr, g_stats_path, len)) + http_handle_stats(sock, ws, read_ptr); + else + HTTPERROR_404; + + /* If routines handled sending themselves, just return */ + if (ws->reply_size == -2) + return 0; + /* If routine failed, let http error take over */ + if (ws->reply_size <= 0) + HTTPERROR_500; + + /* This one is rather ugly, so I take you step by step through it. + + 1. In order to avoid having two buffers, one for header and one for content, we allow all above functions from trackerlogic to + write to a fixed location, leaving SUCCESS_HTTP_HEADER_LENGTH bytes in our work buffer, which is enough for the static string + plus dynamic space needed to expand our Content-Length value. We reserve SUCCESS_HTTP_SIZE_OFF for its expansion and calculate + the space NOT needed to expand in reply_off + */ + reply_off = SUCCESS_HTTP_SIZE_OFF - snprintf(ws->outbuf, 0, "%zd", ws->reply_size); + ws->reply = ws->outbuf + reply_off; + + /* 2. Now we write the header so that its terminating '\0' lands exactly one byte before the content starts. The bound is the + space step 1 reserved, so the '\0' falls where step 3 expects it. Complete packet size is increased by size of header plus + one byte '\n', we will copy over '\0' in next step */ + ws->reply_size += 1 + snprintf(ws->reply, (size_t)(SUCCESS_HTTP_HEADER_LENGTH - reply_off), + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %zd\r\n\r", ws->reply_size); + + /* 3. Finally we join both blocks neatly */ + ws->outbuf[SUCCESS_HTTP_HEADER_LENGTH - 1] = '\n'; + + http_senddata(sock, ws); + return ws->reply_size; +} diff --git a/src/ot_http.h b/src/ot_http.h new file mode 100644 index 0000000..fb56c64 --- /dev/null +++ b/src/ot_http.h @@ -0,0 +1,53 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_HTTP_H__ +#define OT_HTTP_H__ + +#include +#include +#include + +#include "ot_batch.h" +#include "ot_event.h" +#include "ot_buf.h" +#include "trackerlogic.h" + +typedef enum { + STRUCT_HTTP_FLAG_WAITINGFORTASK = 1, + STRUCT_HTTP_FLAG_GZIP = 2, + STRUCT_HTTP_FLAG_BZIP2 = 4, + STRUCT_HTTP_FLAG_ZSTD = 8, + STRUCT_HTTP_FLAG_CHUNKED = 16, + STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 32, + STRUCT_HTTP_FLAG_KEEPALIVE = 64, + STRUCT_HTTP_FLAG_ONSTACK = 128 +} STRUCT_HTTP_FLAG; + +struct http_data { + ot_buf request; + int interest; /* EVENT_IN and EVENT_OUT as asked for */ + ot_batch *batch; + ot_ip6 ip; + STRUCT_HTTP_FLAG flag; +}; + +/* What this connection is waiting for: EVENT_IN, EVENT_OUT, or nothing */ +void http_interest_set(const int64_t s, struct http_data *cookie, int what); + +/* Close a connection and release everything that belongs to it */ +void http_close(const int64_t s); + +/* Promote a stack cookie to a heap one, for persistence and for handing it + over to the event loop or to another thread. Does nothing to one that is + already on the heap. Returns the connection's state, or NULL. */ +struct http_data *http_adopt(const int64_t s); + +ssize_t http_handle_request(const int64_t s, struct ot_workstruct *ws); +ssize_t http_sendiovecdata(const int64_t s, struct ot_workstruct *ws, size_t iovec_entries, struct iovec *iovector, int is_partial); +ssize_t http_issue_error(const int64_t s, struct ot_workstruct *ws, int code); + +extern const char *g_stats_path; +extern ssize_t g_stats_path_len; + +#endif diff --git a/src/ot_iovec.c b/src/ot_iovec.c new file mode 100644 index 0000000..328f160 --- /dev/null +++ b/src/ot_iovec.c @@ -0,0 +1,87 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_iovec.h" + +void *iovec_new_chunk(size_t *iovec_entries, struct iovec **iovector, size_t new_alloc) { + void *new_data; + size_t new_entries = 1 + *iovec_entries; + struct iovec *new_vec = realloc(*iovector, new_entries * sizeof(struct iovec)); + + if (!new_vec) + return NULL; + + *iovector = new_vec; + + /* Only allocate after we have a place to store the pointer */ + new_data = malloc(new_alloc); + if (!new_data) + return NULL; + + new_vec[new_entries - 1].iov_base = new_data; + new_vec[new_entries - 1].iov_len = new_alloc; + + ++*iovec_entries; + return new_data; +} + +void *iovec_append(size_t *iovec_entries, struct iovec **iovector, struct iovec *append_iovector) { + size_t new_entries = *iovec_entries + 1; + struct iovec *new_vec = realloc(*iovector, new_entries * sizeof(struct iovec)); + if (!new_vec) + return NULL; + + /* Take over data from appended iovec */ + new_vec[*iovec_entries].iov_base = append_iovector->iov_base; + new_vec[*iovec_entries].iov_len = append_iovector->iov_len; + + append_iovector->iov_base = NULL; + append_iovector->iov_len = 0; + + *iovector = new_vec; + *iovec_entries = new_entries; + + return new_vec; +} + +void iovec_free(size_t *iovec_entries, struct iovec **iovector) { + size_t i; + for (i = 0; i < *iovec_entries; ++i) + free(((*iovector)[i]).iov_base); + free(*iovector); + *iovector = NULL; + *iovec_entries = 0; +} + +void iovec_trim_last(size_t *iovec_entries, struct iovec **iovector, void *last_ptr) { + if (*iovec_entries) { + char *base = (char *)((*iovector)[*iovec_entries - 1]).iov_base; + size_t new_alloc = ((char *)last_ptr) - base; + + ((*iovector)[*iovec_entries - 1]).iov_base = realloc(base, new_alloc); + ((*iovector)[*iovec_entries - 1]).iov_len = new_alloc; + } +} + +void *iovec_next_chunk(size_t *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc) { + void *new_data; + + iovec_trim_last(iovec_entries, iovector, last_ptr); + + if (!(new_data = iovec_new_chunk(iovec_entries, iovector, new_alloc))) + iovec_free(iovec_entries, iovector); + + return new_data; +} + +size_t iovec_length(const size_t *iovec_entries, const struct iovec **iovector) { + size_t i, length = 0; + for (i = 0; i < *iovec_entries; ++i) + length += ((*iovector)[i]).iov_len; + return length; +} diff --git a/src/ot_iovec.h b/src/ot_iovec.h new file mode 100644 index 0000000..4a8abbb --- /dev/null +++ b/src/ot_iovec.h @@ -0,0 +1,18 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_IOVEC_H__ +#define OT_IOVEC_H__ + +#include + +void *iovec_new_chunk(size_t *iovec_entries, struct iovec **iovector, size_t new_alloc); +void *iovec_append(size_t *iovec_entries, struct iovec **iovector, struct iovec *append_iovector); +void iovec_trim_last(size_t *iovec_entries, struct iovec **iovector, void *last_ptr); +void iovec_free(size_t *iovec_entries, struct iovec **iovector); + +size_t iovec_length(const size_t *iovec_entries, const struct iovec **iovector); + +void *iovec_next_chunk(size_t *iovec_entries, struct iovec **iovector, void *last_ptr, size_t new_alloc); + +#endif diff --git a/src/ot_ip.c b/src/ot_ip.c new file mode 100644 index 0000000..9c76290 --- /dev/null +++ b/src/ot_ip.c @@ -0,0 +1,199 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_hex.h" +#include "ot_ip.h" + +const char ip_v4mapped_prefix[12] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}; + +/* The dotted quad is only ever reached through the v4 mapped form, so both + halves of it stay in here */ + +static size_t ip4_format(char *dest, const uint8_t *ip) { + char *d = dest; + int i; + + for (i = 0; i < 4; ++i) { + unsigned int octet = ip[i]; + + if (i) + *d++ = '.'; + if (octet >= 100) + *d++ = (char)('0' + octet / 100); + if (octet >= 10) + *d++ = (char)('0' + (octet / 10) % 10); + *d++ = (char)('0' + octet % 10); + } + + return (size_t)(d - dest); +} + +static size_t ip4_scan(const char *src, uint8_t ip[4]) { + uint8_t octets[4]; + const char *s = src; + int i; + + for (i = 0; i < 4; ++i) { + unsigned int value = 0; + int digits = 0; + + if (i && *s++ != '.') + return 0; + + /* Leading zeros are tolerated, a value beyond one octet is not */ + while (*s >= '0' && *s <= '9' && digits < 4) { + value = value * 10 + (unsigned int)(*s++ - '0'); + ++digits; + } + if (!digits || value > 255) + return 0; + + octets[i] = (uint8_t)value; + } + + memcpy(ip, octets, sizeof(octets)); + return (size_t)(s - src); +} + +static unsigned int group_at(const char *ip, int n) { return ((unsigned int)(uint8_t)ip[2 * n] << 8) | (uint8_t)ip[2 * n + 1]; } + +size_t ip_format(char *dest, const ot_ip6 ip) { + static const char hexdigit[] = "0123456789abcdef"; + char *d = dest; + int best_start = 0, best_len = 0, i = 0; + + if (ip_is_v4mapped(ip)) + return ip4_format(dest, (const uint8_t *)ip + sizeof(ip_v4mapped_prefix)); + + /* RFC 5952 4.2: the longest run of zero groups becomes "::", never a run + of one, and the first of two equal runs wins */ + while (i < 8) { + int j = i; + + if (group_at(ip, i)) { + ++i; + continue; + } + while (j < 8 && !group_at(ip, j)) + ++j; + if (j - i > best_len) { + best_len = j - i; + best_start = i; + } + i = j; + } + if (best_len < 2) + best_len = 0; + + for (i = 0; i < 8; ++i) { + unsigned int group; + int shift = 12; + + if (best_len && i == best_start) { + *d++ = ':'; + if (i + best_len == 8) + *d++ = ':'; /* nothing follows, so close the run */ + i += best_len - 1; + continue; + } + + if (i) + *d++ = ':'; + + group = group_at(ip, i); + while (shift && !(group >> shift)) + shift -= 4; /* RFC 5952 4.1: no leading zeros */ + for (;; shift -= 4) { + *d++ = hexdigit[(group >> shift) & 15]; + if (!shift) + break; + } + } + + return (size_t)(d - dest); +} + +size_t ip_parse(const char *src, ot_ip6 ip) { + uint8_t prefix[16], suffix[16]; + size_t prefixlen = 0, suffixlen = 0; + const char *s = src; + int have_gap = 0; + int need_colon = 0; + + /* A plain IPv4 address is an address too, stored v4 mapped */ + { + size_t consumed = ip4_scan(src, prefix); + if (consumed) { + memcpy(ip, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); + memcpy(ip + sizeof(ip_v4mapped_prefix), prefix, 4); + return consumed; + } + } + + for (;;) { + unsigned int value = 0; + int digits = 0, nibble; + uint8_t *into = have_gap ? suffix : prefix; + size_t *fill = have_gap ? &suffixlen : &prefixlen; + + /* RFC 4291 2.2: one run of zero groups may be written "::" */ + if (s[0] == ':' && s[1] == ':') { + if (have_gap) + return 0; + have_gap = 1; + need_colon = 0; + s += 2; + continue; + } + + if (need_colon) { + if (*s != ':') + break; + ++s; + } + + /* RFC 4291 2.2 form 3: the final two groups may be a dotted quad */ + if (prefixlen + suffixlen + 4 <= 16) { + size_t consumed = ip4_scan(s, into + *fill); + if (consumed) { + *fill += 4; + s += consumed; + break; + } + } + + while (digits < 4 && (nibble = hex_value(*s)) >= 0) { + value = value * 16 + (unsigned int)nibble; + ++digits; + ++s; + } + if (!digits) { + if (need_colon) + --s; /* leave the separator we ate for the caller */ + break; + } + + if (prefixlen + suffixlen + 2 > 16) + return 0; + + into[(*fill)++] = (uint8_t)(value >> 8); + into[(*fill)++] = (uint8_t)value; + need_colon = 1; + } + + if (!prefixlen && !suffixlen && !have_gap) + return 0; + if (!have_gap && prefixlen + suffixlen != 16) + return 0; /* without "::" every group has to be spelled out */ + + memset(ip, 0, 16); + memcpy(ip, prefix, prefixlen); + memcpy(ip + 16 - suffixlen, suffix, suffixlen); + + return (size_t)(s - src); +} diff --git a/src/ot_ip.h b/src/ot_ip.h new file mode 100644 index 0000000..533403e --- /dev/null +++ b/src/ot_ip.h @@ -0,0 +1,31 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_IP_H__ +#define OT_IP_H__ + +#include +#include + +#include "trackerlogic.h" + +/* opentracker holds every address as sixteen bytes, writing IPv4 as the v4 + mapped form of RFC 4291 section 2.5.5.2. */ +extern const char ip_v4mapped_prefix[12]; + +static inline int ip_is_v4mapped(const void *ip) { return !memcmp(ip, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); } + +/* Read an address and return how many characters it took, or zero if src + does not start with one. A plain IPv4 address is accepted and stored in + its v4 mapped form. + + The count lets the caller carry on where the address ended, which is how + "address:port" and "address/bits" are read. */ +size_t ip_parse(const char *src, ot_ip6 ip); + +/* Write an address and return how many characters it took. Nothing is + terminated, so a caller may keep appending. A v4 mapped address is written + as the plain dotted quad rather than "::ffff:1.2.3.4". */ +size_t ip_format(char *dest, const ot_ip6 ip); + +#endif diff --git a/src/ot_livesync.c b/src/ot_livesync.c new file mode 100644 index 0000000..083fe1a --- /dev/null +++ b/src/ot_livesync.c @@ -0,0 +1,241 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_ip.h" +#include "ot_socket.h" +#include "ot_livesync.h" +#include "ot_mutex.h" +#include "ot_stats.h" +#include "trackerlogic.h" + +#ifdef WANT_SYNC_LIVE + +ot_ip6 groupip_1 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 224, 0, 23, 5}; + +#define LIVESYNC_INCOMING_BUFFSIZE (256 * 256) + +#define LIVESYNC_OUTGOING_BUFFSIZE_PEERS 1480 +#define LIVESYNC_OUTGOING_WATERMARK_PEERS (sizeof(ot_peer) + sizeof(ot_hash)) + +#define LIVESYNC_MAXDELAY 15 /* seconds */ + +enum { OT_SYNC_PEER4, OT_SYNC_PEER6 }; + +/* Forward declaration */ +static void *livesync_worker(void *args); + +/* For incoming packets */ +static int g_socket_in = -1; + +/* For outgoing packets */ +static int g_socket_out = -1; + +static pthread_mutex_t g_outbuf_mutex = PTHREAD_MUTEX_INITIALIZER; +typedef struct { + uint8_t data[LIVESYNC_OUTGOING_BUFFSIZE_PEERS]; + size_t fill; + ot_time next_packet_time; +} sync_buffer; + +static sync_buffer g_v6_buf; +static sync_buffer g_v4_buf; + +static pthread_t thread_id; +static int thread_started; +static _Atomic int thread_left; +void livesync_init(void) { + uint32_t packet_type6 = htonl(OT_SYNC_PEER6); + uint32_t packet_type4 = htonl(OT_SYNC_PEER4); + + if (g_socket_in == -1) + exerr("No socket address for live sync specified."); + + /* Prepare outgoing peers buffer */ + memcpy(g_v6_buf.data, &g_tracker_id, sizeof(g_tracker_id)); + memcpy(g_v4_buf.data, &g_tracker_id, sizeof(g_tracker_id)); + + memcpy(g_v6_buf.data + sizeof(g_tracker_id), &packet_type6, sizeof(packet_type6)); + memcpy(g_v4_buf.data + sizeof(g_tracker_id), &packet_type4, sizeof(packet_type4)); + + g_v6_buf.fill = sizeof(g_tracker_id) + sizeof(uint32_t); + g_v4_buf.fill = sizeof(g_tracker_id) + sizeof(uint32_t); + + g_v6_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); + g_v4_buf.next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); + + thread_started = !pthread_create(&thread_id, NULL, livesync_worker, NULL); +} + +void livesync_deinit(void) { + if (thread_started) { + /* Repeats so it interrupts a read already under way */ + while (!atomic_load(&thread_left)) { + pthread_kill(thread_id, OT_SIGNAL_WAKE); + usleep(1000); + } + pthread_join(thread_id, NULL); + thread_started = 0; + } + + if (g_socket_in != -1) + close(g_socket_in); + if (g_socket_out != -1) + close(g_socket_out); +} + +void livesync_bind_mcast(ot_ip6 ip, uint16_t port) { + if (!ip_is_v4mapped(ip)) + exerr("v6 mcast support not yet available."); + + if (g_socket_in != -1) + exerr("Error: Livesync listen ip specified twice."); + + if ((g_socket_in = socket_mcast_join(groupip_1, ip, port)) == -1) + exerr("Error: Cant bind live sync incoming socket."); + + if ((g_socket_out = socket_mcast_bind(ip, port, 0)) == -1) + exerr("Error: Cant bind live sync outgoing socket."); +} + +/* Caller MUST hold g_outbuf_mutex. Returns with g_outbuf_mutex unlocked */ +static void livesync_issue_peersync(sync_buffer *buf) { + char mycopy[LIVESYNC_OUTGOING_BUFFSIZE_PEERS]; + size_t fill = buf->fill; + + memcpy(mycopy, buf->data, fill); + buf->fill = sizeof(g_tracker_id) + sizeof(uint32_t); + buf->next_packet_time = time_future_seconds(LIVESYNC_MAXDELAY); + + /* From now this thread has a local copy of the buffer and + has modified the protected element */ + pthread_mutex_unlock(&g_outbuf_mutex); + + socket_mcast_send(g_socket_out, mycopy, fill, groupip_1, LIVESYNC_PORT); +} + +static void livesync_handle_peersync(struct ot_workstruct *ws, size_t peer_size) { + size_t off = sizeof(g_tracker_id) + sizeof(uint32_t); + + /* Prepare peer record for v4mapped expansion */ + if (peer_size == OT_PEER_SIZE4) + memcpy(&ws->peer, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); + + /* Now basic sanity checks have been done on the live sync packet + We might add more testing and logging. */ + while ((ssize_t)(off + sizeof(ot_hash) + peer_size) <= ws->request_size) { + char *peer_ip = ws->request + off + sizeof(ot_hash); + if (peer_size == OT_PEER_SIZE4) + memcpy(&ws->peer[0] + sizeof(ip_v4mapped_prefix), peer_ip, peer_size); + else + memcpy(&ws->peer, peer_ip, peer_size); + ws->hash = (ot_hash *)(ws->request + off); + + if (!g_opentracker_running) + return; + + if (OT_PEERFLAG(ws->peer) & PEER_FLAG_STOPPED) + remove_peer_from_torrent(FLAG_MCA, ws); + else + add_peer_to_torrent_and_return_peers(FLAG_MCA, ws, /* amount = */ 0); + + off += sizeof(ot_hash) + peer_size; + } + + stats_issue_event(ws, EVENT_SYNC, 0, (ws->request_size - sizeof(g_tracker_id) - sizeof(uint32_t)) / ((ssize_t)sizeof(ot_hash) + peer_size)); +} + +/* Tickle the live sync module from time to time, so no events get + stuck when there's not enough traffic to fill udp packets fast + enough */ +void livesync_ticker(void) { + /* livesync_issue_peersync sets g_next_packet_time */ + pthread_mutex_lock(&g_outbuf_mutex); + if (time_now_seconds() > g_v6_buf.next_packet_time && g_v6_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t)) + livesync_issue_peersync(&g_v6_buf); + else + pthread_mutex_unlock(&g_outbuf_mutex); + + pthread_mutex_lock(&g_outbuf_mutex); + if (time_now_seconds() > g_v4_buf.next_packet_time && g_v4_buf.fill > sizeof(g_tracker_id) + sizeof(uint32_t)) + livesync_issue_peersync(&g_v4_buf); + else + pthread_mutex_unlock(&g_outbuf_mutex); +} + +/* Inform live sync about whats going on. */ +void livesync_tell(struct ot_workstruct *ws) { + size_t peer_size; /* initialized in next line */ + ot_peer *peer_src = peer_from_peer6(&ws->peer, &peer_size); + sync_buffer *dest_buf = peer_size == OT_PEER_SIZE6 ? &g_v6_buf : &g_v4_buf; + + pthread_mutex_lock(&g_outbuf_mutex); + + memcpy(dest_buf->data + dest_buf->fill, ws->hash, sizeof(ot_hash)); + dest_buf->fill += sizeof(ot_hash); + + memcpy(dest_buf->data + dest_buf->fill, peer_src, peer_size); + dest_buf->fill += peer_size; + + if (dest_buf->fill >= LIVESYNC_OUTGOING_BUFFSIZE_PEERS - LIVESYNC_OUTGOING_WATERMARK_PEERS) + livesync_issue_peersync(dest_buf); + else + pthread_mutex_unlock(&g_outbuf_mutex); +} + +static void *livesync_worker(void *args) { + struct ot_workstruct ws; + ot_ip6 in_ip; + uint16_t in_port; + + (void)args; + + /* Initialize our "thread local storage" */ + ws.inbuf = ws.request = malloc(LIVESYNC_INCOMING_BUFFSIZE); + ws.outbuf = ws.reply = 0; + + while (g_opentracker_running) { + uint32_t packet_type; + + ws.request_size = socket_recv(g_socket_in, ws.inbuf, LIVESYNC_INCOMING_BUFFSIZE, in_ip, &in_port); + + /* Expect at least tracker id and packet type */ + if (ws.request_size <= (ssize_t)(sizeof(g_tracker_id) + sizeof(uint32_t))) + continue; + if (!accesslist_is_blessed(in_ip, OT_PERMISSION_MAY_LIVESYNC)) + continue; + if (!memcmp(ws.inbuf, &g_tracker_id, sizeof(g_tracker_id))) { + /* TODO: log packet coming from ourselves */ + continue; + } + + memcpy(&packet_type, ws.inbuf + sizeof(g_tracker_id), sizeof(packet_type)); + + switch (ntohl(packet_type)) { + case OT_SYNC_PEER6: + livesync_handle_peersync(&ws, OT_PEER_SIZE6); + break; + case OT_SYNC_PEER4: + livesync_handle_peersync(&ws, OT_PEER_SIZE4); + break; + default: + break; + } + } + + free(ws.inbuf); + atomic_store(&thread_left, 1); + return NULL; +} + +#endif diff --git a/src/ot_livesync.h b/src/ot_livesync.h new file mode 100644 index 0000000..4bd3af0 --- /dev/null +++ b/src/ot_livesync.h @@ -0,0 +1,75 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_LIVESYNC_H__ +#define OT_LIVESYNC_H__ + +#include "trackerlogic.h" + +/* + Syncing is done as udp packets in the multicast domain 224.0.42.5 port 9696 + + Each tracker should join the multicast group and send its live sync packets + to that group, using a ttl of 1 + + Format of all sync packets is straight forward, packet type determines + which kind of packet this is: + + 0x0000 0x04 id of tracker instance + 0x0004 0x04 packet type + + ######## + ######## PEER SYNC PROTOCOL ######## + ######## + + Each tracker instance accumulates announce requests until its buffer is + full or a timeout is reached. Then it broadcasts its live sync packer: + + packet type SYNC_LIVE4 + [ 0x0008 0x14 info_hash + 0x001c 0x04 peer's ipv4 address + 0x0020 0x02 peer's port + 0x0024 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 ) + ]* + + packet type SYNC_LIVE6 + [ 0x0008 0x14 info_hash + 0x001c 0x10 peer's ipv6 address + 0x002c 0x02 peer's port + 0x002e 0x02 peer flags v1 ( SEEDING = 0x80, COMPLETE = 0x40, STOPPED = 0x20 ) + ]* +*/ + +#ifdef WANT_SYNC_LIVE + +#define LIVESYNC_PORT 9696 + +void livesync_init(void); +void livesync_deinit(void); + +/* Join multicast group for listening and create sending socket */ +void livesync_bind_mcast(ot_ip6 ip, uint16_t port); + +/* Inform live sync about whats going on. */ +void livesync_tell(struct ot_workstruct *ws); + +/* Tickle the live sync module from time to time, so no events get + stuck when there's not enough traffic to fill udp packets fast + enough */ +void livesync_ticker(void); + +/* Handle an incoming live sync packet */ +void handle_livesync(const int64_t sock); + +#else + +/* If no syncing is required, save calling code from #ifdef + constructions */ +#define livesync_deinit() +#define livesync_init() +#define livesync_ticker() +#define handle_livesync(a) + +#endif + +#endif diff --git a/src/ot_mutex.c b/src/ot_mutex.c new file mode 100644 index 0000000..884b74c --- /dev/null +++ b/src/ot_mutex.c @@ -0,0 +1,70 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include +#include + + +/* Opentracker */ +#include "ot_iovec.h" +#include "ot_mutex.h" +#include "ot_stats.h" +#include "trackerlogic.h" + +/* #define MTX_DBG( STRING ) fprintf( stderr, STRING ) */ +#define MTX_DBG(STRING) + +/* Our global all torrents list */ +static ot_vector all_torrents[OT_BUCKET_COUNT]; +static pthread_mutex_t bucket_mutex[OT_BUCKET_COUNT]; +/* Every bucket's unlock adjusts this, and a bucket lock only keeps other + threads out of that one bucket, so the counter needs its own protection */ +static _Atomic size_t g_torrent_count; + +ot_vector *mutex_bucket_lock(int bucket) { + pthread_mutex_lock(bucket_mutex + bucket); + return all_torrents + bucket; +} + +/* Torrents are spread over buckets by the first four bytes of their info_hash, + which may sit at any alignment, hence the memcpy */ +static int bucket_from_hash(ot_hash const hash) { + uint32_t bucket; + memcpy(&bucket, hash, sizeof(bucket)); + return ntohl(bucket) >> OT_BUCKET_COUNT_SHIFT; +} + +ot_vector *mutex_bucket_lock_by_hash(ot_hash const hash) { return mutex_bucket_lock(bucket_from_hash(hash)); } + +void mutex_bucket_unlock(int bucket, int delta_torrentcount) { + pthread_mutex_unlock(bucket_mutex + bucket); + if (delta_torrentcount) + atomic_fetch_add_explicit(&g_torrent_count, (size_t)delta_torrentcount, memory_order_relaxed); +} + +void mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcount) { + mutex_bucket_unlock(bucket_from_hash(hash), delta_torrentcount); +} + +size_t mutex_get_torrent_count(void) { return atomic_load_explicit(&g_torrent_count, memory_order_relaxed); } + +void mutex_init(void) { + size_t i; + for (i = 0; i < OT_BUCKET_COUNT; ++i) + pthread_mutex_init(bucket_mutex + i, NULL); + memset(all_torrents, 0, sizeof(all_torrents)); +} + +void mutex_deinit(void) { + size_t i; + for (i = 0; i < OT_BUCKET_COUNT; ++i) + pthread_mutex_destroy(bucket_mutex + i); + memset(all_torrents, 0, sizeof(all_torrents)); +} diff --git a/src/ot_mutex.h b/src/ot_mutex.h new file mode 100644 index 0000000..d7ded92 --- /dev/null +++ b/src/ot_mutex.h @@ -0,0 +1,21 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_MUTEX_H__ +#define OT_MUTEX_H__ + +#include +#include "trackerlogic.h" + +void mutex_init(void); +void mutex_deinit(void); + +ot_vector *mutex_bucket_lock(int bucket); +ot_vector *mutex_bucket_lock_by_hash(ot_hash const hash); + +void mutex_bucket_unlock(int bucket, int delta_torrentcount); +void mutex_bucket_unlock_by_hash(ot_hash const hash, int delta_torrentcount); + +size_t mutex_get_torrent_count(void); + +#endif diff --git a/src/ot_random.c b/src/ot_random.c new file mode 100644 index 0000000..43f91bb --- /dev/null +++ b/src/ot_random.c @@ -0,0 +1,51 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "ot_random.h" + +#ifdef WANT_INSECURE_RANDOM_FALLBACK + +#warning WANT_INSECURE_RANDOM_FALLBACK builds without arc4random. UDP connection ids become predictable and announces can be forged. Do not ship this. + +#include + +void random_init(void) { +#ifdef WANT_DEV_RANDOM + srandomdev(); +#else + srandom(time(NULL)); +#endif +} + +/* random() yields 31 bits, so one draw can never set the top bit. Combine + two of them rather than hand out a value that is quietly half as wide as + its type claims. */ +uint32_t random_uint32(void) { return ((uint32_t)random() << 16) ^ (uint32_t)random(); } + +void random_bytes(void *buf, size_t len) { + uint8_t *out = buf; + /* random() leaves the top bit clear and its lowest bits are the weakest, + so take a byte from the middle of each draw */ + while (len--) + *out++ = (uint8_t)(random() >> 8); +} + +#else + +#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) && !defined(__DragonFly__) && !defined(__APPLE__) && \ + !(defined(__GLIBC__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 36))) +#include +#endif + +void random_init(void) {} + +uint32_t random_uint32(void) { return arc4random(); } + +void random_bytes(void *buf, size_t len) { arc4random_buf(buf, len); } + +#endif diff --git a/src/ot_random.h b/src/ot_random.h new file mode 100644 index 0000000..c2d0387 --- /dev/null +++ b/src/ot_random.h @@ -0,0 +1,17 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_RANDOM_H__ +#define OT_RANDOM_H__ + +#include +#include + +/* Seed whatever generator the build settled on. A no-op when backed by + arc4random, which seeds itself from the kernel. */ +void random_init(void); + +uint32_t random_uint32(void); +void random_bytes(void *buf, size_t len); + +#endif diff --git a/src/ot_rijndael.c b/src/ot_rijndael.c new file mode 100644 index 0000000..3f36bde --- /dev/null +++ b/src/ot_rijndael.c @@ -0,0 +1,488 @@ +/** + * rijndael-alg-fst.c + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen + * @author Antoon Bosselaers + * @author Paulo Barreto + * + * This code is hereby placed in the public domain. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "ot_rijndael.h" + +static const uint32_t Te0[256] = { + 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, + 0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, + 0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU, + 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU, + 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U, + 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU, + 0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU, + 0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU, + 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU, + 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU, + 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U, + 0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU, + 0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU, + 0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U, + 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU, + 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU, + 0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU, + 0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU, + 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU, + 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U, + 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU, + 0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU, + 0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU, + 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU, + 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U, + 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U, + 0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U, + 0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U, + 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU, + 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U, + 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U, + 0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU, + 0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU, + 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U, + 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U, + 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U, + 0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU, + 0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U, + 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU, + 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U, + 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU, + 0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U, + 0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U, + 0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU, + 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U, + 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U, + 0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U, + 0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U, + 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U, + 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U, + 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U, + 0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U, + 0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU, + 0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U, + 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U, + 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U, + 0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U, + 0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U, + 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U, + 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU, + 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U, + 0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U, + 0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U, + 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU, +}; +static const uint32_t Te1[256] = { + 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU, + 0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U, + 0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU, + 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U, + 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU, + 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U, + 0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU, + 0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U, + 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U, + 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU, + 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U, + 0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U, + 0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U, + 0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU, + 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U, + 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U, + 0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU, + 0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U, + 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U, + 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U, + 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU, + 0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU, + 0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U, + 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU, + 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU, + 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U, + 0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU, + 0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U, + 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU, + 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U, + 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U, + 0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U, + 0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU, + 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U, + 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU, + 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U, + 0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU, + 0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U, + 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U, + 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU, + 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU, + 0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU, + 0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U, + 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U, + 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU, + 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U, + 0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU, + 0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U, + 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU, + 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U, + 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU, + 0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU, + 0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U, + 0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU, + 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U, + 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU, + 0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U, + 0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U, + 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U, + 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU, + 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU, + 0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U, + 0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU, + 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U, +}; +static const uint32_t Te2[256] = { + 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU, + 0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U, + 0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU, + 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U, + 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU, + 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U, + 0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU, + 0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U, + 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U, + 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU, + 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U, + 0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U, + 0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U, + 0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU, + 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U, + 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U, + 0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU, + 0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U, + 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U, + 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U, + 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU, + 0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU, + 0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U, + 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU, + 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU, + 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U, + 0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU, + 0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U, + 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU, + 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U, + 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U, + 0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U, + 0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU, + 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U, + 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU, + 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U, + 0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU, + 0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U, + 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U, + 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU, + 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU, + 0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU, + 0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U, + 0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U, + 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU, + 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U, + 0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU, + 0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U, + 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU, + 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U, + 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU, + 0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU, + 0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U, + 0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU, + 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U, + 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU, + 0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U, + 0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U, + 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U, + 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU, + 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU, + 0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U, + 0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU, + 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U, +}; +static const uint32_t Te3[256] = { + + 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U, + 0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U, + 0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U, + 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU, + 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU, + 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU, + 0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U, + 0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU, + 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU, + 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U, + 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U, + 0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU, + 0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU, + 0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU, + 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU, + 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU, + 0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U, + 0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU, + 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU, + 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U, + 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U, + 0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U, + 0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U, + 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U, + 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU, + 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U, + 0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU, + 0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU, + 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U, + 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U, + 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U, + 0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU, + 0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U, + 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU, + 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU, + 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U, + 0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U, + 0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU, + 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U, + 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU, + 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U, + 0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U, + 0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U, + 0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U, + 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU, + 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U, + 0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU, + 0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U, + 0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU, + 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U, + 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU, + 0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU, + 0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU, + 0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU, + 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U, + 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U, + 0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U, + 0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U, + 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U, + 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U, + 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU, + 0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, + 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU, + 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU, +}; +static const uint32_t Te4[256] = { + 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, + 0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, + 0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU, + 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U, + 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, + 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, + 0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, + 0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U, + 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U, + 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, + 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, + 0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U, + 0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U, + 0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU, + 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, + 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, + 0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, + 0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U, + 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U, + 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, + 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, + 0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, + 0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U, + 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU, + 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, + 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, + 0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, + 0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U, + 0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU, + 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, + 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, + 0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, + 0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU, + 0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U, + 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, + 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, + 0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, + 0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U, + 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U, + 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, + 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, + 0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU, + 0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U, + 0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U, + 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, + 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, + 0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, + 0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U, + 0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU, + 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, + 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, + 0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, + 0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U, + 0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU, + 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, + 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, + 0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U, + 0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U, + 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U, + 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, + 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, + 0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U, + 0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU, + 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U, +}; +static const uint32_t rcon[] = { + 0x01000000, 0x02000000, 0x04000000, 0x08000000, + 0x10000000, 0x20000000, 0x40000000, 0x80000000, + 0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */ +}; + +#define GETU32(pt) (((uint32_t)(pt)[0] << 24) ^ ((uint32_t)(pt)[1] << 16) ^ ((uint32_t)(pt)[2] << 8) ^ ((uint32_t)(pt)[3])) +#define PUTU32(ct, st) { (ct)[0] = (uint8_t)((st) >> 24); (ct)[1] = (uint8_t)((st) >> 16); (ct)[2] = (uint8_t)((st) >> 8); (ct)[3] = (uint8_t)(st); } + +/** + * Expand the cipher key into the encryption key schedule. + * + * @return the number of rounds for the given cipher key size. + */ +int rijndaelKeySetupEnc128(uint32_t rk[44], const uint8_t cipherKey[] ) { + int i = 0; + uint32_t temp; + + rk[0] = GETU32(cipherKey ); + rk[1] = GETU32(cipherKey + 4); + rk[2] = GETU32(cipherKey + 8); + rk[3] = GETU32(cipherKey + 12); + for (;;) { + temp = rk[3]; + rk[4] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[(temp ) & 0xff] & 0x0000ff00) ^ + (Te4[(temp >> 24) ] & 0x000000ff) ^ + rcon[i]; + rk[5] = rk[1] ^ rk[4]; + rk[6] = rk[2] ^ rk[5]; + rk[7] = rk[3] ^ rk[6]; + if (++i == 10) { + return 10; + } + rk += 4; + } +} + +void rijndaelEncrypt128(const uint32_t rk[44], const uint8_t pt[16], uint8_t ct[16]) { + uint32_t s0, s1, s2, s3, t0, t1, t2, t3; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = GETU32(pt ) ^ rk[0]; + s1 = GETU32(pt + 4) ^ rk[1]; + s2 = GETU32(pt + 8) ^ rk[2]; + s3 = GETU32(pt + 12) ^ rk[3]; + /* round 1: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[ 4]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[ 5]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[ 6]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[ 7]; + /* round 2: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[ 8]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[ 9]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[10]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[11]; + /* round 3: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[12]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[13]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[14]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[15]; + /* round 4: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[16]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[17]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[18]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[19]; + /* round 5: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[20]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[21]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[22]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[23]; + /* round 6: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[24]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[25]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[26]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[27]; + /* round 7: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[28]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[29]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[30]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[31]; + /* round 8: */ + s0 = Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] ^ Te3[t3 & 0xff] ^ rk[32]; + s1 = Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] ^ Te3[t0 & 0xff] ^ rk[33]; + s2 = Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] ^ Te3[t1 & 0xff] ^ rk[34]; + s3 = Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] ^ Te3[t2 & 0xff] ^ rk[35]; + /* round 9: */ + t0 = Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] ^ Te3[s3 & 0xff] ^ rk[36]; + t1 = Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] ^ Te3[s0 & 0xff] ^ rk[37]; + t2 = Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] ^ Te3[s1 & 0xff] ^ rk[38]; + t3 = Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] ^ Te3[s2 & 0xff] ^ rk[39]; + + /* + * apply last round and + * map cipher state to byte array block: + */ + s0 = + (Te4[(t0 >> 24) ] & 0xff000000) ^ + (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t3 ) & 0xff] & 0x000000ff) ^ + rk[40]; + PUTU32(ct , s0); + s1 = + (Te4[(t1 >> 24) ] & 0xff000000) ^ + (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t0 ) & 0xff] & 0x000000ff) ^ + rk[41]; + PUTU32(ct + 4, s1); + s2 = + (Te4[(t2 >> 24) ] & 0xff000000) ^ + (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t1 ) & 0xff] & 0x000000ff) ^ + rk[42]; + PUTU32(ct + 8, s2); + s3 = + (Te4[(t3 >> 24) ] & 0xff000000) ^ + (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[(t2 ) & 0xff] & 0x000000ff) ^ + rk[43]; + PUTU32(ct + 12, s3); +} diff --git a/src/ot_rijndael.h b/src/ot_rijndael.h new file mode 100644 index 0000000..1235215 --- /dev/null +++ b/src/ot_rijndael.h @@ -0,0 +1,20 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. + + The rijndael implementation was taken from + + http://www.cs.ucdavis.edu/~rogaway/ocb/ocb-ref/rijndael-alg-fst.c + + and modified to work with 128 bits (this is 10 rounds) only. */ + +#ifndef OT_RIJNDAEL_H__ +#define OT_RIJNDAEL_H__ + +#include + +int rijndaelKeySetupEnc128(uint32_t rk[44], const uint8_t cipherKey[] ); +void rijndaelEncrypt128(const uint32_t rk[44], const uint8_t pt[16], uint8_t ct[16]); + +extern const char *g_version_rijndael_c; + +#endif diff --git a/src/ot_scan.c b/src/ot_scan.c new file mode 100644 index 0000000..3b8e87c --- /dev/null +++ b/src/ot_scan.c @@ -0,0 +1,65 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include + +/* Opentracker */ +#include "ot_scan.h" + +size_t scan_uint(const char *src, uint64_t *dest, uint64_t max) { + unsigned long long value; + char *end; + + if (*src < '0' || *src > '9') + return 0; + + errno = 0; + value = strtoull(src, &end, 10); + + if (errno == ERANGE || value > (unsigned long long)max) + return 0; + + *dest = (uint64_t)value; + return (size_t)(end - src); +} + +size_t scan_uint_bencode(const char *src, uint64_t *dest, uint64_t max) { + uint64_t value; + size_t consumed = scan_uint(src, &value, max); + + /* bencode writes no leading zeros, so anything longer than a single digit + must not begin with one. */ + if (consumed > 1 && *src == '0') + return 0; + + if (consumed) + *dest = value; + + return consumed; +} + +int scan_int(const char *src, size_t len, int64_t *dest) { + int64_t value = 0; + int negate = 0; + + if (len && *src == '-') + --len, ++src, negate = 1; + if (!len) + return -1; + + while (len--) { + if (*src < '0' || *src > '9') + return -1; + /* A twenty character field of nines does not fit, and silently wrapping + would magically heal a nonsense announce */ + if (value > (INT64_MAX - (*src - '0')) / 10) + return -1; + value = 10 * value + (*src++ - '0'); + } + + *dest = negate ? -value : value; + return 0; +} diff --git a/src/ot_scan.h b/src/ot_scan.h new file mode 100644 index 0000000..a79ec13 --- /dev/null +++ b/src/ot_scan.h @@ -0,0 +1,25 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_SCAN_H__ +#define OT_SCAN_H__ + +#include +#include +#include + +/* Read an unsigned decimal number from src into dest and return how many + characters were consumed. Returns zero and leaves dest untouched when src + does not begin with a digit, or when the value would exceed max. */ +size_t scan_uint(const char *src, uint64_t *dest, uint64_t max); + +/* As scan_uint, but following the bencode integer grammar, which permits a + leading zero on zero alone. */ +size_t scan_uint_bencode(const char *src, uint64_t *dest, uint64_t max); + +/* Read a signed decimal number that has to take up exactly len characters, + which is how a value lifted out of a query string arrives. Returns 0, or + -1 leaving dest untouched. */ +int scan_int(const char *src, size_t len, int64_t *dest); + +#endif diff --git a/src/ot_socket.c b/src/ot_socket.c new file mode 100644 index 0000000..79bb01a --- /dev/null +++ b/src/ot_socket.c @@ -0,0 +1,310 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#define _GNU_SOURCE /* accept4 on glibc */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_ip.h" +#include "ot_socket.h" + +static void sockaddr_from_ip(struct sockaddr_in6 *sa, const ot_ip6 ip, uint16_t port) { + memset(sa, 0, sizeof(*sa)); + sa->sin6_family = AF_INET6; + sa->sin6_port = htons(port); + memcpy(&sa->sin6_addr, ip, sizeof(ot_ip6)); +} + +/* Linux reports an IPv4 peer on a dual stack socket as a v4 mapped + sockaddr_in6, the BSDs and macOS hand back a plain sockaddr_in instead, so + both forms have to be read. */ +static void ip_from_sockaddr(ot_ip6 ip, uint16_t *port, const struct sockaddr_storage *ss) { + if (ss->ss_family == AF_INET) { + const struct sockaddr_in *sa = (const struct sockaddr_in *)ss; + + memcpy(ip, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); + memcpy(ip + sizeof(ip_v4mapped_prefix), &sa->sin_addr, 4); + *port = ntohs(sa->sin_port); + } else { + const struct sockaddr_in6 *sa = (const struct sockaddr_in6 *)ss; + + memcpy(ip, &sa->sin6_addr, sizeof(ot_ip6)); + *port = ntohs(sa->sin6_port); + } +} + +int socket_set_blocking(int s, int blocking) { + int flags = fcntl(s, F_GETFL, 0); + + if (flags == -1) + return -1; + return fcntl(s, F_SETFL, blocking ? (flags & ~O_NONBLOCK) : (flags | O_NONBLOCK)); +} + +static int set_nonblocking(int s) { return socket_set_blocking(s, 0); } + +/* Close without letting the failure we are about to report be overwritten */ +static int close_and_fail(int s) { + int saved = errno; + close(s); + errno = saved; + return -1; +} + +static int socket_v6(int type) { + int s = socket(PF_INET6, type, 0); + int zero = 0; + + if (s == -1) + return -1; + + /* A tracker that answered only one family would be half a tracker */ + if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &zero, sizeof(zero)) == -1) + return close_and_fail(s); + + if (set_nonblocking(s) == -1) + return close_and_fail(s); + + return s; +} + +/* SO_REUSEPORT means three different things. Linux spreads datagrams over + the group. FreeBSD permits the bind and spreads nothing, which is what + SO_REUSEPORT_LB was added for. Darwin hands every datagram to whichever + socket bound last, so a group there starves all but one reader. */ +#if defined(SO_REUSEPORT_LB) +#define OT_BALANCE_OPTION SO_REUSEPORT_LB +#elif defined(SO_REUSEPORT) && !defined(__APPLE__) +#define OT_BALANCE_OPTION SO_REUSEPORT +#endif + +int socket_can_balance(void) { +#ifdef OT_BALANCE_OPTION + return 1; +#else + return 0; +#endif +} + +/* The only place in the tree that needs to know what a kernel calls this */ +static void socket_set_sharing(int s, socket_sharing sharing) { + int one = 1; + + setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); +#ifdef OT_BALANCE_OPTION + if (sharing == SOCKET_BALANCED) { + setsockopt(s, SOL_SOCKET, OT_BALANCE_OPTION, &one, sizeof(one)); + return; + } +#endif +#ifdef SO_REUSEPORT + if (sharing != SOCKET_EXCLUSIVE) + setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &one, sizeof(one)); +#endif +} + +static int socket_bind_v6(int s, const ot_ip6 ip, uint16_t port, socket_sharing sharing) { + struct sockaddr_in6 sa; + + socket_set_sharing(s, sharing); + + sockaddr_from_ip(&sa, ip, port); + return bind(s, (struct sockaddr *)&sa, sizeof(sa)); +} + +/* Defer the listening socket reporting it to be ready to accept until enough + sensible data is available for an accept+read pair. Where available. */ +int socket_defer_accept(int s) { +#if defined(TCP_DEFER_ACCEPT) + int patience = OT_CLIENT_TIMEOUT; + + return setsockopt(s, IPPROTO_TCP, TCP_DEFER_ACCEPT, &patience, sizeof(patience)); +#elif defined(SO_ACCEPTFILTER) + struct accept_filter_arg filter; + const char name[sizeof(filter.af_name)] = "dataready"; + + memset(&filter, 0, sizeof(filter)); + memcpy(filter.af_name, name, sizeof(name)); + return setsockopt(s, SOL_SOCKET, SO_ACCEPTFILTER, &filter, sizeof(filter)); +#else + (void)s; + return -1; +#endif +} + +int socket_tcp_listen(const ot_ip6 ip, uint16_t port, socket_sharing sharing) { + int s = socket_v6(SOCK_STREAM); + + if (s == -1) + return -1; + if (socket_bind_v6(s, ip, port, sharing) == -1 || listen(s, SOMAXCONN) == -1) + return close_and_fail(s); + + return s; +} + +int socket_udp_bind(const ot_ip6 ip, uint16_t port, socket_sharing sharing) { + int s = socket_v6(SOCK_DGRAM); + + if (s == -1) + return -1; + if (socket_bind_v6(s, ip, port, sharing) == -1) + return close_and_fail(s); + + return s; +} + +int socket_tcp_connect(const ot_ip6 srcip, uint16_t srcport, const ot_ip6 dstip, uint16_t dstport) { + struct sockaddr_in6 sa; + int s = socket_v6(SOCK_STREAM); + + if (s == -1) + return -1; + if (socket_bind_v6(s, srcip, srcport, SOCKET_EXCLUSIVE) == -1) + return close_and_fail(s); + + sockaddr_from_ip(&sa, dstip, dstport); + if (connect(s, (struct sockaddr *)&sa, sizeof(sa)) == -1 && errno != EINPROGRESS && errno != EWOULDBLOCK) + return close_and_fail(s); + + return s; +} + +int socket_is_connected(int s) { + int error = 0; + socklen_t len = sizeof(error); + + if (getsockopt(s, SOL_SOCKET, SO_ERROR, &error, &len) == -1) + return 0; + if (error) { + errno = error; + return 0; + } + return 1; +} + +/* Linux hands over a blocking socket whatever the listening socket was, the + BSDs and macOS inherit the listener's flags, so the accepted socket has to + be told either way. accept4 carries the flag itself, which saves the two + fcntl calls it otherwise takes. Darwin has neither accept4 nor the flag. */ +int socket_accept(int s, ot_ip6 ip, uint16_t *port) { + struct sockaddr_storage ss; + socklen_t len = sizeof(ss); + int fd; + + memset(&ss, 0, sizeof(ss)); +#ifdef SOCK_NONBLOCK + if ((fd = accept4(s, (struct sockaddr *)&ss, &len, SOCK_NONBLOCK)) == -1) + return -1; +#else + if ((fd = accept(s, (struct sockaddr *)&ss, &len)) == -1) + return -1; + if (set_nonblocking(fd) == -1) + return close_and_fail(fd); +#endif + + ip_from_sockaddr(ip, port, &ss); + return fd; +} + +ssize_t socket_recv(int s, void *buf, size_t len, ot_ip6 ip, uint16_t *port) { + struct sockaddr_storage ss; + socklen_t sslen = sizeof(ss); + ssize_t received; + + memset(&ss, 0, sizeof(ss)); + received = recvfrom(s, buf, len, 0, (struct sockaddr *)&ss, &sslen); + if (received == -1) + return -1; + + ip_from_sockaddr(ip, port, &ss); + return received; +} + +ssize_t socket_send(int s, const void *buf, size_t len, const ot_ip6 ip, uint16_t port) { + struct sockaddr_in6 sa; + + sockaddr_from_ip(&sa, ip, port); + return sendto(s, buf, len, 0, (struct sockaddr *)&sa, sizeof(sa)); +} + +#ifdef WANT_SYNC_LIVE +static void sockaddr4_from_ip(struct sockaddr_in *sa, const ot_ip6 ip, uint16_t port) { + memset(sa, 0, sizeof(*sa)); + sa->sin_family = AF_INET; + sa->sin_port = htons(port); + memcpy(&sa->sin_addr, ip + sizeof(ip_v4mapped_prefix), 4); +} + +/* A multicast group is meant to have several listeners, so these are always + shared */ +static int socket_v4_bind(const ot_ip6 ip, uint16_t port) { + struct sockaddr_in sa; + int s; + + if ((s = socket(PF_INET, SOCK_DGRAM, 0)) == -1) + return -1; + + socket_set_sharing(s, SOCKET_SHARED); + + sockaddr4_from_ip(&sa, ip, port); + if (bind(s, (struct sockaddr *)&sa, sizeof(sa)) == -1) + return close_and_fail(s); + + return s; +} + +int socket_mcast_join(const ot_ip6 group, const ot_ip6 interface, uint16_t port) { + struct ip_mreq mreq; + ot_ip6 any; + int s; + + /* The group is what is listened to, so the socket takes the wildcard and + the join names the interface to receive it on */ + memcpy(any, ip_v4mapped_prefix, sizeof(ip_v4mapped_prefix)); + memset(any + sizeof(ip_v4mapped_prefix), 0, 4); + + if ((s = socket_v4_bind(any, port)) == -1) + return -1; + + memset(&mreq, 0, sizeof(mreq)); + memcpy(&mreq.imr_multiaddr, group + sizeof(ip_v4mapped_prefix), 4); + memcpy(&mreq.imr_interface, interface + sizeof(ip_v4mapped_prefix), 4); + if (setsockopt(s, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) == -1) + return close_and_fail(s); + + return s; +} + +int socket_mcast_bind(const ot_ip6 interface, uint16_t port, int loopback) { + /* The BSDs and macOS want a byte for the IPv4 multicast options, Linux + accepts either that or an int. */ + unsigned char ttl = 1, loop = (unsigned char)(loopback != 0); + int s = socket_v4_bind(interface, port); + + if (s == -1) + return -1; + if (set_nonblocking(s) == -1) + return close_and_fail(s); + + if (setsockopt(s, IPPROTO_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)) == -1 || + setsockopt(s, IPPROTO_IP, IP_MULTICAST_LOOP, &loop, sizeof(loop)) == -1) + return close_and_fail(s); + + return s; +} + +ssize_t socket_mcast_send(int s, const void *buf, size_t len, const ot_ip6 group, uint16_t port) { + struct sockaddr_in sa; + + sockaddr4_from_ip(&sa, group, port); + return sendto(s, buf, len, 0, (struct sockaddr *)&sa, sizeof(sa)); +} +#endif diff --git a/src/ot_socket.h b/src/ot_socket.h new file mode 100644 index 0000000..5c1d840 --- /dev/null +++ b/src/ot_socket.h @@ -0,0 +1,56 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_SOCKET_H__ +#define OT_SOCKET_H__ + +#include +#include + +#include "trackerlogic.h" + +/* Every socket opentracker opens is an AF_INET6 socket with IPV6_V6ONLY + cleared, so one descriptor serves both families and an IPv4 peer arrives + as the v4 mapped address */ + +/* How an address may be shared with other sockets. Which socket option says + so differs between kernels, which is socket_set_sharing's business. + + SHARED lets another socket bind the same address. BALANCED asks the kernel + to spread arriving datagrams over the sockets that did, which is a + different thing and not everywhere available: ask socket_can_balance + before binding a group that depends on it. */ +typedef enum { SOCKET_EXCLUSIVE, SOCKET_SHARED, SOCKET_BALANCED } socket_sharing; + +/* Whether this kernel spreads datagrams over a group of sockets bound to one + address. Where it does not, binding a group is worse than one socket with + several readers, so the caller wants to know first. */ +int socket_can_balance(void); + +/* Create, bind and, for TCP, listen. The returned socket is non blocking + Returns -1 on failure with errno set */ +int socket_tcp_listen(const ot_ip6 ip, uint16_t port, socket_sharing sharing); +int socket_udp_bind(const ot_ip6 ip, uint16_t port, socket_sharing sharing); + +int socket_tcp_connect(const ot_ip6 srcip, uint16_t srcport, const ot_ip6 dstip, uint16_t dstport); +int socket_is_connected(int s); +int socket_accept(int s, ot_ip6 ip, uint16_t *port); +int socket_set_blocking(int s, int blocking); +int socket_defer_accept(int s); + +ssize_t socket_recv(int s, void *buf, size_t len, ot_ip6 ip, uint16_t *port); +ssize_t socket_send(int s, const void *buf, size_t len, const ot_ip6 ip, uint16_t port); + +#ifdef WANT_SYNC_LIVE +/* + socket_mcast_join takes the wildcard address and names the interface in the + join, and hands back a blocking socket, since the thread that reads it does + nothing else. socket_mcast_bind hands back a non blocking one, because the + thread that sends from it is also serving clients. Incoming datagrams are + read with socket_recv */ +int socket_mcast_join(const ot_ip6 group, const ot_ip6 interface, uint16_t port); +int socket_mcast_bind(const ot_ip6 interface, uint16_t port, int loopback); +ssize_t socket_mcast_send(int s, const void *buf, size_t len, const ot_ip6 group, uint16_t port); +#endif + +#endif diff --git a/src/ot_stats.c b/src/ot_stats.c new file mode 100644 index 0000000..2d6335b --- /dev/null +++ b/src/ot_stats.c @@ -0,0 +1,866 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef WANT_SYSLOGS +#include +#endif + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_event.h" +#include "ot_hex.h" +#include "ot_iovec.h" +#include "ot_ip.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_fmt.h" +#include "ot_stats.h" +#include "trackerlogic.h" + +#ifndef NO_FULLSCRAPE_LOGGING +#define LOG_TO_STDERR(...) fprintf(stderr, __VA_ARGS__) +#else +#define LOG_TO_STDERR(...) +#endif + +/* Forward declaration */ +static void stats_make(size_t *iovec_entries, struct iovec **iovector, ot_tasktype mode); +#define OT_STATS_TMPSIZE 8192 + +/* Each serving thread keeps its own statistics tally. The list is linked + through intrusive next pointers within the workstruct. + + Registration happens once per thread and the adding up once per statistics + request, so this lock is never on anybody's hot path. */ +static struct ot_workstruct *g_counter_contexts; +static ot_counter g_retired[OT_COUNTER_COUNT]; +static pthread_mutex_t g_counter_mutex = PTHREAD_MUTEX_INITIALIZER; + +void stats_counters_register(struct ot_workstruct *ws) { + pthread_mutex_lock(&g_counter_mutex); + ws->counters_next = g_counter_contexts; + g_counter_contexts = ws; + pthread_mutex_unlock(&g_counter_mutex); +} + +void stats_counters_retire(struct ot_workstruct *ws) { + struct ot_workstruct **link; + size_t i; + + pthread_mutex_lock(&g_counter_mutex); + for (link = &g_counter_contexts; *link; link = &(*link)->counters_next) + if (*link == ws) { + *link = ws->counters_next; + break; + } + + /* When retiring a worker thread, collect the counter tally in the global */ + for (i = 0; i < OT_COUNTER_COUNT; ++i) + atomic_fetch_add_explicit(g_retired + i, atomic_load_explicit(ws->counters + i, memory_order_relaxed), memory_order_relaxed); + pthread_mutex_unlock(&g_counter_mutex); +} + +/* What every thread has counted so far, added together */ +static void counters_total(unsigned long long *total) { + struct ot_workstruct *ws; + size_t i; + + pthread_mutex_lock(&g_counter_mutex); + for (i = 0; i < OT_COUNTER_COUNT; ++i) + total[i] = atomic_load_explicit(g_retired + i, memory_order_relaxed); + for (ws = g_counter_contexts; ws; ws = ws->counters_next) + for (i = 0; i < OT_COUNTER_COUNT; ++i) + total[i] += atomic_load_explicit(ws->counters + i, memory_order_relaxed); + pthread_mutex_unlock(&g_counter_mutex); +} + +#define COUNT(index, by) atomic_fetch_add_explicit(ws->counters + (index), (unsigned long long)(by), memory_order_relaxed) + +static const char *ot_failed_request_names[] = { + "302 Redirect", "400 Parse Error", "400 Invalid Parameter", "400 Invalid Parameter (compact=0)", "400 Not Modest", + "402 Payment Required", "403 Access Denied", "404 Not found", "500 Internal Server Error"}; + +static time_t ot_start_time; + +#define STATS_NETWORK_NODE_BITWIDTH 4 +#define STATS_NETWORK_NODE_COUNT (1 << STATS_NETWORK_NODE_BITWIDTH) + +#define __BYTE(P, D) (((uint8_t *)P)[D / 8]) +#define __MSK (STATS_NETWORK_NODE_COUNT - 1) +#define __SHFT(D) ((D ^ STATS_NETWORK_NODE_BITWIDTH) & STATS_NETWORK_NODE_BITWIDTH) + +#define __LDR(P, D) ((__BYTE((P), (D)) >> __SHFT((D))) & __MSK) +#define __STR(P, D, V) __BYTE((P), (D)) = (__BYTE((P), (D)) & ~(__MSK << __SHFT((D)))) | ((V) << __SHFT((D))) + +#if 0 +// XXX +#define STATS_NETWORK_NODE_MAXDEPTH (68 - STATS_NETWORK_NODE_BITWIDTH) +#define STATS_NETWORK_NODE_LIMIT (48 - STATS_NETWORK_NODE_BITWIDTH) +#endif +#define STATS_NETWORK_NODE_MAXDEPTH (28 - STATS_NETWORK_NODE_BITWIDTH) +#define STATS_NETWORK_NODE_LIMIT (24 - STATS_NETWORK_NODE_BITWIDTH) + +typedef union stats_network_node stats_network_node; +union stats_network_node { + size_t counters[STATS_NETWORK_NODE_COUNT]; + stats_network_node *children[STATS_NETWORK_NODE_COUNT]; +}; + +static int stat_increase_network_count(stats_network_node **pnode, int depth, uintptr_t ip) { + int foo = __LDR(ip, depth); + stats_network_node *node; + + if (!*pnode) { + *pnode = malloc(sizeof(stats_network_node)); + if (!*pnode) + return -1; + memset(*pnode, 0, sizeof(stats_network_node)); + } + node = *pnode; + + if (depth < STATS_NETWORK_NODE_MAXDEPTH) + return stat_increase_network_count(node->children + foo, depth + STATS_NETWORK_NODE_BITWIDTH, ip); + + node->counters[foo]++; + return 0; +} + +static int stats_shift_down_network_count(stats_network_node **node, int depth, int shift) { + size_t i, rest = 0; + + if (!*node) + return 0; + + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) + if (depth < STATS_NETWORK_NODE_MAXDEPTH) + rest += stats_shift_down_network_count((*node)->children + i, depth + STATS_NETWORK_NODE_BITWIDTH, shift); + else + rest += (*node)->counters[i] >>= shift; + + if (!rest) { + free(*node); + *node = NULL; + } + + return rest; +} + +static size_t stats_get_highscore_networks(stats_network_node *node, int depth, ot_ip6 node_value, size_t *scores, ot_ip6 *networks, int network_count, + int limit) { + size_t i, score = 0; + + if (!node) + return 0; + + if (depth < limit) { + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) + if (node->children[i]) { + __STR(node_value, depth, i); + score += stats_get_highscore_networks(node->children[i], depth + STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit); + } + return score; + } + + if (depth > limit && depth < STATS_NETWORK_NODE_MAXDEPTH) { + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) + if (node->children[i]) + score += stats_get_highscore_networks(node->children[i], depth + STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit); + return score; + } + + if (depth > limit && depth == STATS_NETWORK_NODE_MAXDEPTH) { + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) + score += node->counters[i]; + return score; + } + + /* if( depth == limit ) */ + for (i = 0; i < STATS_NETWORK_NODE_COUNT; ++i) { + int j = 1; + size_t node_score; + + if (depth == STATS_NETWORK_NODE_MAXDEPTH) + node_score = node->counters[i]; + else + node_score = stats_get_highscore_networks(node->children[i], depth + STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit); + + score += node_score; + + if (node_score <= scores[0]) + continue; + + __STR(node_value, depth, i); + while (j < network_count && node_score > scores[j]) + ++j; + --j; + + memmove(scores, scores + 1, j * sizeof(*scores)); + memmove(networks, networks + 1, j * sizeof(*networks)); + scores[j] = node_score; + memcpy(networks + j, node_value, sizeof(*networks)); + } + + return score; +} + +static void stats_return_busy_networks(char **reply, const char *reply_end, stats_network_node *tree, size_t amount, int limit) { + ot_ip6 networks[amount]; + ot_ip6 node_value; + size_t scores[amount]; + size_t i; + + memset(scores, 0, sizeof(scores)); + memset(networks, 0, sizeof(networks)); + memset(node_value, 0, sizeof(node_value)); + + stats_get_highscore_networks(tree, 0, node_value, scores, networks, amount, limit); + + fmt_append(reply, reply_end, "Networks, limit /%d:\n", limit + STATS_NETWORK_NODE_BITWIDTH); + for (i = amount; i > 0; --i) { + if (scores[i - 1]) { + char ip_readable[64]; + + ip_readable[ip_format(ip_readable, networks[i - 1])] = 0; + fmt_append(reply, reply_end, "%08zd: %s\n", scores[i - 1], ip_readable); + } + } + fmt_append(reply, reply_end, "\n"); +} + +static void stats_slash24s_txt(char **reply, const char *reply_end, size_t amount) { + stats_network_node *slash24s_network_counters_root = NULL; + size_t i, bucket, peer_size = OT_PEER_SIZE4; + + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + ot_vector *torrents_list = mutex_bucket_lock(bucket); + for (i = 0; i < torrents_list->size; ++i) { + ot_peerlist *peer_list = (((ot_torrent *)(torrents_list->data))[i]).peer_list4; + ot_vector *bucket_list = &peer_list->peers; + int num_buckets = 1; + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets = bucket_list->size; + bucket_list = (ot_vector *)bucket_list->data; + } + + while (num_buckets--) { + ot_peer *peers = (ot_peer *)bucket_list->data; + size_t numpeers = bucket_list->size; + while (numpeers--) { + if (stat_increase_network_count(&slash24s_network_counters_root, 0, (uintptr_t)(peers))) + goto bailout_unlock; + peers += peer_size; + } + ++bucket_list; + } + } + mutex_bucket_unlock(bucket, 0); + if (!g_opentracker_running) + goto done; + } + + /* The tree is built. Now analyze */ + stats_return_busy_networks(reply, reply_end, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_MAXDEPTH); + stats_return_busy_networks(reply, reply_end, slash24s_network_counters_root, amount, STATS_NETWORK_NODE_LIMIT); + goto done; + +bailout_unlock: + mutex_bucket_unlock(bucket, 0); +done: + stats_shift_down_network_count(&slash24s_network_counters_root, 0, sizeof(int) * 8 - 1); +} + +#ifdef WANT_SPOT_WOODPECKER +static stats_network_node *stats_woodpeckers_tree; +static pthread_mutex_t g_woodpeckers_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void stats_return_woodpeckers(char **reply, const char *reply_end, int amount) { + pthread_mutex_lock(&g_woodpeckers_mutex); + stats_return_busy_networks(reply, reply_end, stats_woodpeckers_tree, amount, STATS_NETWORK_NODE_MAXDEPTH); + pthread_mutex_unlock(&g_woodpeckers_mutex); +} +#endif + +typedef struct { + unsigned long long torrent_count; + unsigned long long peer_count; + unsigned long long seed_count; +} torrent_stats; + +static int torrent_statter(ot_torrent *torrent, uintptr_t data) { + torrent_stats *stats = (torrent_stats *)data; + stats->torrent_count++; + stats->peer_count += torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + stats->seed_count += torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + return 0; +} + +/* Converter function from memory to human readable hex strings */ +typedef struct { + size_t val; + ot_hash hash; +} ot_record; + +static inline void stats_insert_into_highscore(size_t val, ot_record* list, size_t amount, ot_hash* hash) { + size_t i; + for (i = 0; (i < amount) && (list[i].val > val); ++i) + {} + /* if we score better than a record in the list, we need to make room in the high + score array by moving lower scoring entries up */ + if (i < amount) { + memmove(list + i + 1, list + i, (amount - i - 1) * sizeof(ot_record)); + + memcpy(&list[i].hash, hash, sizeof(ot_hash)); + list[i].val = val; + } +} + + +/* Fetches stats from tracker */ +void stats_top_txt(char **reply, const char *reply_end, size_t amount) { + size_t j, idx, bucket; + ot_record top100s[100], top100c[100], top100l[100]; + char hex_out[42]; + + if (amount > 100) + amount = 100; + + memset(top100s, 0, sizeof(top100s)); + memset(top100c, 0, sizeof(top100c)); + memset(top100l, 0, sizeof(top100l)); + + /* Iterate over complete torrent list */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + ot_vector *torrents_list = mutex_bucket_lock(bucket); + for (j = 0; j < torrents_list->size; ++j) { + ot_torrent *torrent = (ot_torrent *)(torrents_list->data) + j; + size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + size_t leech_count = peer_count - seed_count; + + stats_insert_into_highscore(peer_count, top100c, amount, &torrent->hash); + stats_insert_into_highscore(seed_count, top100s, amount, &torrent->hash); + stats_insert_into_highscore(leech_count, top100l, amount, &torrent->hash); + } + mutex_bucket_unlock(bucket, 0); + if (!g_opentracker_running) + return; + } + + fmt_append(reply, reply_end, "Top %zd torrents by peers:\n", amount); + for (idx = 0; idx < amount; ++idx) + if (top100c[idx].val) + fmt_append(reply, reply_end, "\t%zd\t%s\n", top100c[idx].val, hex_from_hash(hex_out, top100c[idx].hash)); + fmt_append(reply, reply_end, "Top %zd torrents by seeds:\n", amount); + for (idx = 0; idx < amount; ++idx) + if (top100s[idx].val) + fmt_append(reply, reply_end, "\t%zd\t%s\n", top100s[idx].val, hex_from_hash(hex_out, top100s[idx].hash)); + fmt_append(reply, reply_end, "Top %zd torrents by leechers:\n", amount); + for (idx = 0; idx < amount; ++idx) + if (top100l[idx].val) + fmt_append(reply, reply_end, "\t%zd\t%s\n", top100l[idx].val, hex_from_hash(hex_out, top100l[idx].hash)); +} + +static unsigned long events_per_time(unsigned long long events, time_t t) { return events / ((unsigned int)t ? (unsigned int)t : 1); } + +static void stats_connections_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", + total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], + total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], (int)t, (int)(t / 3600), + events_per_time(total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], t), + events_per_time(total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t)); +} + +static void stats_udpconnections_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker udp4 stats, %lu conns/s :: %lu success/s.", total[OT_COUNTER_UDP_CONNECTIONS], + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], (int)t, (int)(t / 3600), + events_per_time(total[OT_COUNTER_UDP_CONNECTIONS], t), + events_per_time(total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t)); +} + +static void stats_tcpconnections_mrtg(char **reply, const char *reply_end) { + time_t t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker tcp4 stats, %lu conns/s :: %lu success/s.", total[OT_COUNTER_TCP_CONNECTIONS], + total[OT_COUNTER_TCP_ANNOUNCES], (int)t, (int)(t / 3600), events_per_time(total[OT_COUNTER_TCP_CONNECTIONS], t), + events_per_time(total[OT_COUNTER_TCP_ANNOUNCES], t)); +} + +static void stats_scrape_mrtg(char **reply, const char *reply_end) { + time_t t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker scrape stats, %lu scrape/s (tcp and udp)", + total[OT_COUNTER_TCP_SCRAPES], total[OT_COUNTER_UDP_SCRAPES], (int)t, (int)(t / 3600), + events_per_time((total[OT_COUNTER_TCP_SCRAPES] + total[OT_COUNTER_UDP_SCRAPES]), t)); +} + +static void stats_fullscrapes_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker full scrape stats, %lu conns/s :: %lu bytes/s.", + total[OT_COUNTER_FULLSCRAPES] * 1000, total[OT_COUNTER_FULLSCRAPE_BYTES], (int)t, (int)(t / 3600), + events_per_time(total[OT_COUNTER_FULLSCRAPES], t), + events_per_time(total[OT_COUNTER_FULLSCRAPE_BYTES], t)); +} + +static void stats_peers_mrtg(char **reply, const char *reply_end) { + torrent_stats stats = {0, 0, 0}; + + iterate_all_torrents(torrent_statter, (uintptr_t)&stats); + + fmt_append(reply, reply_end, "%llu\n%llu\nopentracker serving %llu torrents\nopentracker", stats.peer_count, stats.seed_count, + stats.torrent_count); +} + +static void stats_torrents_mrtg(char **reply, const char *reply_end) { + size_t torrent_count = mutex_get_torrent_count(); + + fmt_append(reply, reply_end, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker", torrent_count, (size_t)0, torrent_count); +} + +static void stats_httperrors_txt(char **reply, const char *reply_end) { + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "302 RED %llu\n400 ... %llu\n400 PAR %llu\n400 COM %llu\n403 IP %llu\n404 INV %llu\n500 SRV %llu\n", + total[OT_COUNTER_FAILED + 0], total[OT_COUNTER_FAILED + 1], total[OT_COUNTER_FAILED + 2], total[OT_COUNTER_FAILED + 3], + total[OT_COUNTER_FAILED + 4], total[OT_COUNTER_FAILED + 5], total[OT_COUNTER_FAILED + 6]); +} + +static void stats_return_renew_bucket(char **reply, const char *reply_end) { + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + int i; + + for (i = 0; i < OT_PEER_TIMEOUT; ++i) + fmt_append(reply, reply_end, "%02i %llu\n", i, total[OT_COUNTER_RENEWED + i]); +} + +static void stats_return_sync_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker connections, %lu conns/s :: %lu success/s.", total[OT_COUNTER_SYNCS], 0LL, + (int)t, (int)(t / 3600), events_per_time(total[OT_COUNTER_TCP_CONNECTIONS] + total[OT_COUNTER_UDP_CONNECTIONS], t), + events_per_time(total[OT_COUNTER_TCP_ANNOUNCES] + total[OT_COUNTER_UDP_ANNOUNCES] + total[OT_COUNTER_UDP_CONNECTS], t)); +} + +static void stats_return_completed_mrtg(char **reply, const char *reply_end) { + ot_time t = time(NULL) - ot_start_time; + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + + fmt_append(reply, reply_end, "%llu\n%llu\n%i seconds (%i hours)\nopentracker, %lu completed/h.", total[OT_COUNTER_COMPLETED], 0LL, (int)t, (int)(t / 3600), + events_per_time(total[OT_COUNTER_COMPLETED], t / 3600)); +} + +#ifdef WANT_LOG_NUMWANT +extern unsigned long long numwants[201]; +static void stats_return_numwants(char **reply, const char *reply_end) { + int i; + for (i = 0; i <= 200; ++i) + fmt_append(reply, reply_end, "%03d => %lld\n", i, numwants[i]); +} +#endif + +static void stats_return_everything(char **reply, const char *reply_end) { + unsigned long long total[OT_COUNTER_COUNT]; + + counters_total(total); + + torrent_stats stats = {0, 0, 0}; + int i; + + iterate_all_torrents(torrent_statter, (uintptr_t)&stats); + + fmt_append(reply, reply_end, "\n"); + fmt_append(reply, reply_end, "\n"); + fmt_append(reply, reply_end, " %llu\n", (unsigned long long)(time(NULL) - ot_start_time)); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " %zd\n", mutex_get_torrent_count()); + fmt_append(reply, reply_end, " %llu\n", stats.torrent_count); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n %llu\n \n", stats.peer_count); + fmt_append(reply, reply_end, " \n %llu\n \n", stats.seed_count); + fmt_append(reply, reply_end, " \n %llu\n \n", total[OT_COUNTER_COMPLETED]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n %llu\n %llu\n %llu\n \n", + total[OT_COUNTER_TCP_CONNECTIONS], total[OT_COUNTER_TCP_ANNOUNCES], total[OT_COUNTER_TCP_SCRAPES]); + fmt_append(reply, reply_end, " \n %llu\n %llu\n %llu\n %llu\n %llu\n \n", + total[OT_COUNTER_UDP_CONNECTIONS], total[OT_COUNTER_UDP_CONNECTS], total[OT_COUNTER_UDP_ANNOUNCES], total[OT_COUNTER_UDP_SCRAPES], + total[OT_COUNTER_UDP_CONNID_MISSMATCH]); + fmt_append(reply, reply_end, " \n %llu\n \n", total[OT_COUNTER_SYNCS]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n"); + for (i = 0; i < OT_PEER_TIMEOUT; ++i) + fmt_append(reply, reply_end, " %llu\n", i, total[OT_COUNTER_RENEWED + i]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n"); + for (i = 0; i < CODE_HTTPERROR_COUNT; ++i) + fmt_append(reply, reply_end, " %llu\n", ot_failed_request_names[i], total[OT_COUNTER_FAILED + i]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, " \n %llu\n \n", total[OT_COUNTER_STALLS]); + fmt_append(reply, reply_end, " \n"); + fmt_append(reply, reply_end, ""); +} + +/* The names as the Makefile spells them, so that what a binary reports maps + straight back onto the FEATURES lines it was built from */ +static const char *const build_features[] = { +#ifdef WANT_V4_ONLY + "WANT_V4_ONLY", +#endif +#ifdef WANT_ACCESSLIST_BLACK + "WANT_ACCESSLIST_BLACK", +#endif +#ifdef WANT_ACCESSLIST_WHITE + "WANT_ACCESSLIST_WHITE", +#endif +#ifdef WANT_DYNAMIC_ACCESSLIST + "WANT_DYNAMIC_ACCESSLIST", +#endif +#ifdef WANT_SYNC_LIVE + "WANT_SYNC_LIVE", +#endif +#ifdef WANT_IP_FROM_QUERY_STRING + "WANT_IP_FROM_QUERY_STRING", +#endif +#ifdef WANT_IP_FROM_PROXY + "WANT_IP_FROM_PROXY", +#endif +#ifdef WANT_COMPRESSION_GZIP + "WANT_COMPRESSION_GZIP", +#endif +#ifdef WANT_COMPRESSION_GZIP_ALWAYS + "WANT_COMPRESSION_GZIP_ALWAYS", +#endif +#ifdef WANT_COMPRESSION_ZSTD + "WANT_COMPRESSION_ZSTD", +#endif +#ifdef WANT_COMPRESSION_ZSTD_ALWAYS + "WANT_COMPRESSION_ZSTD_ALWAYS", +#endif +#ifdef WANT_FULLSCRAPE + "WANT_FULLSCRAPE", +#endif +#ifdef WANT_MODEST_FULLSCRAPES + "WANT_MODEST_FULLSCRAPES", +#endif +#ifdef WANT_KEEPALIVE + "WANT_KEEPALIVE", +#endif +#ifdef WANT_RESTRICT_STATS + "WANT_RESTRICT_STATS", +#endif +#ifdef WANT_LOG_NUMWANT + "WANT_LOG_NUMWANT", +#endif +#ifdef WANT_SPOT_WOODPECKER + "WANT_SPOT_WOODPECKER", +#endif +#ifdef WANT_SYSLOGS + "WANT_SYSLOGS", +#endif +#ifdef WANT_DEV_RANDOM + "WANT_DEV_RANDOM", +#endif +#ifdef WANT_INSECURE_RANDOM_FALLBACK + "WANT_INSECURE_RANDOM_FALLBACK", +#endif +#ifdef _DEBUG + "_DEBUG", +#endif +#ifdef _DEBUG_HTTPERROR + "_DEBUG_HTTPERROR", +#endif +#ifdef _DEBUG_RANDOMTORRENTS + "_DEBUG_RANDOMTORRENTS", +#endif + NULL}; + +void stats_return_tracker_features(char **reply, const char *reply_end) { + size_t i; + + fmt_append(reply, reply_end, "events: %s\n", event_mechanism()); + fmt_append(reply, reply_end, "features:"); + for (i = 0; build_features[i]; ++i) + fmt_append(reply, reply_end, " %s", build_features[i]); + if (!build_features[0]) + fmt_append(reply, reply_end, " none"); + fmt_append(reply, reply_end, "\n"); +} + +void stats_return_tracker_version(char **reply, const char *reply_end) { + fmt_append(reply, reply_end, "opentracker %s", OT_VERSION); + if (*GIT_VERSION && strcmp(GIT_VERSION, OT_TAG)) + fmt_append(reply, reply_end, " (%s)", GIT_VERSION); + fmt_append(reply, reply_end, "\n"); + if (*OT_TAG) + fmt_append(reply, reply_end, "https://erdgeist.org/gitweb/opentracker/tag/?h=%s\n", OT_TAG); +} + +void return_stats_for_tracker(char **reply, const char *reply_end, int mode, int format) { + (void)format; + switch (mode & TASK_TASK_MASK) { + case TASK_STATS_CONNS: + stats_connections_mrtg(reply, reply_end); + break; + case TASK_STATS_SCRAPE: + stats_scrape_mrtg(reply, reply_end); + break; + case TASK_STATS_UDP: + stats_udpconnections_mrtg(reply, reply_end); + break; + case TASK_STATS_TCP: + stats_tcpconnections_mrtg(reply, reply_end); + break; + case TASK_STATS_FULLSCRAPE: + stats_fullscrapes_mrtg(reply, reply_end); + break; + case TASK_STATS_COMPLETED: + stats_return_completed_mrtg(reply, reply_end); + break; + case TASK_STATS_HTTPERRORS: + stats_httperrors_txt(reply, reply_end); + break; + case TASK_STATS_RENEW: + stats_return_renew_bucket(reply, reply_end); + break; + case TASK_STATS_SYNCS: + stats_return_sync_mrtg(reply, reply_end); + break; +#ifdef WANT_LOG_NUMWANT + case TASK_STATS_NUMWANTS: + stats_return_numwants(reply, reply_end); + break; +#endif + default: + break; + } +} + +static void stats_make(size_t *iovec_entries, struct iovec **iovector, ot_tasktype mode) { + char *r; + const char *r_end; + + *iovec_entries = 0; + *iovector = NULL; + if (!(r = iovec_new_chunk(iovec_entries, iovector, OT_STATS_TMPSIZE))) + return; + r_end = r + OT_STATS_TMPSIZE; + + switch (mode & TASK_TASK_MASK) { + case TASK_STATS_TORRENTS: + stats_torrents_mrtg(&r, r_end); + break; + case TASK_STATS_PEERS: + stats_peers_mrtg(&r, r_end); + break; + case TASK_STATS_SLASH24S: + stats_slash24s_txt(&r, r_end, 128); + break; + case TASK_STATS_TOP10: + stats_top_txt(&r, r_end, 10); + break; + case TASK_STATS_TOP100: + r = iovec_next_chunk(iovec_entries, iovector, r, 4 * OT_STATS_TMPSIZE); + if (!r) + return; + r_end = r + 4 * OT_STATS_TMPSIZE; + stats_top_txt(&r, r_end, 100); + break; + case TASK_STATS_EVERYTHING: + r = iovec_next_chunk(iovec_entries, iovector, r, OT_STATS_TMPSIZE + 64 * OT_PEER_TIMEOUT); + if (!r) + return; + r_end = r + OT_STATS_TMPSIZE + 64 * OT_PEER_TIMEOUT; + stats_return_everything(&r, r_end); + break; +#ifdef WANT_SPOT_WOODPECKER + case TASK_STATS_WOODPECKERS: + stats_return_woodpeckers(&r, r_end, 128); + break; +#endif + default: + iovec_free(iovec_entries, iovector); + return; + } + iovec_trim_last(iovec_entries, iovector, r); +} + +void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data) { + if (!ws) + return; + + switch (event) { + case EVENT_ACCEPT: + if (proto == FLAG_TCP) + COUNT(OT_COUNTER_TCP_CONNECTIONS, 1); + else + COUNT(OT_COUNTER_UDP_CONNECTIONS, 1); + break; + case EVENT_ANNOUNCE: + if (proto == FLAG_TCP) + COUNT(OT_COUNTER_TCP_ANNOUNCES, 1); + else + COUNT(OT_COUNTER_UDP_ANNOUNCES, 1); + break; + case EVENT_CONNECT: + if (proto == FLAG_TCP) + COUNT(OT_COUNTER_TCP_CONNECTS, 1); + else + COUNT(OT_COUNTER_UDP_CONNECTS, 1); + break; + case EVENT_COMPLETED: +#ifdef WANT_SYSLOGS + { + char timestring[64]; + char hash_hex[42], peerid_hex[42], ip_readable[64]; + struct tm time_now; + time_t ttt; + + time(&ttt); + localtime_r(&ttt, &time_now); + strftime(timestring, sizeof(timestring), "%FT%T%z", &time_now); + + hex_from_hash(hash_hex, *ws->hash); + if (ws->peer_id) + hex_from_hash(peerid_hex, (uint8_t *)ws->peer_id); + else { + *peerid_hex = 0; + } + + ip_readable[ip_format(ip_readable, (char *)&ws->peer)] = 0; + syslog(LOG_INFO, "time=%s event=completed info_hash=%s peer_id=%s ip=%s", timestring, hash_hex, peerid_hex, ip_readable); + } +#endif + COUNT(OT_COUNTER_COMPLETED, 1); + break; + case EVENT_SCRAPE: + if (proto == FLAG_TCP) + COUNT(OT_COUNTER_TCP_SCRAPES, 1); + else + COUNT(OT_COUNTER_UDP_SCRAPES, 1); + break; + case EVENT_FULLSCRAPE: + COUNT(OT_COUNTER_FULLSCRAPES, 1); + COUNT(OT_COUNTER_FULLSCRAPE_BYTES, event_data); + break; + case EVENT_FULLSCRAPE_REQUEST: { + ot_ip6 *ip = (ot_ip6 *)event_data; /* ugly hack to transfer ip to stats */ + char _debug[512]; + int off = snprintf(_debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(time_now_seconds() - ot_start_time) / 60); + off += ip_format(_debug + off, *ip); + off += snprintf(_debug + off, sizeof(_debug) - off, " - FULL SCRAPE\n"); + (void)write(2, _debug, off); + COUNT(OT_COUNTER_FULLSCRAPE_REQUESTS, 1); + } break; + case EVENT_FULLSCRAPE_REQUEST_GZIP: { + ot_ip6 *ip = (ot_ip6 *)event_data; /* ugly hack to transfer ip to stats */ + char _debug[512]; + int off = snprintf(_debug, sizeof(_debug), "[%08d] scrp: ", (unsigned int)(time_now_seconds() - ot_start_time) / 60); + off += ip_format(_debug + off, *ip); + off += snprintf(_debug + off, sizeof(_debug) - off, " - FULL SCRAPE\n"); + (void)write(2, _debug, off); + COUNT(OT_COUNTER_FULLSCRAPE_REQUESTS, 1); + } break; + case EVENT_FAILED: + COUNT(OT_COUNTER_FAILED + event_data, 1); + break; + case EVENT_RENEW: + COUNT(OT_COUNTER_RENEWED + event_data, 1); + break; + case EVENT_SYNC: + COUNT(OT_COUNTER_SYNCS, event_data); + break; + case EVENT_BUCKET_LOCKED: + COUNT(OT_COUNTER_STALLS, 1); + break; +#ifdef WANT_SPOT_WOODPECKER + case EVENT_WOODPECKER: + pthread_mutex_lock(&g_woodpeckers_mutex); + stat_increase_network_count(&stats_woodpeckers_tree, 0, event_data); + pthread_mutex_unlock(&g_woodpeckers_mutex); + break; +#endif + case EVENT_CONNID_MISSMATCH: + COUNT(OT_COUNTER_UDP_CONNID_MISSMATCH, 1); + default: + break; + } +} + +void stats_cleanup(void) { +#ifdef WANT_SPOT_WOODPECKER + pthread_mutex_lock(&g_woodpeckers_mutex); + stats_shift_down_network_count(&stats_woodpeckers_tree, 0, 1); + pthread_mutex_unlock(&g_woodpeckers_mutex); +#endif +} + +static void *stats_worker(void *args) { + size_t iovec_entries; + struct iovec *iovector; + + (void)args; + + while (g_opentracker_running) { + ot_tasktype tasktype = TASK_STATS; + ot_taskid taskid = workqueue_poptask(&tasktype); + if (!taskid) + break; + stats_make(&iovec_entries, &iovector, tasktype); + if (workqueue_pushresult(taskid, iovec_entries, iovector)) + iovec_free(&iovec_entries, &iovector); + } + return NULL; +} + +void stats_deliver(int64_t sock, int tasktype) { workqueue_pushtask(sock, tasktype); } + +static pthread_t thread_id; +void stats_init(void) { + ot_start_time = time_now_seconds(); + pthread_create(&thread_id, NULL, stats_worker, NULL); +} + +void stats_deinit(void) { + workqueue_wakeall(); + pthread_join(thread_id, NULL); +} diff --git a/src/ot_stats.h b/src/ot_stats.h new file mode 100644 index 0000000..45a547e --- /dev/null +++ b/src/ot_stats.h @@ -0,0 +1,40 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_STATS_H__ +#define OT_STATS_H__ + +#include "trackerlogic.h" + +typedef enum { + EVENT_ACCEPT, + EVENT_READ, + EVENT_CONNECT, /* UDP only */ + EVENT_ANNOUNCE, + EVENT_COMPLETED, + EVENT_RENEW, + EVENT_SYNC, + EVENT_SCRAPE, + EVENT_FULLSCRAPE_REQUEST, + EVENT_FULLSCRAPE_REQUEST_GZIP, + EVENT_FULLSCRAPE_REQUEST_ZSTD, + EVENT_FULLSCRAPE, /* TCP only */ + EVENT_FAILED, + EVENT_BUCKET_LOCKED, + EVENT_WOODPECKER, + EVENT_CONNID_MISSMATCH +} ot_status_event; + +void stats_counters_register(struct ot_workstruct *ws); +void stats_counters_retire(struct ot_workstruct *ws); + +void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data); +void stats_deliver(int64_t sock, int tasktype); +void stats_cleanup(void); +void return_stats_for_tracker(char **reply, const char *reply_end, int mode, int format); +void stats_return_tracker_version(char **reply, const char *reply_end); +void stats_return_tracker_features(char **reply, const char *reply_end); +void stats_init(void); +void stats_deinit(void); + +#endif diff --git a/src/ot_tcp.c b/src/ot_tcp.c new file mode 100644 index 0000000..1351b92 --- /dev/null +++ b/src/ot_tcp.c @@ -0,0 +1,180 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_batch.h" +#include "ot_buf.h" +#include "ot_event.h" +#include "ot_fdstate.h" +#include "ot_http.h" +#include "ot_mutex.h" +#include "ot_socket.h" +#include "ot_worker.h" +#include "ot_stats.h" +#include "ot_tcp.h" +#include "trackerlogic.h" + +static ssize_t header_complete(char *request, ssize_t byte_count) { + ssize_t i = 0, state = 0; + + for (i = 1; i < byte_count; i += 2) + if (request[i] <= 13) { + i--; + for (state = 0; i < byte_count; ++i) { + char c = request[i]; + if (c == '\r' || c == '\n') + state = (state >> 2) | ((c << 6) & 0xc0); + else + break; + if (state >= 0xa0 || state == 0x99) + return i + 1; + } + } + return 0; +} + +void tcp_read(const int64_t sock, struct ot_workstruct *ws) { + struct http_data *cookie = fdstate_cookie(sock); + ssize_t byte_count = read((int)sock, ws->inbuf, G_INBUF_SIZE); + + if (byte_count == 0 || (byte_count == -1 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR)) { + http_close(sock); + return; + } + + if (byte_count == -1) + return; + + ws->request = NULL; + ws->request_size = 0; + ws->header_size = 0; + ws->keep_alive = 0; + + /* If we get the whole request in one packet, handle it without copying */ + if (!cookie->request.data) { + if ((ws->header_size = header_complete(ws->inbuf, byte_count))) { + ws->request = ws->inbuf; + ws->request_size = byte_count; + http_handle_request(sock, ws); + } else if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count)) + http_close(sock); + return; + } + + if (buf_cat(&cookie->request, ws->inbuf, (size_t)byte_count) || cookie->request.size > G_INBUF_SIZE) { + http_issue_error(sock, ws, CODE_HTTPERROR_500); + return; + } + + while ((ws->header_size = header_complete(cookie->request.data, (ssize_t)cookie->request.size))) { + ws->request = cookie->request.data; + ws->request_size = (ssize_t)cookie->request.size; + http_handle_request(sock, ws); +#ifdef WANT_KEEPALIVE + if (!ws->keep_alive) +#endif + return; + } +} + +void tcp_write(const int64_t sock) { + struct http_data *cookie = fdstate_cookie(sock); + int chunked = 0; + int keep_alive = 0; + + if (cookie) { + chunked = (cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER) != 0; + keep_alive = (cookie->flag & STRUCT_HTTP_FLAG_KEEPALIVE) != 0; + + if (cookie->batch) { + if (batch_send(sock, cookie->batch) == BATCH_FAILED) { + http_close(sock); + return; + } + if (batch_pending(cookie->batch)) + return; + } + } + + /* In a chunked transfer, wait for the next piece once everything queued has gone out */ + if (chunked) + http_interest_set(sock, cookie, 0); + else if (keep_alive) { + /* The reply is out and the client asked to keep the connection, so go + back to waiting for the next request */ + http_interest_set(sock, cookie, EVENT_IN); + fdstate_set_expiry(sock, time_future_seconds(OT_CLIENT_TIMEOUT)); + } else + http_close(sock); +} + +/* The fast path, assuming that the complete request has usually arrived by the time a + connection is accepted, so it is answered here outside the event loop. The more load + a tracker has, the more likely it is that by the time it accepts, the roundtrip to + the peer has completed. + If the request could not be completely handled, the stack cookie is promoted to heap + and the connection handed over to the event loop to finish */ +void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws) { + int64_t sock; + ot_ip6 ip; + uint16_t port; + + while ((sock = socket_accept(serversocket, ip, &port)) != -1) { + struct http_data cookie; + uint32_t generation; + + memset(&cookie, 0, sizeof(cookie)); + memcpy(cookie.ip, ip, sizeof(ot_ip6)); + cookie.interest = EVENT_IN; + cookie.flag = STRUCT_HTTP_FLAG_ONSTACK; + + generation = fdstate_open(sock, &cookie, time_future_seconds(OT_CLIENT_TIMEOUT)); + if (!fdstate_claim((int)sock, generation)) { + fdstate_close(sock); + event_close((int)sock); + continue; + } + + stats_issue_event(ws, EVENT_ACCEPT, FLAG_TCP, (uintptr_t)ip); + + tcp_read(sock, ws); + + /* Gone already if it was answered and closed inline */ + if (fdstate_generation((int)sock) != generation) + continue; + + /* Handling it may already have given it a heap home and handed it to + another thread, and then it is no longer this frame's to touch */ + if (fdstate_cookie((int)sock) != &cookie) + fdstate_release((int)sock); + else if (!http_adopt(sock)) + http_close(sock); + else + fdstate_release((int)sock); + } +} + +/* A worker blocks in accept and nowhere else: the connection it is handed is + non blocking, so a client that stops talking is given to the event loop + rather than keeping this thread. */ +static void tcp_run(struct ot_worker *me) { + while (g_opentracker_running) + tcp_accept(me->sock, &me->ws); +} + +void tcp_worker_install(int64_t sock) { + socket_set_blocking((int)sock, 1); + socket_defer_accept((int)sock); + + if (worker_start(tcp_run, sock) == -1) + fprintf(stderr, "Warning: could not install a tcp worker, this socket stays with the workers already running.\n"); +} diff --git a/src/ot_tcp.h b/src/ot_tcp.h new file mode 100644 index 0000000..afed142 --- /dev/null +++ b/src/ot_tcp.h @@ -0,0 +1,24 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_TCP_H__ +#define OT_TCP_H__ + +#include + +#include "trackerlogic.h" + +/* Take whatever connections are waiting on a listening socket and answer + them, each on this thread's stack, handing a connection that cannot be + finished to the event loop */ +void tcp_accept(const int64_t serversocket, struct ot_workstruct *ws); + +/* Handler connections in the event loop has become readable or writable */ +void tcp_read(const int64_t sock, struct ot_workstruct *ws); +void tcp_write(const int64_t sock); + +/* One thread accepting on one socket. Called once per worker. */ +void tcp_worker_install(int64_t sock); + + +#endif diff --git a/src/ot_udp.c b/src/ot_udp.c new file mode 100644 index 0000000..644b71d --- /dev/null +++ b/src/ot_udp.c @@ -0,0 +1,226 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_ip.h" +#include "ot_random.h" +#include "ot_rijndael.h" +#include "ot_socket.h" +#include "ot_worker.h" +#include "ot_stats.h" +#include "ot_udp.h" +#include "trackerlogic.h" + +static uint32_t g_rijndael_round_key[44] = {0}; +static volatile uint32_t g_key_of_the_hour[2] = {0}; +static volatile ot_time g_hour_of_the_key; + + +static void udp_generate_rijndael_round_key(void) { + uint32_t key[16]; + random_bytes(key, sizeof(key)); + rijndaelKeySetupEnc128(g_rijndael_round_key, (uint8_t *)key); + + g_key_of_the_hour[0] = random_uint32(); + g_hour_of_the_key = time_now_minutes(); +} + +/* Generate current and previous connection id for ip */ +static void udp_make_connectionid(uint32_t connid[2], const ot_ip6 remoteip, int age) { + uint32_t plain[4], crypt[4]; + int i; + uint32_t current_key_of_the_hour; + + if (time_now_minutes() - g_hour_of_the_key >= 60) { + uint32_t old_key_of_the_hour = g_key_of_the_hour[0]; + uint32_t new_key_of_the_hour = random_uint32(); + /* If in the meantime another thread has performed + key rotation, do not overwrite their results */ + if (time_now_minutes() - g_hour_of_the_key >= 60) { + /* Upgrade en bloc */ + g_hour_of_the_key = time_now_minutes(); + g_key_of_the_hour[0] = new_key_of_the_hour; + g_key_of_the_hour[1] = old_key_of_the_hour; + } + } + + memcpy(plain, remoteip, sizeof(plain)); + current_key_of_the_hour = g_key_of_the_hour[age]; + for (i = 0; i < 4; ++i) + plain[i] ^= current_key_of_the_hour; + rijndaelEncrypt128(g_rijndael_round_key, (uint8_t *)plain, (uint8_t *)crypt); + connid[0] = crypt[0] ^ crypt[1]; + connid[1] = crypt[2] ^ crypt[3]; +} + +/* UDP implementation according to http://xbtt.sourceforge.net/udp_tracker_protocol.html */ +int handle_udp6(int64_t serversocket, struct ot_workstruct *ws) { + ot_ip6 remoteip; + uint32_t *inpacket = (uint32_t *)ws->inbuf; + uint32_t *outpacket = (uint32_t *)ws->outbuf; + uint32_t left, event; + uint32_t connid[2]; + uint32_t action; + uint16_t port, remoteport; + size_t byte_count, scrape_count; + + ssize_t result = socket_recv(serversocket, ws->inbuf, G_INBUF_SIZE, remoteip, &remoteport); + if (result <= 0) + return 0; + byte_count = (size_t)result; + + stats_issue_event(ws, EVENT_ACCEPT, FLAG_UDP, (uintptr_t)remoteip); + stats_issue_event(ws, EVENT_READ, FLAG_UDP, byte_count); + + /* Minimum udp tracker packet size, also catches error */ + if (byte_count < 16) + return 1; + + /* Get action to take. Ignore error messages and broken packets */ + action = ntohl(inpacket[2]); + if (action > 2) + return 1; + + /* Generate the connection id we give out and expect to and from + the requesting ip address, this prevents udp spoofing */ + udp_make_connectionid(connid, remoteip, 0); + + /* Initialise hash pointer */ + ws->hash = NULL; + ws->peer_id = NULL; + + /* If action is not 0 (connect), then we expect the derived + connection id in first 64 bit */ + if ((action > 0) && (inpacket[0] != connid[0] || inpacket[1] != connid[1])) { + /* If connection id does not match, try the one that was + valid in the previous hour. Only if this also does not + match, return an error packet */ + udp_make_connectionid(connid, remoteip, 1); + if (inpacket[0] != connid[0] || inpacket[1] != connid[1]) { + const char err_idmiss[] = "Connection ID missmatch."; + size_t s = sizeof(err_idmiss); + outpacket[0] = htonl(3); + outpacket[1] = inpacket[3]; + memcpy(&outpacket[2], &err_idmiss[0], s); + socket_send(serversocket, ws->outbuf, 8 + s, remoteip, remoteport); + stats_issue_event(ws, EVENT_CONNID_MISSMATCH, FLAG_UDP, 8 + s); + return 1; + } + } + + switch (action) { + case 0: /* This is a connect action */ + /* look for udp bittorrent magic id */ + if ((ntohl(inpacket[0]) != 0x00000417) || (ntohl(inpacket[1]) != 0x27101980)) + return 1; + + outpacket[0] = 0; + outpacket[1] = inpacket[3]; + outpacket[2] = connid[0]; + outpacket[3] = connid[1]; + + socket_send(serversocket, ws->outbuf, 16, remoteip, remoteport); + stats_issue_event(ws, EVENT_CONNECT, FLAG_UDP, 16); + break; + case 1: /* This is an announce action */ + /* Minimum udp announce packet size */ + if (byte_count < 98) + return 1; + + /* We do only want to know, if it is zero */ + left = inpacket[64 / 4] | inpacket[68 / 4]; + + event = ntohl(inpacket[80 / 4]); + port = *(uint16_t *)(((char *)inpacket) + 96); + ws->hash = (ot_hash *)(((char *)inpacket) + 16); + + OT_SETIP(ws->peer, remoteip); + OT_SETPORT(ws->peer, &port); + OT_PEERFLAG(ws->peer) = 0; + + switch (event) { + case 1: + OT_PEERFLAG(ws->peer) |= PEER_FLAG_COMPLETED; + break; + case 3: + OT_PEERFLAG(ws->peer) |= PEER_FLAG_STOPPED; + break; + default: + break; + } + + if (!left) + OT_PEERFLAG(ws->peer) |= PEER_FLAG_SEEDING; + + outpacket[0] = htonl(1); /* announce action */ + outpacket[1] = inpacket[12 / 4]; + + if (OT_PEERFLAG(ws->peer) & PEER_FLAG_STOPPED) { /* Peer is gone. */ + ws->reply = ws->outbuf; + ws->reply_size = remove_peer_from_torrent(FLAG_UDP, ws); + } else { + /* Limit amount of peers to OT_MAX_PEERS_UDP */ + uint32_t numwant = ntohl(inpacket[92 / 4]); + size_t max_peers = ip_is_v4mapped(remoteip) ? OT_MAX_PEERS_UDP4 : OT_MAX_PEERS_UDP6; + if (numwant > max_peers) + numwant = max_peers; + + ws->reply = ws->outbuf + 8; + ws->reply_size = add_peer_to_torrent_and_return_peers(FLAG_UDP, ws, numwant); + + if (!ws->reply_size) { + const char err_unauth[] = "Requested download is not authorized for use with this tracker."; + size_t s = sizeof(err_unauth); + outpacket[0] = htonl(3); + memcpy(&outpacket[2], &err_unauth[0], s); + socket_send(serversocket, ws->outbuf, 8 + s, remoteip, remoteport); + return 1; + } + ws->reply_size += 8; + } + + socket_send(serversocket, ws->outbuf, ws->reply_size, remoteip, remoteport); + stats_issue_event(ws, EVENT_ANNOUNCE, FLAG_UDP, ws->reply_size); + break; + + case 2: /* This is a scrape action */ + outpacket[0] = htonl(2); /* scrape action */ + outpacket[1] = inpacket[12 / 4]; + + for (scrape_count = 0; (scrape_count * 20 < byte_count - 16) && (scrape_count <= 74); scrape_count++) + return_udp_scrape_for_torrent(*(ot_hash *)(((char *)inpacket) + 16 + 20 * scrape_count), ((char *)outpacket) + 8 + 12 * scrape_count); + + socket_send(serversocket, ws->outbuf, 8 + 12 * scrape_count, remoteip, remoteport); + stats_issue_event(ws, EVENT_SCRAPE, FLAG_UDP, scrape_count); + break; + } + return 1; +} + +static void udp_run(struct ot_worker *me) { + while (g_opentracker_running) + handle_udp6(me->sock, &me->ws); +} + +void udp_init(void) { + if (!g_rijndael_round_key[0]) + udp_generate_rijndael_round_key(); +} + +void udp_worker_install(int64_t sock) { +#ifdef _DEBUG + fprintf(stderr, "Installing a worker on udp socket %ld\n", (unsigned long)sock); +#endif + socket_set_blocking((int)sock, 1); + + if (worker_start(udp_run, sock) == -1) + fprintf(stderr, "Warning: could not install a udp worker, this socket stays with the workers already running.\n"); +} diff --git a/src/ot_udp.h b/src/ot_udp.h new file mode 100644 index 0000000..2710744 --- /dev/null +++ b/src/ot_udp.h @@ -0,0 +1,17 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_UDP_H__ +#define OT_UDP_H__ + +#include + +#include "trackerlogic.h" + +void udp_init(void); +/* One thread reading one socket. Called once per worker. */ +void udp_worker_install(int64_t sock); + +int handle_udp6(int64_t serversocket, struct ot_workstruct *ws); + +#endif diff --git a/src/ot_urlencoded.c b/src/ot_urlencoded.c new file mode 100644 index 0000000..4109bb5 --- /dev/null +++ b/src/ot_urlencoded.c @@ -0,0 +1,168 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* Opentracker */ +#include "ot_urlencoded.h" + +/* System */ +#include + +/* Idea is to do a in place replacement or guarantee at least + strlen( string ) bytes in deststring + watch http://www.ietf.org/rfc/rfc2396.txt + unreserved = alphanum | mark + mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")" + we add '%' to the matrix to not stop at encoded chars. + After losing too many requests to being too strict, add the following characters to reserved matrix + relax = "+" | "," | "/" | ";" | "<" | ">" | ":" +*/ + +/* This matrix holds for each ascii character the information, + whether it is a non-terminating character for one of the three + scan states we are in, that is 'path', 'param' and 'value' from + /path?param=value¶m=value, it is encoded in bit 0, 1 and 2 + respectively + + The top bit of lower nibble indicates, whether this character is + a hard terminator, ie. \0, \n or \s, where the whole scanning + process should terminate + */ +static const unsigned char is_unreserved[256] = { + 8,0,0,0,0,0,0,0,0,0,8,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 8,7,8,8,8,7,0,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,4,7,6, + 4,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,7, + 8,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,7,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +}; + +/* Do a fast nibble to hex representation conversion */ +static unsigned char fromhex(unsigned char x) { + x -= '0'; + if (x <= 9) + return x; + x &= ~0x20; + x -= 'A' - '0'; + if (x < 6) + return x + 10; + return 0xff; +} + +/* Skip the value of a param=value pair */ +void urlencoded_skip_value(char **string) { + const unsigned char *s = *(const unsigned char **)string; + unsigned char f; + + /* Since we are asked to skip the 'value', we assume to stop at + terminators for a 'value' string position */ + while ((f = is_unreserved[*s++]) & SCAN_SEARCHPATH_VALUE) + ; + + /* If we stopped at a hard terminator like \0 or \n, make the + next urlencoded_parse encounter it again */ + if (f & SCAN_SEARCHPATH_TERMINATOR) + --s; + + *string = (char *)s; +} + +int urlencoded_match_keyword(const ot_keywords *keywords, char **string, SCAN_SEARCHPATH_FLAG flags) { + char *deststring = *string; + ssize_t match_length = urlencoded_parse(string, deststring, flags); + + if (match_length < 0) + return match_length; + if (match_length == 0) + return -3; + + while (keywords->key) { + if (!strncmp(keywords->key, deststring, match_length) && !keywords->key[match_length]) + return keywords->value; + keywords++; + } + + return -3; +} + +ssize_t urlencoded_parse(char **string, char *deststring, SCAN_SEARCHPATH_FLAG flags) { + const unsigned char *s = *(const unsigned char **)string; + unsigned char *d = (unsigned char *)deststring; + unsigned char b, c; + + /* This is the main decoding loop. + 'flag' determines, which characters are non-terminating in current context + (ie. stop at '=' and '&' if scanning for a 'param'; stop at '?' if scanning for the path ) + */ + while (is_unreserved[c = *s++] & flags) { + + /* When encountering an url escaped character, try to decode */ + if (c == '%') { + if ((b = fromhex(*s++)) == 0xff) + return -1; + if ((c = fromhex(*s++)) == 0xff) + return -1; + c |= (b << 4); + } + + /* Write (possibly decoded) character to output */ + *d++ = c; + } + + switch (c) { + case 0: + case '\r': + case '\n': + case ' ': + /* If we started scanning on a hard terminator, indicate we've finished */ + if (d == (unsigned char *)deststring) + return -2; + + /* Else make the next call to urlencoded_parse encounter it again */ + --s; + break; + case '?': + if (flags != SCAN_PATH) + return -1; + break; + case '=': + if (flags != SCAN_SEARCHPATH_PARAM) + return -1; + break; + case '&': + if (flags == SCAN_PATH) + return -1; + if (flags == SCAN_SEARCHPATH_PARAM) + --s; + break; + default: + return -1; + } + + *string = (char *)s; + return d - (unsigned char *)deststring; +} + + +size_t urlencoded_format(char *dest, const uint8_t *src, size_t len) { + static const char safe[] = "$/.=~-_"; + static const char hexdigit[] = "0123456789abcdef"; + size_t written = 0, i; + + for (i = 0; i < len; ++i) { + uint8_t c = src[i]; + + /* strchr would find the terminator, so a zero byte never counts as safe */ + if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c && strchr(safe, c))) { + dest[written++] = (char)c; + } else { + dest[written] = '%'; + dest[written + 1] = hexdigit[c >> 4]; + dest[written + 2] = hexdigit[c & 15]; + written += 3; + } + } + + return written; +} diff --git a/src/ot_urlencoded.h b/src/ot_urlencoded.h new file mode 100644 index 0000000..9882468 --- /dev/null +++ b/src/ot_urlencoded.h @@ -0,0 +1,53 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef SCAN_URLENCODED_QUERY_H__ +#define SCAN_URLENCODED_QUERY_H__ + +#include +#include +#include + +typedef struct { + const char *key; + int value; +} ot_keywords; + +typedef enum { + SCAN_PATH = 1, + SCAN_SEARCHPATH_PARAM = 2, + SCAN_SEARCHPATH_VALUE = 4, + SCAN_SEARCHPATH_TERMINATOR = 8 +} SCAN_SEARCHPATH_FLAG; + +/* string in: pointer to source + out: pointer to next scan position + deststring pointer to destination + flags determines, what to parse + returns number of valid converted characters in deststring + or -1 for parse error + or -2 for terminator found +*/ +ssize_t urlencoded_parse(char **string, char *deststring, SCAN_SEARCHPATH_FLAG flags); + +/* string in: pointer to source + out: pointer to next scan position + flags determines, what to parse + returns value for matched keyword + or -1 for parse error + or -2 for terminator found + or -3 for no keyword matched + */ +int urlencoded_match_keyword(const ot_keywords *keywords, char **string, SCAN_SEARCHPATH_FLAG flags); + +/* string in: pointer to value of a param=value pair to skip + out: pointer to next scan position on return +*/ +void urlencoded_skip_value(char **string); + + +/* The other direction: percent encode len bytes from src into dest and return + how many characters were written. dest needs room for three times len. */ +size_t urlencoded_format(char *dest, const uint8_t *src, size_t len); + +#endif diff --git a/src/ot_vector.c b/src/ot_vector.c new file mode 100644 index 0000000..ab0566c --- /dev/null +++ b/src/ot_vector.c @@ -0,0 +1,287 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include + +/* Opentracker */ +#include "trackerlogic.h" + +static int vector_compare_peer6(const void *peer1, const void *peer2) { return memcmp(peer1, peer2, OT_PEER_COMPARE_SIZE6); } +static int vector_compare_peer4(const void *peer1, const void *peer2) { return memcmp(peer1, peer2, OT_PEER_COMPARE_SIZE4); } + +/* This function gives us a binary search that returns a pointer, even if + no exact match is found. In that case it sets exactmatch 0 and gives + calling functions the chance to insert data +*/ +void *vector_binary_search(const void *const key, const void *base, const size_t member_count, const size_t member_size, size_t compare_size, int *exactmatch) { + size_t interval = member_count; + + while (interval) { + uint8_t *lookat = ((uint8_t *)base) + member_size * (interval / 2); + int cmp = memcmp(lookat, key, compare_size); + if (cmp == 0) { + base = lookat; + break; + } + if (cmp < 0) { + base = lookat + member_size; + interval--; + } + interval /= 2; + } + + *exactmatch = interval; + return (void *)base; +} + +static uint8_t vector_hash_peer(ot_peer const *peer, size_t compare_size, size_t bucket_count) { + unsigned int hash = 5381; + uint8_t *p = (uint8_t *)peer; + while (compare_size--) + hash += (hash << 5) + *(p++); + return hash % bucket_count; +} + +/* This is the generic insert operation for our vector type. + It tries to locate the object at "key" with size "member_size" by comparing its first "compare_size" bytes with + those of objects in vector. Our special "vector_binary_search" function does that and either returns the match or a + pointer to where the object is to be inserted. vector_find_or_insert makes space for the object and copies it, + if it wasn't found in vector. Caller needs to check the passed "exactmatch" variable to see, whether an insert + took place. If resizing the vector failed, NULL is returned, else the pointer to the object in vector. +*/ +void *vector_find_or_insert(ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch) { + uint8_t *match = vector_binary_search(key, vector->data, vector->size, member_size, compare_size, exactmatch); + size_t match_offset, bytes_after; + + if (*exactmatch) + return match; + + /* Hold the insertion point as an offset rather than a pointer: it survives + the realloc below, and an empty vector has no data pointer to offset from + in the first place */ + match_offset = vector->size ? (size_t)(match - (uint8_t *)vector->data) : 0; + bytes_after = member_size * vector->size - match_offset; + + if (vector->size + 1 > vector->space) { + size_t new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS; + uint8_t *new_data = realloc(vector->data, new_space * member_size); + + if (!new_data) + return NULL; + + vector->data = new_data; + vector->space = new_space; + } + + match = (uint8_t *)vector->data + match_offset; + memmove(match + member_size, match, bytes_after); + + vector->size++; + return match; +} + +ot_peer *vector_find_or_insert_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size, int *exactmatch) { + ot_peer *match; + const size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + size_t match_offset, bytes_after; + + /* If space is zero but size is set, we're dealing with a list of vector->size buckets */ + if (vector->space < vector->size) + vector = ((ot_vector *)vector->data) + vector_hash_peer(peer, compare_size, vector->size); + match = vector_binary_search(peer, vector->data, vector->size, peer_size, compare_size, exactmatch); + + if (*exactmatch) + return match; + + /* Hold the insertion point as an offset rather than a pointer: it survives + the realloc below, and an empty vector has no data pointer to offset from + in the first place. bytes_after is what has to move up by peer_size to + make room for the new peer */ + match_offset = vector->size ? (size_t)(match - (ot_peer *)vector->data) : 0; + bytes_after = vector->size * peer_size - match_offset; + + if (vector->size + 1 > vector->space) { + size_t new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS; + ot_peer *new_data = realloc(vector->data, new_space * peer_size); + + if (!new_data) + return NULL; + + vector->data = new_data; + vector->space = new_space; + } + + match = (ot_peer *)vector->data + match_offset; + memmove(match + peer_size, match, bytes_after); + + vector->size++; + return match; +} + +/* This is the non-generic delete from vector-operation specialized for peers in pools. + It returns 0 if no peer was found (and thus not removed) + 1 if a non-seeding peer was removed + 2 if a seeding peer was removed +*/ +int vector_remove_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size) { + int exactmatch, was_seeder; + ot_peer *match, *end; + size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + + if (!vector->size) + return 0; + + /* If space is zero but size is set, we're dealing with a list of vector->size buckets */ + if (vector->space < vector->size) + vector = ((ot_vector *)vector->data) + vector_hash_peer(peer, compare_size, vector->size); + + end = ((ot_peer *)vector->data) + peer_size * vector->size; + match = (ot_peer *)vector_binary_search(peer, vector->data, vector->size, peer_size, compare_size, &exactmatch); + if (!exactmatch) + return 0; + + was_seeder = (OT_PEERFLAG_D(match, peer_size) & PEER_FLAG_SEEDING) ? 2 : 1; + memmove(match, match + peer_size, end - match - peer_size); + + vector->size--; + vector_fixup_peers(vector, peer_size); + return was_seeder; +} + +void vector_remove_torrent(ot_vector *vector, ot_torrent *match) { + ot_torrent *end; + + if (!vector->size) + return; + + end = ((ot_torrent *)vector->data) + vector->size; + + free_peerlist(match->peer_list6); + free_peerlist(match->peer_list4); + + memmove(match, match + 1, sizeof(ot_torrent) * (end - match - 1)); + if ((--vector->size * OT_VECTOR_SHRINK_THRESH < vector->space) && (vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS)) { + vector->space /= OT_VECTOR_SHRINK_RATIO; + vector->data = realloc(vector->data, vector->space * sizeof(ot_torrent)); + } +} + +void vector_clean_list(ot_vector *vector, size_t num_buckets) { + while (num_buckets--) + free(vector[num_buckets].data); + free(vector); + return; +} + +void vector_redistribute_buckets(ot_peerlist *peer_list, size_t peer_size) { + size_t bucket, bucket_size_new, num_buckets_new, num_buckets_old = 1; + ot_vector *bucket_list_new, *bucket_list_old = &peer_list->peers; + int (*sort_func)(const void *, const void *) = peer_size == OT_PEER_SIZE6 ? &vector_compare_peer6 : &vector_compare_peer4; + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets_old = peer_list->peers.size; + bucket_list_old = peer_list->peers.data; + } + + if (peer_list->peer_count < 255) + num_buckets_new = 1; + else if (peer_list->peer_count > 8192) + num_buckets_new = 64; + else if (peer_list->peer_count >= 512 && peer_list->peer_count < 4096) + num_buckets_new = 16; + else if (peer_list->peer_count < 512 && num_buckets_old <= 16) + num_buckets_new = num_buckets_old; + else if (peer_list->peer_count < 512) + num_buckets_new = 1; + else if (peer_list->peer_count < 8192 && num_buckets_old > 1) + num_buckets_new = num_buckets_old; + else + num_buckets_new = 16; + + if (num_buckets_new == num_buckets_old) + return; + + /* Assume near perfect distribution */ + bucket_list_new = calloc(1, num_buckets_new * sizeof(ot_vector)); + if (!bucket_list_new) + return; + + bucket_size_new = OT_VECTOR_MIN_MEMBERS; + while (bucket_size_new * num_buckets_new < peer_list->peer_count) + bucket_size_new *= OT_VECTOR_GROW_RATIO; + + /* preallocate vectors to hold all peers */ + for (bucket = 0; bucket < num_buckets_new; ++bucket) { + bucket_list_new[bucket].space = bucket_size_new; + bucket_list_new[bucket].data = malloc(bucket_size_new * peer_size); + if (!bucket_list_new[bucket].data) { + vector_clean_list(bucket_list_new, num_buckets_new); + return; + } + } + + /* Now sort them into the correct bucket */ + for (bucket = 0; bucket < num_buckets_old; ++bucket) { + ot_peer *peers_old = bucket_list_old[bucket].data; + size_t peer_count_old = bucket_list_old[bucket].size; + while (peer_count_old--) { + ot_vector *bucket_dest = bucket_list_new; + if (num_buckets_new > 1) + bucket_dest += vector_hash_peer(peers_old, OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size), num_buckets_new); + if (bucket_dest->size + 1 > bucket_dest->space) { + void *tmp = realloc(bucket_dest->data, peer_size * OT_VECTOR_GROW_RATIO * bucket_dest->space); + if (!tmp) { + vector_clean_list(bucket_list_new, num_buckets_new); + return; + } + bucket_dest->data = tmp; + bucket_dest->space *= OT_VECTOR_GROW_RATIO; + } + memcpy((ot_peer *)bucket_dest->data + peer_size * bucket_dest->size++, peers_old, peer_size); + peers_old += peer_size; + } + } + + /* Now sort each bucket to later allow bsearch */ + for (bucket = 0; bucket < num_buckets_new; ++bucket) + qsort(bucket_list_new[bucket].data, bucket_list_new[bucket].size, peer_size, sort_func); + + /* Everything worked fine. Now link new bucket_list to peer_list */ + if (OT_PEERLIST_HASBUCKETS(peer_list)) + vector_clean_list((ot_vector *)peer_list->peers.data, peer_list->peers.size); + else + free(peer_list->peers.data); + + if (num_buckets_new > 1) { + peer_list->peers.data = bucket_list_new; + peer_list->peers.size = num_buckets_new; + peer_list->peers.space = 0; /* Magic marker for "is list of buckets" */ + } else { + peer_list->peers.data = bucket_list_new->data; + peer_list->peers.size = bucket_list_new->size; + peer_list->peers.space = bucket_list_new->space; + free(bucket_list_new); + } +} + +void vector_fixup_peers(ot_vector *vector, size_t peer_size) { + int need_fix = 0; + + if (!vector->size) { + free(vector->data); + vector->data = NULL; + vector->space = 0; + return; + } + + while ((vector->size * OT_VECTOR_SHRINK_THRESH < vector->space) && (vector->space >= OT_VECTOR_SHRINK_RATIO * OT_VECTOR_MIN_MEMBERS)) { + vector->space /= OT_VECTOR_SHRINK_RATIO; + need_fix++; + } + if (need_fix) + vector->data = realloc(vector->data, vector->space * peer_size); +} diff --git a/src/ot_vector.h b/src/ot_vector.h new file mode 100644 index 0000000..8a5ba86 --- /dev/null +++ b/src/ot_vector.h @@ -0,0 +1,39 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_VECTOR_H__ +#define OT_VECTOR_H__ + +#include + +/* These defines control vectors behaviour */ +#define OT_VECTOR_MIN_MEMBERS 2 +#define OT_VECTOR_GROW_RATIO 2 +#define OT_VECTOR_SHRINK_THRESH 4 +#define OT_VECTOR_SHRINK_RATIO 2 + +#define OT_PEER_BUCKET_MINCOUNT 512 +#define OT_PEER_BUCKET_MAXCOUNT 256 + +typedef struct { + void *data; + size_t size; + size_t space; +} ot_vector; + +/* trackerlogic.h needs ot_vector, break circular dependency here */ +#include "trackerlogic.h" + +void *vector_binary_search(const void *const key, const void *base, const size_t member_count, const size_t member_size, size_t compare_size, int *exactmatch); +void *vector_find_or_insert(ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch); +ot_peer *vector_find_or_insert_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size, int *exactmatch); + +int vector_remove_peer(ot_vector *vector, ot_peer const *peer, size_t peer_size); +void vector_remove_torrent(ot_vector *vector, ot_torrent *match); + +/* For ot_clean.c */ +void vector_redistribute_buckets(ot_peerlist *peer_list, size_t peer_size); +void vector_fixup_peers(ot_vector *vector, size_t peer_size); +void vector_clean_list(ot_vector *vector, size_t num_buckets); + +#endif diff --git a/src/ot_worker.c b/src/ot_worker.c new file mode 100644 index 0000000..432d272 --- /dev/null +++ b/src/ot_worker.c @@ -0,0 +1,73 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_worker.h" +#include "trackerlogic.h" + +static struct ot_worker **g_workers; +static size_t g_worker_count; + +/* Everything a worker needs doing around its own loop, in one place, so that + a new kind of worker cannot miss a piece of it. */ +static void *worker_trampoline(void *args) { + struct ot_worker *me = args; + + if (workstruct_init(&me->ws) == -1) + fprintf(stderr, "Warning: a worker could not get its buffers and will not run.\n"); + else { + me->run(me); + workstruct_deinit(&me->ws); + } + + atomic_store(&me->left, 1); + return NULL; +} + +int worker_start(ot_worker_run run, int64_t sock) { + struct ot_worker **grown = realloc(g_workers, (g_worker_count + 1) * sizeof(*g_workers)); + struct ot_worker *worker; + + if (!grown) + return -1; + g_workers = grown; + + if (!(worker = calloc(1, sizeof(*worker)))) + return -1; + worker->sock = sock; + worker->run = run; + + if (pthread_create(&worker->thread, NULL, worker_trampoline, worker)) { + free(worker); + return -1; + } + g_workers[g_worker_count++] = worker; + return 0; +} + +void worker_stop_all(void) { + size_t i; + + for (i = 0; i < g_worker_count; ++i) { + struct ot_worker *worker = g_workers[i]; + + /* Repeated because a signal only interrupts a call already under way */ + while (!atomic_load(&worker->left)) { + pthread_kill(worker->thread, OT_SIGNAL_WAKE); + usleep(1000); + } + pthread_join(worker->thread, NULL); + free(worker); + } + + free(g_workers); + g_workers = NULL; + g_worker_count = 0; +} diff --git a/src/ot_worker.h b/src/ot_worker.h new file mode 100644 index 0000000..4954773 --- /dev/null +++ b/src/ot_worker.h @@ -0,0 +1,37 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_WORKER_H__ +#define OT_WORKER_H__ + +#include +#include +#include + +#include "trackerlogic.h" + +struct ot_worker; + +/* What a worker does between being started and being told to stop. It reads + worker->sock and works out of worker->ws; everything around it, from the + buffers to the counters to saying goodbye, is handled here. */ +typedef void (*ot_worker_run)(struct ot_worker *worker); + +/* A thread that serves requests on a socket of its own, with the buffers and + counters it needs. Each is allocated on its own, so taking on another one + never moves the record a running worker holds. */ +struct ot_worker { + pthread_t thread; + _Atomic int left; + int64_t sock; + ot_worker_run run; + struct ot_workstruct ws; +}; + +/* Start one. Returns -1 if it could not be started. */ +int worker_start(ot_worker_run run, int64_t sock); + +/* Wake every worker until it has left, then join it */ +void worker_stop_all(void); + +#endif diff --git a/src/ot_workqueue.c b/src/ot_workqueue.c new file mode 100644 index 0000000..56a7cd2 --- /dev/null +++ b/src/ot_workqueue.c @@ -0,0 +1,241 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include + +/* Opentracker */ +#include "ot_iovec.h" +#include "ot_workqueue.h" +#include "trackerlogic.h" + +/* Pushing a result wakes the thread that will send it */ +extern int g_self_pipe[2]; + +struct ot_task { + ot_taskid taskid; + ot_tasktype tasktype; + int64_t sock; + size_t iovec_entries; + struct iovec *iovec; + struct ot_task *next; +}; + +static ot_taskid next_free_taskid = 1; +static struct ot_task *tasklist; +static pthread_mutex_t tasklist_mutex; +static pthread_cond_t tasklist_being_filled; + +int workqueue_pushtask(int64_t sock, ot_tasktype tasktype) { + struct ot_task **tmptask, *task; + + task = malloc(sizeof(struct ot_task)); + if (!task) + return -1; + + task->taskid = 0; + task->tasktype = tasktype; + task->sock = sock; + task->iovec_entries = 0; + task->iovec = NULL; + task->next = 0; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + /* Skip to end of list */ + tmptask = &tasklist; + while (*tmptask) + tmptask = &(*tmptask)->next; + *tmptask = task; + + /* Inform waiting workers and release lock */ + pthread_cond_broadcast(&tasklist_being_filled); + pthread_mutex_unlock(&tasklist_mutex); + return 0; +} + +void workqueue_canceltask(int64_t sock) { + struct ot_task **task; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = &tasklist; *task; task = &((*task)->next)) + if ((*task)->sock == sock) { + struct iovec *iovec = (*task)->iovec; + struct ot_task *ptask = *task; + size_t i; + + /* Free task's iovec, the buffers it points at and the vector itself */ + for (i = 0; i < (*task)->iovec_entries; ++i) + free(iovec[i].iov_base); + free(iovec); + + *task = (*task)->next; + free(ptask); + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); +} + +ot_taskid workqueue_poptask(ot_tasktype *tasktype) { + struct ot_task *task; + ot_taskid taskid = 0; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + while (!taskid) { + /* Skip to the first unassigned task this worker wants to do */ + for (task = tasklist; task; task = task->next) + if (!task->taskid && (TASK_CLASS_MASK & task->tasktype) == *tasktype) { + /* If we found an outstanding task, assign a taskid to it + and leave the loop */ + task->taskid = taskid = ++next_free_taskid; + *tasktype = task->tasktype; + break; + } + + /* Wait until the next task is being fed, unless the tracker is on its + way out, in which case the worker is meant to leave rather than wait */ + if (!taskid) { + if (!g_opentracker_running) + break; + pthread_cond_wait(&tasklist_being_filled, &tasklist_mutex); + } + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); + + return taskid; +} + +void workqueue_pushsuccess(ot_taskid taskid) { + struct ot_task **task; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = &tasklist; *task; task = &((*task)->next)) + if ((*task)->taskid == taskid) { + struct ot_task *ptask = *task; + *task = (*task)->next; + free(ptask); + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); +} + +int workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovec) { + struct ot_task *task; + const char byte = 'o'; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = tasklist; task; task = task->next) + if (task->taskid == taskid) { + task->iovec_entries = iovec_entries; + task->iovec = iovec; + task->tasktype = TASK_DONE; + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); + + /* A full pipe has already done this nudge's job, so a failure is fine */ + (void)!write(g_self_pipe[1], &byte, 1); + + /* Indicate whether the worker has to throw away results */ + return task ? 0 : -1; +} + +int workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec) { + struct ot_task *task; + const char byte = 'o'; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = tasklist; task; task = task->next) + if (task->taskid == taskid) { + if (iovec) { + if (iovec_append(&task->iovec_entries, &task->iovec, iovec)) + task->tasktype = TASK_DONE_PARTIAL; + else + task = NULL; + } else + task->tasktype = TASK_DONE; + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); + + /* A full pipe has already done this nudge's job, so a failure is fine */ + (void)!write(g_self_pipe[1], &byte, 1); + + /* Indicate whether the worker has to throw away results */ + return task ? 0 : -1; +} + +int64_t workqueue_popresult(size_t *iovec_entries, struct iovec **iovec, int *is_partial) { + struct ot_task **task; + int64_t sock = -1; + + *is_partial = 0; + + /* Want exclusive access to tasklist */ + pthread_mutex_lock(&tasklist_mutex); + + for (task = &tasklist; *task; task = &((*task)->next)) + if (((*task)->tasktype & TASK_CLASS_MASK) == TASK_DONE) { + struct ot_task *ptask = *task; + *iovec_entries = ptask->iovec_entries; + *iovec = ptask->iovec; + sock = ptask->sock; + + if ((*task)->tasktype == TASK_DONE) { + *task = ptask->next; + free(ptask); + } else { + ptask->iovec_entries = 0; + ptask->iovec = NULL; + *is_partial = 1; + /* Prevent task from showing up immediately again unless new data was added */ + (*task)->tasktype = TASK_FULLSCRAPE; + } + break; + } + + /* Release lock */ + pthread_mutex_unlock(&tasklist_mutex); + return sock; +} + +void workqueue_wakeall(void) { + pthread_mutex_lock(&tasklist_mutex); + pthread_cond_broadcast(&tasklist_being_filled); + pthread_mutex_unlock(&tasklist_mutex); +} + +void workqueue_init(void) { + pthread_mutex_init(&tasklist_mutex, NULL); + pthread_cond_init(&tasklist_being_filled, NULL); +} + +void workqueue_deinit(void) { + pthread_mutex_destroy(&tasklist_mutex); + pthread_cond_destroy(&tasklist_being_filled); +} diff --git a/src/ot_workqueue.h b/src/ot_workqueue.h new file mode 100644 index 0000000..82ae29b --- /dev/null +++ b/src/ot_workqueue.h @@ -0,0 +1,74 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_WORKQUEUE_H__ +#define OT_WORKQUEUE_H__ + +#include +#include + +#include "trackerlogic.h" + +/* Work the tracker hands to a thread of its own: a full scrape or an + expensive stats page, which take too long to build while a client waits. + The task goes in here, the reply comes back out. */ + +typedef enum { + TASK_STATS_CONNS = 0x0001, + TASK_STATS_TCP = 0x0002, + TASK_STATS_UDP = 0x0003, + TASK_STATS_SCRAPE = 0x0004, + TASK_STATS_FULLSCRAPE = 0x0005, + TASK_STATS_TPB = 0x0006, + TASK_STATS_HTTPERRORS = 0x0007, + TASK_STATS_RENEW = 0x000a, + TASK_STATS_SYNCS = 0x000b, + TASK_STATS_COMPLETED = 0x000c, + TASK_STATS_NUMWANTS = 0x000d, + + TASK_STATS = 0x0100, /* Mask */ + TASK_STATS_TORRENTS = 0x0101, + TASK_STATS_PEERS = 0x0102, + TASK_STATS_SLASH24S = 0x0103, + TASK_STATS_TOP10 = 0x0104, + TASK_STATS_TOP100 = 0x0105, + TASK_STATS_EVERYTHING = 0x0106, + TASK_STATS_WOODPECKERS = 0x0108, + + TASK_FULLSCRAPE = 0x0200, /* Default mode */ + TASK_FULLSCRAPE_TPB_BINARY = 0x0201, + TASK_FULLSCRAPE_TPB_ASCII = 0x0202, + TASK_FULLSCRAPE_TPB_ASCII_PLUS = 0x0203, + TASK_FULLSCRAPE_TPB_URLENCODED = 0x0204, + TASK_FULLSCRAPE_TRACKERSTATE = 0x0205, + + TASK_DMEM = 0x0300, + + TASK_DONE = 0x0f00, + TASK_DONE_PARTIAL = 0x0f01, + + TASK_FLAG_GZIP = 0x1000, + TASK_FLAG_BZIP2 = 0x2000, + TASK_FLAG_ZSTD = 0x4000, + TASK_FLAG_CHUNKED = 0x8000, + + TASK_TASK_MASK = 0x0fff, + TASK_CLASS_MASK = 0x0f00, + TASK_FLAGS_MASK = 0xf000 +} ot_tasktype; + +typedef unsigned long ot_taskid; + +int workqueue_pushtask(int64_t sock, ot_tasktype tasktype); +void workqueue_canceltask(int64_t sock); +void workqueue_pushsuccess(ot_taskid taskid); +ot_taskid workqueue_poptask(ot_tasktype *tasktype); +int workqueue_pushresult(ot_taskid taskid, size_t iovec_entries, struct iovec *iovector); +int workqueue_pushchunked(ot_taskid taskid, struct iovec *iovec); +int64_t workqueue_popresult(size_t *iovec_entries, struct iovec **iovector, int *is_partial); + +void workqueue_init(void); +void workqueue_deinit(void); +void workqueue_wakeall(void); + +#endif diff --git a/src/proxy.c b/src/proxy.c new file mode 100644 index 0000000..8589db5 --- /dev/null +++ b/src/proxy.c @@ -0,0 +1,871 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* This relays sync traffic between trackers, and between multicast domains + that have no route to each other, so that no tracker has to reach out to + the outside world by itself. Defunc. */ + +/* System */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Libowfat */ +#include "io.h" + +/* Opentracker */ +#include "ot_batch.h" +#include "ot_fdstate.h" +#include "ot_ip.h" +#include "ot_mutex.h" +#include "ot_random.h" +#include "ot_scan.h" +#include "ot_socket.h" +#include "ot_stats.h" +#include "ot_vector.h" +#include "trackerlogic.h" + +#include "ot_livesync.h" + +ot_ip6 g_serverip; +uint16_t g_serverport = 9009; +uint32_t g_tracker_id; +ot_ip6 groupip_1 = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 224, 0, 23, 5}; +int g_self_pipe[2]; + +/* If you have more than 10 peers, don't use this proxy + Use 20 slots for 10 peers to have room for 10 incoming connection slots + */ +#define MAX_PEERS 20 + +#define LIVESYNC_INCOMING_BUFFSIZE (256 * 256) +#define STREAMSYNC_OUTGOING_BUFFSIZE (256 * 256) + +#define LIVESYNC_OUTGOING_BUFFSIZE_PEERS 1480 +#define LIVESYNC_OUTGOING_WATERMARK_PEERS (sizeof(ot_peer) + sizeof(ot_hash)) +#define LIVESYNC_MAXDELAY 15 /* seconds */ + +/* The amount of time a complete sync cycle should take */ +#define OT_SYNC_INTERVAL_MINUTES 2 + +/* So after each bucket wait 1 / OT_BUCKET_COUNT intervals */ +#define OT_SYNC_SLEEP (((OT_SYNC_INTERVAL_MINUTES) * 60 * 1000000) / (OT_BUCKET_COUNT)) + +enum { OT_SYNC_PEER4, OT_SYNC_PEER6 }; +enum { FLAG_SERVERSOCKET = 1 }; + +/* For incoming packets */ +static int g_socket_in = -1; +static uint8_t g_inbuffer[LIVESYNC_INCOMING_BUFFSIZE]; + +/* For outgoing packets */ +static int g_socket_out = -1; +static uint8_t g_peerbuffer_start[LIVESYNC_OUTGOING_BUFFSIZE_PEERS]; +static uint8_t *g_peerbuffer_pos; +static uint8_t *g_peerbuffer_highwater = g_peerbuffer_start + LIVESYNC_OUTGOING_BUFFSIZE_PEERS - LIVESYNC_OUTGOING_WATERMARK_PEERS; +static ot_time g_next_packet_time; + +static void *livesync_worker(void *args); +static void *streamsync_worker(void *args); +static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer); + +void exerr(const char *message) { + fprintf(stderr, "%s\n", message); + exit(111); +} + +void stats_issue_event(struct ot_workstruct *ws, ot_status_event event, PROTO_FLAG proto, uintptr_t event_data) { + (void)ws; + (void)event; + (void)proto; + (void)event_data; +} + +void livesync_bind_mcast(ot_ip6 ip, uint16_t port) { + if (!ip_is_v4mapped(ip)) + exerr("v6 mcast support not yet available."); + + if (g_socket_in != -1) + exerr("Error: Livesync listen ip specified twice."); + + if ((g_socket_in = socket_mcast_join(groupip_1, ip, port)) == -1) + exerr("Error: Cant bind live sync incoming socket."); + + if ((g_socket_out = socket_mcast_bind(ip, port, 1)) == -1) + exerr("Error: Cant bind live sync outgoing socket."); +} + +size_t add_peer_to_torrent_proxy(ot_hash hash, ot_peer *peer, size_t peer_size) { + int exactmatch; + ot_torrent *torrent; + ot_peerlist *peer_list; + ot_peer *peer_dest; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); + size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + + torrent = vector_find_or_insert(torrents_list, (void *)hash, sizeof(ot_torrent), compare_size, &exactmatch); + if (!torrent) + return -1; + + if (!exactmatch) { + /* Create a new torrent entry, then */ + memcpy(torrent->hash, hash, sizeof(ot_hash)); + + if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) { + vector_remove_torrent(torrents_list, torrent); + mutex_bucket_unlock_by_hash(hash, 0); + return -1; + } + + memset(torrent->peer_list6, 0, sizeof(ot_peerlist)); + memset(torrent->peer_list4, 0, sizeof(ot_peerlist)); + } + + peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + + /* Check for peer in torrent */ + peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer, peer_size, &exactmatch); + if (!peer_dest) { + mutex_bucket_unlock_by_hash(hash, 0); + return -1; + } + /* Tell peer that it's fresh */ + OT_PEERTIME(peer, peer_size) = 0; + + /* If we hadn't had a match create peer there */ + if (!exactmatch) { + peer_list->peer_count++; + if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_SEEDING) + peer_list->seed_count++; + } + memcpy(peer_dest, peer, peer_size); + mutex_bucket_unlock_by_hash(hash, 0); + return 0; +} + +size_t remove_peer_from_torrent_proxy(ot_hash hash, ot_peer *peer, size_t peer_size) { + int exactmatch; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); + ot_torrent *torrent = vector_binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + + if (exactmatch) { + ot_peerlist *peer_list = peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + switch (vector_remove_peer(&peer_list->peers, peer, peer_size)) { + case 2: + peer_list->seed_count--; /* Intentional fallthrough */ + case 1: + peer_list->peer_count--; /* Intentional fallthrough */ + default: + break; + } + } + + mutex_bucket_unlock_by_hash(hash, 0); + return 0; +} + +void free_peerlist(ot_peerlist *peer_list) { + if (peer_list->peers.data) { + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + ot_vector *bucket_list = (ot_vector *)(peer_list->peers.data); + + while (peer_list->peers.size--) + free(bucket_list++->data); + } + free(peer_list->peers.data); + } + free(peer_list); +} + +static void livesync_handle_peersync(ssize_t datalen, size_t peer_size) { + int off = sizeof(g_tracker_id) + sizeof(uint32_t); + + fprintf(stderr, "."); + + while ((ssize_t)(off + sizeof(ot_hash) + peer_size) <= datalen) { + ot_peer *peer = (ot_peer *)(g_inbuffer + off + sizeof(ot_hash)); + ot_hash *hash = (ot_hash *)(g_inbuffer + off); + + if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_STOPPED) + remove_peer_from_torrent_proxy(*hash, peer, peer_size); + else + add_peer_to_torrent_proxy(*hash, peer, peer_size); + + off += sizeof(ot_hash) + peer_size; + } +} + +int usage(char *self) { + fprintf(stderr, "Usage: %s -L -l : -c :\n", self); + return 0; +} + +enum { + FLAG_OUTGOING = 0x80, + + FLAG_DISCONNECTED = 0x00, + FLAG_CONNECTING = 0x01, + FLAG_WAITTRACKERID = 0x02, + FLAG_CONNECTED = 0x03, + + FLAG_MASK = 0x07 +}; + +#define PROXYPEER_NEEDSCONNECT(flag) ((flag) == FLAG_OUTGOING) +#define PROXYPEER_ISCONNECTED(flag) (((flag) & FLAG_MASK) == FLAG_CONNECTED) +#define PROXYPEER_SETDISCONNECTED(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_DISCONNECTED) +#define PROXYPEER_SETCONNECTING(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_CONNECTING) +#define PROXYPEER_SETWAITTRACKERID(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_WAITTRACKERID) +#define PROXYPEER_SETCONNECTED(flag) (flag) = (((flag) & FLAG_OUTGOING) | FLAG_CONNECTED) + +typedef struct { + int state; /* Whether we want to connect, how far our handshake is, etc. */ + ot_ip6 ip; /* The peer to connect to */ + uint16_t port; /* The peers port */ + uint8_t indata[8192 * 16]; /* Any data not processed yet */ + size_t indata_length; /* Length of unprocessed data */ + uint32_t tracker_id; /* How the other end greeted */ + int64_t fd; /* A file handle, if connected, <= 0 is disconnected (0 initially, -1 else) */ + ot_batch outdata; /* The batch containing our sync data */ + + size_t packet_tcount; /* Number of unprocessed torrents in packet we currently receive */ + uint8_t packet_tprefix; /* Prefix byte for all torrents in current packet */ + uint8_t packet_type; /* Type of current packet */ + uint32_t packet_tid; /* Tracker id for current packet */ + +} proxy_peer; +static void process_indata(proxy_peer *peer); + +void reset_info_block(proxy_peer *peer) { + peer->indata_length = 0; + peer->tracker_id = 0; + peer->fd = -1; + peer->packet_tcount = 0; + batch_reset(&peer->outdata); + PROXYPEER_SETDISCONNECTED(peer->state); +} + +/* Number of connections to peers + * If a peer's IP is set, we try to reconnect, when the connection drops + * If we already have a connected tracker_id in our records for an _incoming_ connection, drop it + * Multiple connections to/from the same ip are okay, if tracker_id doesn't match + * Reconnect attempts occur only twice a minute + */ +static size_t g_connection_count; +static ot_time g_connection_reconn; +static proxy_peer g_connections[MAX_PEERS]; + +static void handle_reconnects(void) { + size_t i; + for (i = 0; i < g_connection_count; ++i) + if (PROXYPEER_NEEDSCONNECT(g_connections[i].state)) { + int64_t newfd = socket_tcp_connect(g_serverip, g_serverport, g_connections[i].ip, g_connections[i].port); + fprintf(stderr, "(Re)connecting to peer..."); + if (newfd == -1) + continue; /* No socket for you */ + io_fd(newfd); + io_wantwrite(newfd); /* So we will be informed when it is connected */ + fdstate_open(newfd, g_connections + i, 0); + + /* Prepare connection info block */ + reset_info_block(g_connections + i); + g_connections[i].fd = newfd; + PROXYPEER_SETCONNECTING(g_connections[i].state); + } + g_connection_reconn = time(NULL) + 30; +} + +/* Handle incoming connection requests, check against whitelist */ +static void handle_accept(int64_t serversocket) { + int64_t newfd; + ot_ip6 ip; + uint16_t port; + + while ((newfd = socket_accept(serversocket, ip, &port)) != -1) { + + /* XXX some access control */ + + /* Put fd into a non-blocking mode */ + io_nonblock(newfd); + + if (!io_fd(newfd)) + io_close(newfd); + else { + /* Find a new home for our incoming connection */ + int i; + for (i = 0; i < MAX_PEERS; ++i) + if (g_connections[i].state == FLAG_DISCONNECTED) + break; + if (i == MAX_PEERS) { + fprintf(stderr, "No room for incoming connection."); + close(newfd); + continue; + } + + /* Prepare connection info block */ + reset_info_block(g_connections + i); + PROXYPEER_SETCONNECTING(g_connections[i].state); + g_connections[i].port = port; + g_connections[i].fd = newfd; + + fdstate_open(newfd, g_connections + i, 0); + + /* We expect the connecting side to begin with its tracker_id */ + io_wantread(newfd); + } + } + + return; +} + +/* New sync data on the stream */ +static void handle_read(int64_t peersocket) { + int i; + int64_t datalen; + uint32_t tracker_id; + proxy_peer *peer = fdstate_cookie(peersocket); + + if (!peer) { + /* Can't happen ;) */ + io_close(peersocket); + return; + } + switch (peer->state & FLAG_MASK) { + case FLAG_DISCONNECTED: + io_close(peersocket); + break; /* Shouldnt happen */ + case FLAG_CONNECTING: + case FLAG_WAITTRACKERID: + /* We want at least the first four bytes to come at once, to avoid keeping extra states (for now) + This also catches 0 bytes reads == EOF and negative values, denoting connection errors */ + if (io_tryread(peersocket, (void *)&tracker_id, sizeof(tracker_id)) != sizeof(tracker_id)) + goto close_socket; + + /* See, if we already have a connection to that peer */ + for (i = 0; i < MAX_PEERS; ++i) + if ((g_connections[i].state & FLAG_MASK) == FLAG_CONNECTED && g_connections[i].tracker_id == tracker_id) { + fprintf(stderr, "Peer already connected. Closing connection.\n"); + goto close_socket; + } + + /* Also no need for soliloquy */ + if (tracker_id == g_tracker_id) + goto close_socket; + + /* The new connection is good, send our tracker_id on incoming connections */ + if (peer->state == FLAG_CONNECTING) + if (io_trywrite(peersocket, (void *)&g_tracker_id, sizeof(g_tracker_id)) != sizeof(g_tracker_id)) + goto close_socket; + + peer->tracker_id = tracker_id; + PROXYPEER_SETCONNECTED(peer->state); + + if (peer->state & FLAG_OUTGOING) + fprintf(stderr, "succeeded.\n"); + else + fprintf(stderr, "Incoming connection successful.\n"); + + break; + close_socket: + fprintf(stderr, "Handshake incomplete, closing socket\n"); + io_close(peersocket); + reset_info_block(peer); + break; + case FLAG_CONNECTED: + /* Here we acutally expect data from peer + indata_length should be less than 20+256*7 bytes, for incomplete torrent entries */ + datalen = io_tryread(peersocket, (void *)(peer->indata + peer->indata_length), sizeof(peer->indata) - peer->indata_length); + if (!datalen || datalen < -1) { + fprintf(stderr, "Connection closed by remote peer.\n"); + io_close(peersocket); + reset_info_block(peer); + } else if (datalen > 0) { + peer->indata_length += datalen; + process_indata(peer); + } + break; + } +} + +/* Can write new sync data to the stream */ +static void handle_write(int64_t peersocket) { + proxy_peer *peer = fdstate_cookie(peersocket); + + if (!peer) { + /* Can't happen ;) */ + io_close(peersocket); + return; + } + + switch (peer->state & FLAG_MASK) { + case FLAG_DISCONNECTED: + default: /* Should not happen */ + io_close(peersocket); + break; + case FLAG_CONNECTING: + /* Ensure that the connection is established and handle connection error */ + if (peer->state & FLAG_OUTGOING && !socket_is_connected(peersocket)) { + fprintf(stderr, "failed\n"); + reset_info_block(peer); + io_close(peersocket); + break; + } + + if (io_trywrite(peersocket, (void *)&g_tracker_id, sizeof(g_tracker_id)) == sizeof(g_tracker_id)) { + PROXYPEER_SETWAITTRACKERID(peer->state); + io_dontwantwrite(peersocket); + io_wantread(peersocket); + } else { + fprintf(stderr, "Handshake incomplete, closing socket\n"); + io_close(peersocket); + reset_info_block(peer); + } + break; + case FLAG_CONNECTED: + switch (batch_send(peersocket, &peer->outdata)) { + case BATCH_DONE: + io_dontwantwrite(peersocket); + break; + case BATCH_FAILED: + io_close(peersocket); + reset_info_block(peer); + break; + default: /* Normal operation or eagain */ + break; + } + break; + } + + return; +} + +static void server_mainloop() { + int64_t sock; + uint32_t packet_type; + + /* inlined livesync_init() */ + memset(g_peerbuffer_start, 0, sizeof(g_peerbuffer_start)); + g_peerbuffer_pos = g_peerbuffer_start; + memcpy(g_peerbuffer_pos, &g_tracker_id, sizeof(g_tracker_id)); + packet_type = htonl(OT_SYNC_PEER); + memcpy(g_peerbuffer_pos + sizeof(g_tracker_id), &packet_type, sizeof(packet_type)); + g_peerbuffer_pos += sizeof(g_tracker_id) + sizeof(uint32_t); + g_next_packet_time = time(NULL) + LIVESYNC_MAXDELAY; + + while (1) { + /* See if we need to connect to anyone */ + if (time(NULL) > g_connection_reconn) + handle_reconnects(); + + /* Wait for io events until next approx reconn check time */ + io_waituntil2(30 * 1000); + + /* Loop over readable sockets */ + while ((sock = io_canread()) != -1) { + const void *cookie = fdstate_cookie(sock); + if ((uintptr_t)cookie == FLAG_SERVERSOCKET) + handle_accept(sock); + else + handle_read(sock); + } + + /* Loop over writable sockets */ + while ((sock = io_canwrite()) != -1) + handle_write(sock); + + livesync_ticker(); + } +} + +static void panic(const char *routine) { + fprintf(stderr, "%s: %s\n", routine, strerror(errno)); + exit(111); +} + +static int64_t ot_try_bind(ot_ip6 ip, uint16_t port) { + int64_t sock = socket_tcp_listen(ip, port, SOCKET_EXCLUSIVE); + + if (sock == -1) + panic("socket_tcp_listen"); + + if (!io_fd(sock)) + panic("io_fd"); + + fdstate_open(sock, (void *)FLAG_SERVERSOCKET, 0); + io_wantread(sock); + return sock; +} + +static int scan_ip6_port(const char *src, ot_ip6 ip, uint16_t *port) { + const char *s = src; + uint64_t value; + int off, bracket = 0; + while (isspace(*s)) + ++s; + if (*s == '[') + ++s, ++bracket; /* for v6 style notation */ + if (!(off = ip_parse(s, ip))) + return 0; + s += off; + if (*s == 0 || isspace(*s)) + return s - src; + if (*s == ']' && bracket) + ++s; + if (!ip_is_v4mapped(ip)) { + if ((bracket && *(s) != ':') || (*(s) != '.')) + return 0; + s++; + } else { + if (*(s++) != ':') + return 0; + } + if (!(off = scan_uint(s, &value, 65535))) + return 0; + *port = (uint16_t)value; + return off + s - src; +} + +int main(int argc, char **argv) { + static pthread_t sync_in_thread_id; + static pthread_t sync_out_thread_id; + ot_ip6 serverip; + uint16_t tmpport; + int scanon = 1, lbound = 0, sbound = 0; + + random_init(); + if (fdstate_init() == -1) + panic("fdstate_init"); + g_tracker_id = random_uint32(); + + while (scanon) { + switch (getopt(argc, argv, ":l:c:L:h")) { + case -1: + scanon = 0; + break; + case 'l': + tmpport = 0; + if (!scan_ip6_port(optarg, serverip, &tmpport) || !tmpport) { + usage(argv[0]); + exit(1); + } + ot_try_bind(serverip, tmpport); + ++sbound; + break; + case 'c': + if (g_connection_count > MAX_PEERS / 2) + exerr("Connection limit exceeded.\n"); + tmpport = 0; + if (!scan_ip6_port(optarg, g_connections[g_connection_count].ip, &g_connections[g_connection_count].port) || !g_connections[g_connection_count].port) { + usage(argv[0]); + exit(1); + } + g_connections[g_connection_count++].state = FLAG_OUTGOING; + break; + case 'L': + tmpport = 9696; + if (!scan_ip6_port(optarg, serverip, &tmpport) || !tmpport) { + usage(argv[0]); + exit(1); + } + livesync_bind_mcast(serverip, tmpport); + ++lbound; + break; + default: + case '?': + usage(argv[0]); + exit(1); + } + } + + if (!lbound) + exerr("No livesync port bound."); + if (!g_connection_count && !sbound) + exerr("No streamsync port bound."); + pthread_create(&sync_in_thread_id, NULL, livesync_worker, NULL); + pthread_create(&sync_out_thread_id, NULL, streamsync_worker, NULL); + + server_mainloop(); + return 0; +} + +static void *streamsync_worker(void *args) { + (void)args; + while (1) { + size_t bucket; + /* For each bucket... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + /* Get exclusive access to that bucket */ + ot_vector *torrents_list = mutex_bucket_lock(bucket); + size_t tor_offset, count_def = 0, count_one = 0, count_two = 0, count_peers = 0; + size_t mem, mem_a = 0, mem_b = 0; + uint8_t *ptr = 0, *ptr_a, *ptr_b, *ptr_c; + + if (!torrents_list->size) + goto unlock_continue; + + /* For each torrent in this bucket.. */ + for (tor_offset = 0; tor_offset < torrents_list->size; ++tor_offset) { + /* Address torrents members */ + ot_peerlist *peer_list = (((ot_torrent *)(torrents_list->data))[tor_offset]).peer_list; + switch (peer_list->peer_count) { + case 2: + count_two++; + break; + case 1: + count_one++; + break; + case 0: + break; + default: + count_def++; + count_peers += peer_list->peer_count; + } + } + + /* Maximal memory requirement: max 3 blocks, max torrents * 20 + max peers * 7 */ + mem = 3 * (1 + 1 + 2) + (count_one + count_two) * (19 + 1) + count_def * (19 + 8) + (count_one + 2 * count_two + count_peers) * 7; + + fprintf(stderr, "Mem: %zd\n", mem); + + ptr = ptr_a = ptr_b = ptr_c = malloc(mem); + if (!ptr) + goto unlock_continue; + + if (count_one > 4 || !count_def) { + mem_a = 1 + 1 + 2 + count_one * (19 + 7); + ptr_b += mem_a; + ptr_c += mem_a; + ptr_a[0] = 1; /* Offset 0: packet type 1 */ + ptr_a[1] = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */ + ptr_a[2] = count_one >> 8; + ptr_a[3] = count_one & 255; + ptr_a += 4; + } else + count_def += count_one; + + if (count_two > 4 || !count_def) { + mem_b = 1 + 1 + 2 + count_two * (19 + 14); + ptr_c += mem_b; + ptr_b[0] = 2; /* Offset 0: packet type 2 */ + ptr_b[1] = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */ + ptr_b[2] = count_two >> 8; + ptr_b[3] = count_two & 255; + ptr_b += 4; + } else + count_def += count_two; + + if (count_def) { + ptr_c[0] = 0; /* Offset 0: packet type 0 */ + ptr_c[1] = (bucket << 8) >> OT_BUCKET_COUNT_BITS; /* Offset 1: the shared prefix */ + ptr_c[2] = count_def >> 8; + ptr_c[3] = count_def & 255; + ptr_c += 4; + } + + /* For each torrent in this bucket.. */ + for (tor_offset = 0; tor_offset < torrents_list->size; ++tor_offset) { + /* Address torrents members */ + ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + tor_offset; + ot_peerlist *peer_list = torrent->peer_list; + ot_peer *peers = (ot_peer *)(peer_list->peers.data); + uint8_t **dst; + + /* Determine destination slot */ + count_peers = peer_list->peer_count; + switch (count_peers) { + case 0: + continue; + case 1: + dst = mem_a ? &ptr_a : &ptr_c; + break; + case 2: + dst = mem_b ? &ptr_b : &ptr_c; + break; + default: + dst = &ptr_c; + break; + } + + /* Copy tail of info_hash, advance pointer */ + memcpy(*dst, ((uint8_t *)torrent->hash) + 1, sizeof(ot_hash) - 1); + *dst += sizeof(ot_hash) - 1; + + /* Encode peer count */ + if (dst == &ptr_c) + while (count_peers) { + if (count_peers <= 0x7f) + *(*dst)++ = count_peers; + else + *(*dst)++ = 0x80 | (count_peers & 0x7f); + count_peers >>= 7; + } + + /* Copy peers */ + count_peers = peer_list->peer_count; + while (count_peers--) { + memcpy(*dst, peers++, OT_IP_SIZE + 3); + *dst += OT_IP_SIZE + 3; + } + free_peerlist(peer_list); + } + + free(torrents_list->data); + memset(torrents_list, 0, sizeof(*torrents_list)); + unlock_continue: + mutex_bucket_unlock(bucket, 0); + + if (ptr) { + int i; + + if (ptr_b > ptr_c) + ptr_c = ptr_b; + if (ptr_a > ptr_c) + ptr_c = ptr_a; + mem = ptr_c - ptr; + + for (i = 0; i < MAX_PEERS; ++i) { + if (PROXYPEER_ISCONNECTED(g_connections[i].state)) { + void *tmp = malloc(mem); + if (tmp) { + memcpy(tmp, ptr, mem); + batch_add(&g_connections[i].outdata, tmp, mem); + io_wantwrite(g_connections[i].fd); + } + } + } + + free(ptr); + } + usleep(OT_SYNC_SLEEP); + } + } + return 0; +} + +static void livesync_issue_peersync() { + socket_mcast_send(g_socket_out, g_peerbuffer_start, (size_t)(g_peerbuffer_pos - g_peerbuffer_start), groupip_1, LIVESYNC_PORT); + g_peerbuffer_pos = g_peerbuffer_start + sizeof(g_tracker_id) + sizeof(uint32_t); + g_next_packet_time = time(NULL) + LIVESYNC_MAXDELAY; +} + +void livesync_ticker() { + /* livesync_issue_peersync sets g_next_packet_time */ + if (time(NULL) > g_next_packet_time && g_peerbuffer_pos > g_peerbuffer_start + sizeof(g_tracker_id)) + livesync_issue_peersync(); +} + +static void livesync_proxytell(uint8_t prefix, uint8_t *info_hash, uint8_t *peer) { + // unsigned int i; + + *g_peerbuffer_pos = prefix; + memcpy(g_peerbuffer_pos + 1, info_hash, sizeof(ot_hash) - 1); + memcpy(g_peerbuffer_pos + sizeof(ot_hash), peer, sizeof(ot_peer) - 1); + +#if 0 + /* Dump info_hash */ + for( i=0; i= g_peerbuffer_highwater) + livesync_issue_peersync(); +} + +static void process_indata(proxy_peer *peer) { + size_t consumed, peers; + uint8_t *data = peer->indata, *hash; + uint8_t *dataend = data + peer->indata_length; + + while (1) { + /* If we're not inside of a packet, make a new one */ + if (!peer->packet_tcount) { + /* Ensure the header is complete or postpone processing */ + if (data + 4 > dataend) + break; + peer->packet_type = data[0]; + peer->packet_tprefix = data[1]; + peer->packet_tcount = data[2] * 256 + data[3]; + data += 4; + printf("type: %hhu, prefix: %02X, torrentcount: %zd\n", peer->packet_type, peer->packet_tprefix, peer->packet_tcount); + } + + /* Ensure size for a minimal torrent block */ + if (data + sizeof(ot_hash) + OT_IP_SIZE + 3 > dataend) + break; + + /* Advance pointer to peer count or peers */ + hash = data; + data += sizeof(ot_hash) - 1; + + /* Type 0 has peer count encoded before each peers */ + peers = peer->packet_type; + if (!peers) { + int shift = 0; + do + peers |= (0x7f & *data) << (7 * shift); + while (*(data++) & 0x80 && shift++ < 6); + } +#if 0 +printf( "peers: %zd\n", peers ); +#endif + /* Ensure enough data being read to hold all peers */ + if (data + (OT_IP_SIZE + 3) * peers > dataend) { + data = hash; + break; + } + while (peers--) { + livesync_proxytell(peer->packet_tprefix, hash, data); + data += OT_IP_SIZE + 3; + } + --peer->packet_tcount; + } + + consumed = data - peer->indata; + memmove(peer->indata, data, peer->indata_length - consumed); + peer->indata_length -= consumed; +} + +static void *livesync_worker(void *args) { + (void)args; + while (1) { + ot_ip6 in_ip; + uint16_t in_port; + uint32_t packet_type; + ssize_t datalen = socket_recv(g_socket_in, g_inbuffer, LIVESYNC_INCOMING_BUFFSIZE, in_ip, &in_port); + + /* Expect at least tracker id and packet type */ + if (datalen <= (ssize_t)(sizeof(g_tracker_id) + sizeof(uint32_t))) + continue; + if (!memcmp(g_inbuffer, &g_tracker_id, sizeof(g_tracker_id))) { + /* drop packet coming from ourselves */ + continue; + } + memcpy(&packet_type, g_inbuffer + sizeof(g_tracker_id), sizeof(packet_type)); + + switch (ntohl(packet_type)) { + case OT_SYNC_PEER4: + livesync_handle_peersync(datalen, OT_PEER_SIZE4); + break; + case OT_SYNC_PEER6: + livesync_handle_peersync(datalen, OT_PEER_SIZE6); + break; + default: + // fprintf( stderr, "Received an unknown live sync packet type %u.\n", ntohl( packet_type ) ); + break; + } + } + return 0; +} diff --git a/src/trackerlogic.c b/src/trackerlogic.c new file mode 100644 index 0000000..9565090 --- /dev/null +++ b/src/trackerlogic.c @@ -0,0 +1,628 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include + + +/* Opentracker */ +#include "ot_accesslist.h" +#include "ot_clean.h" +#include "ot_fmt.h" +#include "ot_fullscrape.h" +#include "ot_http.h" +#include "ot_ip.h" +#include "ot_livesync.h" +#include "ot_mutex.h" +#include "ot_workqueue.h" +#include "ot_random.h" +#include "ot_stats.h" +#include "ot_udp.h" +#include "ot_vector.h" +#include "trackerlogic.h" + +/* Forward declaration */ +size_t return_peers_for_torrent(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto); + +void free_peerlist(ot_peerlist *peer_list) { + if (peer_list && peer_list->peers.data) { + if (OT_PEERLIST_HASBUCKETS(peer_list)) + vector_clean_list((ot_vector *)peer_list->peers.data, peer_list->peers.size); + else + free(peer_list->peers.data); + } + free(peer_list); +} + +void add_torrent_from_saved_state(ot_hash const hash, ot_time base, size_t down_count) { + int exactmatch; + ot_torrent *torrent; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); + + if (!accesslist_hashisvalid(hash)) { + mutex_bucket_unlock_by_hash(hash, 0); + return; + } + + torrent = vector_find_or_insert(torrents_list, (void *)hash, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + if (!torrent || exactmatch) { + mutex_bucket_unlock_by_hash(hash, 0); + return; + } + + /* Create a new torrent entry, then */ + memset(torrent, 0, sizeof(ot_torrent)); + memcpy(torrent->hash, hash, sizeof(ot_hash)); + + if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) { + vector_remove_torrent(torrents_list, torrent); + mutex_bucket_unlock_by_hash(hash, 0); + return; + } + + memset(torrent->peer_list6, 0, sizeof(ot_peerlist)); + memset(torrent->peer_list4, 0, sizeof(ot_peerlist)); + torrent->peer_list6->base = base; + torrent->peer_list4->base = base; + torrent->peer_list6->down_count = down_count; + torrent->peer_list4->down_count = down_count; + + mutex_bucket_unlock_by_hash(hash, 1); +} + +size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstruct *ws, size_t amount) { + int exactmatch, delta_torrentcount = 0; + ot_torrent *torrent; + ot_peer *peer_dest; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(*ws->hash); + ot_peerlist *peer_list; + size_t peer_size; /* initialized in next line */ + ot_peer const *peer_src = peer_from_peer6(&ws->peer, &peer_size); + + if (!accesslist_hashisvalid(*ws->hash)) { + mutex_bucket_unlock_by_hash(*ws->hash, 0); + if (proto == FLAG_TCP) { + const char invalid_hash[] = "d14:failure reason63:Requested download is not authorized for use with this tracker.e"; + memcpy(ws->reply, invalid_hash, strlen(invalid_hash)); + return strlen(invalid_hash); + } + return 0; + } + + torrent = vector_find_or_insert(torrents_list, (void *)ws->hash, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + if (!torrent) { + mutex_bucket_unlock_by_hash(*ws->hash, 0); + return 0; + } + + if (!exactmatch) { + /* Create a new torrent entry, then */ + memset(torrent, 0, sizeof(ot_torrent)); + memcpy(torrent->hash, *ws->hash, sizeof(ot_hash)); + + if (!(torrent->peer_list6 = malloc(sizeof(ot_peerlist))) || !(torrent->peer_list4 = malloc(sizeof(ot_peerlist)))) { + vector_remove_torrent(torrents_list, torrent); + mutex_bucket_unlock_by_hash(*ws->hash, 0); + return 0; + } + + memset(torrent->peer_list6, 0, sizeof(ot_peerlist)); + memset(torrent->peer_list4, 0, sizeof(ot_peerlist)); + delta_torrentcount = 1; + } else + clean_single_torrent(torrent); + + torrent->peer_list6->base = time_now_minutes(); + torrent->peer_list4->base = time_now_minutes(); + + peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + + /* Check for peer in torrent */ + peer_dest = vector_find_or_insert_peer(&(peer_list->peers), peer_src, peer_size, &exactmatch); + if (!peer_dest) { + mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount); + return 0; + } + + /* Tell peer that it's fresh */ + OT_PEERTIME(ws->peer, OT_PEER_SIZE6) = 0; + + /* Sanitize flags: Whoever claims to have completed download, must be a seeder */ + if ((OT_PEERFLAG(ws->peer) & (PEER_FLAG_COMPLETED | PEER_FLAG_SEEDING)) == PEER_FLAG_COMPLETED) + OT_PEERFLAG(ws->peer) ^= PEER_FLAG_COMPLETED; + + /* If we hadn't had a match create peer there */ + if (!exactmatch) { + +#ifdef WANT_SYNC_LIVE + if (proto == FLAG_MCA) + OT_PEERFLAG(ws->peer) |= PEER_FLAG_FROM_SYNC; + else + livesync_tell(ws); +#endif + + peer_list->peer_count++; + if (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED) { + peer_list->down_count++; + stats_issue_event(ws, EVENT_COMPLETED, 0, 0); + } + if (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING) + peer_list->seed_count++; + + } else { + stats_issue_event(ws, EVENT_RENEW, 0, OT_PEERTIME(peer_dest, peer_size)); +#ifdef WANT_SPOT_WOODPECKER + if ((OT_PEERTIME(peer_dest, peer_size) > 0) && (OT_PEERTIME(peer_dest, peer_size) < 20)) + stats_issue_event(ws, EVENT_WOODPECKER, 0, (uintptr_t)&ws->peer); +#endif +#ifdef WANT_SYNC_LIVE + /* Won't live sync peers that come back too fast. Only exception: + fresh "completed" reports */ + if (proto != FLAG_MCA) { + if (OT_PEERTIME(peer_dest, peer_size) > OT_CLIENT_SYNC_RENEW_BOUNDARY || + (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED))) + livesync_tell(ws); + } +#endif + + if ((OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && !(OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING)) + peer_list->seed_count--; + if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_SEEDING) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_SEEDING)) + peer_list->seed_count++; + if (!(OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) && (OT_PEERFLAG(ws->peer) & PEER_FLAG_COMPLETED)) { + peer_list->down_count++; + stats_issue_event(ws, EVENT_COMPLETED, 0, 0); + } + if (OT_PEERFLAG_D(peer_dest, peer_size) & PEER_FLAG_COMPLETED) + OT_PEERFLAG(ws->peer) |= PEER_FLAG_COMPLETED; + } + + memcpy(peer_dest, peer_src, peer_size); +#ifdef WANT_SYNC + if (proto == FLAG_MCA) { + mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount); + return 0; + } +#endif + + ws->reply_size = return_peers_for_torrent(ws, torrent, amount, ws->reply, proto); + mutex_bucket_unlock_by_hash(*ws->hash, delta_torrentcount); + return ws->reply_size; +} + +static size_t return_peers_all(ot_peerlist *peer_list, size_t peer_size, char *reply) { + size_t bucket, num_buckets = 1; + ot_vector *bucket_list = &peer_list->peers; + size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + size_t result = compare_size * peer_list->peer_count; + char *r_end = reply + result; + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets = bucket_list->size; + bucket_list = (ot_vector *)bucket_list->data; + } + + for (bucket = 0; bucket < num_buckets; ++bucket) { + ot_peer *peers = bucket_list[bucket].data; + size_t peer_count = bucket_list[bucket].size; + while (peer_count--) { + if (OT_PEERFLAG_D(peers, peer_size) & PEER_FLAG_SEEDING) { + r_end -= compare_size; + memcpy(r_end, peers, compare_size); + } else { + memcpy(reply, peers, compare_size); + reply += compare_size; + } + peers += peer_size; + } + } + return result; +} + +static size_t return_peers_selection(struct ot_workstruct *ws, ot_peerlist *peer_list, size_t peer_size, size_t amount, char *reply) { + size_t bucket_offset, bucket_index = 0, num_buckets = 1; + ot_vector *bucket_list = &peer_list->peers; + unsigned int shifted_pc = peer_list->peer_count; + unsigned int shifted_step = 0; + unsigned int shift = 0; + size_t compare_size = OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(peer_size); + size_t result = compare_size * amount; + char *r_end = reply + result; + + if (OT_PEERLIST_HASBUCKETS(peer_list)) { + num_buckets = bucket_list->size; + bucket_list = (ot_vector *)bucket_list->data; + } + + /* Make fixpoint arithmetic as exact as possible */ +#define MAXPRECBIT (1 << (8 * sizeof(int) - 3)) + while (!(shifted_pc & MAXPRECBIT)) { + shifted_pc <<= 1; + shift++; + } + shifted_step = shifted_pc / amount; +#undef MAXPRECBIT + + /* Initialize somewhere in the middle of peers so that + fixpoint's aliasing doesn't alway miss the same peers */ + bucket_offset = nrand48(ws->rand48_state) % peer_list->peer_count; + + while (amount--) { + ot_peer *peer; + + /* This is the aliased, non shifted range, next value may fall into */ + unsigned int diff = (((amount + 1) * shifted_step) >> shift) - ((amount * shifted_step) >> shift); + bucket_offset += 1 + nrand48(ws->rand48_state) % diff; + + while (bucket_offset >= bucket_list[bucket_index].size) { + bucket_offset -= bucket_list[bucket_index].size; + bucket_index = (bucket_index + 1) % num_buckets; + } + peer = (ot_peer*)bucket_list[bucket_index].data + peer_size * bucket_offset; + if (OT_PEERFLAG_D(peer, peer_size) & PEER_FLAG_SEEDING) { + r_end -= compare_size; + memcpy(r_end, peer, compare_size); + } else { + memcpy(reply, peer, compare_size); + reply += compare_size; + } + } + return result; +} + +static size_t return_peers_for_torrent_udp(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply) { + char *r = reply; + size_t peer_size = peer_size_from_peer6(&ws->peer); + ot_peerlist *peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + + if (amount > peer_list->peer_count) + amount = peer_list->peer_count; + + *(uint32_t *)(r + 0) = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM); + *(uint32_t *)(r + 4) = htonl(peer_count - seed_count); + *(uint32_t *)(r + 8) = htonl(seed_count); + r += 12; + + if (amount) { + if (amount == peer_list->peer_count) + r += return_peers_all(peer_list, peer_size, r); + else + r += return_peers_selection(ws, peer_list, peer_size, amount, r); + } + return r - reply; +} + +static size_t return_peers_for_torrent_tcp(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply) { + char *r = reply; + const char *r_end = ws->outbuf + G_OUTBUF_SIZE; + int erval = OT_CLIENT_REQUEST_INTERVAL_RANDOM; + size_t seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + size_t down_count = torrent->peer_list6->down_count + torrent->peer_list4->down_count; + size_t peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - seed_count; + + /* Simple case: amount of peers in both lists is less than requested, here we return all results */ + size_t amount_v4 = torrent->peer_list4->peer_count; + size_t amount_v6 = torrent->peer_list6->peer_count; + + /* Complex case: both lists have more than enough entries and we need to split between v4 and v6 clients */ + if (amount_v4 + amount_v6 > amount) { + size_t amount_left, percent_v6 = 0, percent_v4 = 0, left_v6, left_v4; + const size_t SCALE = 1024; + + /* If possible, fill at least a quarter of peer from each family */ + if (amount / 4 <= amount_v4) + amount_v4 = amount / 4; + if (amount / 4 <= amount_v6) + amount_v6 = amount / 4; + + /* Fill the rest according to which family's pool provides more peers */ + amount_left = amount - (amount_v4 + amount_v6); + + left_v4 = torrent->peer_list4->peer_count - amount_v4; + left_v6 = torrent->peer_list6->peer_count - amount_v6; + + if (left_v4 + left_v6) { + percent_v4 = (SCALE * left_v4) / (left_v4 + left_v6); + percent_v6 = (SCALE * left_v6) / (left_v4 + left_v6); + } + + amount_v4 += (amount_left * percent_v4) / SCALE; + amount_v6 += (amount_left * percent_v6) / SCALE; + + /* Integer division rounding can leave out a peer */ + if (amount_v4 + amount_v6 < amount && amount_v6 < torrent->peer_list6->peer_count) + ++amount_v6; + if (amount_v4 + amount_v6 < amount && amount_v4 < torrent->peer_list4->peer_count) + ++amount_v4; + } + + fmt_append(&r, r_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zde8:intervali%ie12:min intervali%ie", seed_count, down_count, peer_count, erval, + erval / 2); + + if (amount_v4) { + fmt_append(&r, r_end, PEERS_BENCODED4 "%zd:", OT_PEER_COMPARE_SIZE4 * amount_v4); + if (amount_v4 == torrent->peer_list4->peer_count) + r += return_peers_all(torrent->peer_list4, OT_PEER_SIZE4, r); + else + r += return_peers_selection(ws, torrent->peer_list4, OT_PEER_SIZE4, amount_v4, r); + } + + if (amount_v6) { + fmt_append(&r, r_end, PEERS_BENCODED6 "%zd:", OT_PEER_COMPARE_SIZE6 * amount_v6); + if (amount_v6 == torrent->peer_list6->peer_count) + r += return_peers_all(torrent->peer_list6, OT_PEER_SIZE6, r); + else + r += return_peers_selection(ws, torrent->peer_list6, OT_PEER_SIZE6, amount_v6, r); + } + + fmt_append(&r, r_end, "e"); + + return r - reply; +} + +/* Compiles a list of random peers for a torrent + * Reply must have enough space to hold: + * 92 + 6 * amount bytes for TCP/IPv4 + * 92 + 18 * amount bytes for TCP/IPv6 + * 12 + 6 * amount bytes for UDP/IPv4 + * 12 + 18 * amount bytes for UDP/IPv6 + * Does not yet check not to return self + */ +size_t return_peers_for_torrent(struct ot_workstruct *ws, ot_torrent *torrent, size_t amount, char *reply, PROTO_FLAG proto) { + return proto == FLAG_TCP ? return_peers_for_torrent_tcp(ws, torrent, amount, reply) : return_peers_for_torrent_udp(ws, torrent, amount, reply); +} + +/* Fetches scrape info for a specific torrent */ +size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply) { + int exactmatch, delta_torrentcount = 0; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(hash); + ot_torrent *torrent = vector_binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + + if (!exactmatch) { + memset(reply, 0, 12); + } else { + uint32_t *r = (uint32_t *)reply; + + if (clean_single_torrent(torrent)) { + vector_remove_torrent(torrents_list, torrent); + memset(reply, 0, 12); + delta_torrentcount = -1; + } else { + r[0] = htonl(torrent->peer_list6->seed_count + torrent->peer_list4->seed_count); + r[1] = htonl(torrent->peer_list6->down_count + torrent->peer_list4->down_count); + r[2] = htonl(torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count); + } + } + mutex_bucket_unlock_by_hash(hash, delta_torrentcount); + return 12; +} + +/* Fetches scrape info for a specific torrent */ +void return_tcp_scrape_for_torrent(ot_hash const *hash_list, size_t amount, char **reply, const char *reply_end) { + int exactmatch; + size_t i; + + fmt_append(reply, reply_end, "d5:filesd"); + + for (i = 0; i < amount; ++i) { + int delta_torrentcount = 0; + ot_hash const *hash = hash_list + i; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(*hash); + ot_torrent *torrent = vector_binary_search(hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + + if (exactmatch) { + if (clean_single_torrent(torrent)) { + vector_remove_torrent(torrents_list, torrent); + delta_torrentcount = -1; + } else { + fmt_append(reply, reply_end, "20:"); + fmt_bytes(reply, reply_end, hash, sizeof(ot_hash)); + fmt_append(reply, reply_end, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", + torrent->peer_list6->seed_count + torrent->peer_list4->seed_count, torrent->peer_list6->down_count + torrent->peer_list4->down_count, + torrent->peer_list6->peer_count + torrent->peer_list4->peer_count - torrent->peer_list6->seed_count - torrent->peer_list4->seed_count); + } + } + mutex_bucket_unlock_by_hash(*hash, delta_torrentcount); + } + + fmt_append(reply, reply_end, "ee"); +} + +static ot_peerlist dummy_list; +size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws) { + int exactmatch; + ot_vector *torrents_list = mutex_bucket_lock_by_hash(*ws->hash); + ot_torrent *torrent = vector_binary_search(ws->hash, torrents_list->data, torrents_list->size, sizeof(ot_torrent), OT_HASH_COMPARE_SIZE, &exactmatch); + ot_peerlist *peer_list = &dummy_list; + size_t peer_size; /* initialized in next line */ + ot_peer const *peer_src = peer_from_peer6(&ws->peer, &peer_size); + size_t peer_count = 0, seed_count = 0; + +#ifdef WANT_SYNC_LIVE + if (proto != FLAG_MCA) { + OT_PEERFLAG(ws->peer) |= PEER_FLAG_STOPPED; + livesync_tell(ws); + } +#endif + + if (exactmatch) { + peer_list = peer_size == OT_PEER_SIZE6 ? torrent->peer_list6 : torrent->peer_list4; + switch (vector_remove_peer(&peer_list->peers, peer_src, peer_size)) { + case 2: + peer_list->seed_count--; /* Intentional fallthrough */ + case 1: + peer_list->peer_count--; /* Intentional fallthrough */ + default: + break; + } + + peer_count = torrent->peer_list6->peer_count + torrent->peer_list4->peer_count; + seed_count = torrent->peer_list6->seed_count + torrent->peer_list4->seed_count; + } + + if (proto == FLAG_TCP) { + int erval = OT_CLIENT_REQUEST_INTERVAL_RANDOM; + char *r = ws->reply; + + fmt_append(&r, ws->outbuf + G_OUTBUF_SIZE, "d8:completei%zde10:incompletei%zde8:intervali%ie12:min intervali%ie%s0:e", seed_count, peer_count - seed_count, + erval, erval / 2, peer_size == OT_PEER_SIZE6 ? PEERS_BENCODED6 : PEERS_BENCODED4); + ws->reply_size = r - ws->reply; + } + + /* Handle UDP reply */ + if (proto == FLAG_UDP) { + ((uint32_t *)ws->reply)[2] = htonl(OT_CLIENT_REQUEST_INTERVAL_RANDOM); + ((uint32_t *)ws->reply)[3] = htonl(peer_count - seed_count); + ((uint32_t *)ws->reply)[4] = htonl(seed_count); + ws->reply_size = 20; + } + + mutex_bucket_unlock_by_hash(*ws->hash, 0); + return ws->reply_size; +} + +void iterate_all_torrents(int (*for_each)(ot_torrent *torrent, uintptr_t data), uintptr_t data) { + size_t bucket, j; + + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + ot_vector *torrents_list = mutex_bucket_lock(bucket); + ot_torrent *torrents = (ot_torrent *)(torrents_list->data); + + for (j = 0; j < torrents_list->size; ++j) + if (for_each(torrents + j, data)) + break; + + mutex_bucket_unlock(bucket, 0); + if (!g_opentracker_running) + return; + } +} + +ot_peer *peer_from_peer6(ot_peer6 *peer, size_t *peer_size) { + ot_ip6 *ip = (ot_ip6 *)peer; + if (!ip_is_v4mapped(ip)) { + *peer_size = OT_PEER_SIZE6; + return (ot_peer *)peer; + } + *peer_size = OT_PEER_SIZE4; + return (ot_peer *)(((uint8_t *)peer) + 12); +} + +size_t peer_size_from_peer6(ot_peer6 *peer) { + ot_ip6 *ip = (ot_ip6 *)peer; + if (!ip_is_v4mapped(ip)) + return OT_PEER_SIZE6; + return OT_PEER_SIZE4; +} + +#ifdef _DEBUG_RANDOMTORRENTS +void trackerlogic_add_random_torrents(size_t amount) { + struct ot_workstruct ws; + memset(&ws, 0, sizeof(ws)); + + ws.inbuf = malloc(G_INBUF_SIZE); + ws.outbuf = malloc(G_OUTBUF_SIZE); + ws.reply = ws.outbuf; + ws.hash = (ot_hash *)ws.inbuf; + + while (amount--) { + random_bytes(ws.hash, sizeof(ot_hash)); + random_bytes(&ws.peer, sizeof(ws.peer)); + + OT_PEERFLAG(ws.peer) &= PEER_FLAG_SEEDING | PEER_FLAG_COMPLETED | PEER_FLAG_STOPPED; + + add_peer_to_torrent_and_return_peers(FLAG_TCP, &ws, 1); + } + + free(ws.inbuf); + free(ws.outbuf); +} +#endif + +void exerr(const char *message) { + fprintf(stderr, "%s\n", message); + exit(111); +} + +void trackerlogic_init(void) { + g_tracker_id = random_uint32(); + + if (!g_stats_path) + g_stats_path = "stats"; + g_stats_path_len = strlen(g_stats_path); + + /* Initialise background worker threads */ + mutex_init(); + workqueue_init(); + clean_init(); + fullscrape_init(); + accesslist_init(); + livesync_init(); + stats_init(); +} + +int workstruct_init(struct ot_workstruct *ws) { + memset(ws, 0, sizeof(*ws)); + ws->inbuf = malloc(G_INBUF_SIZE); + ws->outbuf = malloc(G_OUTBUF_SIZE); +#ifdef _DEBUG_HTTPERROR + ws->debugbuf = malloc(G_DEBUGBUF_SIZE); +#endif + if (!ws->inbuf || !ws->outbuf) + return -1; + + /* Picking which peers to hand out starts here, so two workers seeded alike + would hand out alike */ + random_bytes(ws->rand48_state, sizeof(ws->rand48_state)); + + stats_counters_register(ws); + return 0; +} + +void workstruct_deinit(struct ot_workstruct *ws) { + /* The tally goes to the global before this struct goes out of scope */ + stats_counters_retire(ws); + + free(ws->inbuf); + free(ws->outbuf); +#ifdef _DEBUG_HTTPERROR + free(ws->debugbuf); +#endif +} + +void trackerlogic_deinit(void) { + int delta_torrentcount = 0; + size_t bucket, j; + + /* Stop the background workers first */ + stats_deinit(); + livesync_deinit(); + accesslist_deinit(); + fullscrape_deinit(); + clean_deinit(); + + /* Free all torrents... */ + for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) { + ot_vector *torrents_list = mutex_bucket_lock(bucket); + if (torrents_list->size) { + for (j = 0; j < torrents_list->size; ++j) { + ot_torrent *torrent = ((ot_torrent *)(torrents_list->data)) + j; + free_peerlist(torrent->peer_list6); + free_peerlist(torrent->peer_list4); + delta_torrentcount -= 1; + } + free(torrents_list->data); + } + mutex_bucket_unlock(bucket, delta_torrentcount); + } + + /* Release mutexes */ + workqueue_deinit(); + mutex_deinit(); +} diff --git a/src/trackerlogic.h b/src/trackerlogic.h new file mode 100644 index 0000000..7292d36 --- /dev/null +++ b/src/trackerlogic.h @@ -0,0 +1,283 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef OT_TRACKERLOGIC_H__ +#define OT_TRACKERLOGIC_H__ + +#include +#include +#include +#include +#include +#include +#include + +typedef uint8_t ot_hash[20]; +typedef time_t ot_time; +typedef char ot_ip6[16]; +typedef struct { + ot_ip6 address; + int bits; +} ot_net; +/* List of peers should fit in a single UDP packet (around 1200 bytes) */ +#define OT_MAX_PEERS_UDP6 66 +#define OT_MAX_PEERS_UDP4 200 + +#define OT_IP_SIZE6 16 +#define OT_IP_SIZE4 4 +#define OT_PORT_SIZE 2 +#define OT_FLAG_SIZE 1 +#define OT_TIME_SIZE 1 + +/* Some tracker behaviour tunable */ +#ifndef OT_CLIENT_TIMEOUT +#define OT_CLIENT_TIMEOUT 30 +#endif +#ifndef OT_CLIENT_TIMEOUT_CHECKINTERVAL +#define OT_CLIENT_TIMEOUT_CHECKINTERVAL 10 +#endif +#define OT_CLIENT_TIMEOUT_SEND (60 * 15) +#define OT_CLIENT_REQUEST_INTERVAL (60 * 30) +#define OT_CLIENT_REQUEST_VARIATION (60 * 6) + +#define OT_TORRENT_TIMEOUT_HOURS 24 +#define OT_TORRENT_TIMEOUT (60 * OT_TORRENT_TIMEOUT_HOURS) + +#define OT_CLIENT_REQUEST_INTERVAL_RANDOM \ + (OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION / 2 + (int)(nrand48(ws->rand48_state) % OT_CLIENT_REQUEST_VARIATION)) + +/* If WANT_MODEST_FULLSCRAPES is on, ip addresses may not + fullscrape more frequently than this amount in seconds */ +#define OT_MODEST_PEER_TIMEOUT (60 * 5) + +/* If peers come back before 10 minutes, don't live sync them */ +#define OT_CLIENT_SYNC_RENEW_BOUNDARY 10 + +/* Number of tracker admin ip addresses allowed */ +#define OT_ADMINIP_MAX 64 +#define OT_MAX_THREADS 64 + +/* Number of minutes after announce before peer is removed */ +#define OT_PEER_TIMEOUT 45 + +/* We maintain a list of 1024 pointers to sorted list of ot_torrent structs + Sort key is, of course, its hash */ +/* Set from the Makefile, 8 to 16. More buckets means shorter vectors and a + cheaper insert, which only pays for a tracker carrying millions of + torrents; see DESIGN. */ +#ifndef OT_BUCKET_COUNT_BITS +#define OT_BUCKET_COUNT_BITS 10 +#endif + +#define OT_BUCKET_COUNT (1 << OT_BUCKET_COUNT_BITS) +#define OT_BUCKET_COUNT_SHIFT (32 - OT_BUCKET_COUNT_BITS) + +/* if _DEBUG_RANDOMTORRENTS is set, this is the amount of torrents to create + on startup */ +#define RANDOMTORRENTS (1024 * 1024 * 1) + +#if defined(CLOCK_REALTIME_COARSE) +#define OT_CLOCK_CHEAP CLOCK_REALTIME_COARSE /* Linux */ +#elif defined(CLOCK_REALTIME_FAST) +#define OT_CLOCK_CHEAP CLOCK_REALTIME_FAST /* the BSDs */ +#else +#define OT_CLOCK_CHEAP CLOCK_REALTIME /* macOS and whatever else */ +#endif + +static inline ot_time time_now_seconds(void) { + struct timespec now; + clock_gettime(OT_CLOCK_CHEAP, &now); + return (ot_time)now.tv_sec; +} + +static inline ot_time time_now_minutes(void) { return time_now_seconds() / 60; } + +/* For the deadlines the tracker sets on itself, which read better as the + span they allow than as an addition to the current second */ +static inline ot_time time_future_seconds(int span) { return time_now_seconds() + span; } +/* Cleared from the interrupt handler and read by every worker */ +extern _Atomic int g_opentracker_running; + +/* Sent to a thread parked in a blocking read */ +#define OT_SIGNAL_WAKE SIGUSR1 + + +extern uint32_t g_tracker_id; +typedef enum { FLAG_TCP, FLAG_UDP, FLAG_MCA, FLAG_SELFPIPE } PROTO_FLAG; + +#define OT_PEER_COMPARE_SIZE6 ((OT_IP_SIZE6) + (OT_PORT_SIZE)) +#define OT_PEER_COMPARE_SIZE4 ((OT_IP_SIZE4) + (OT_PORT_SIZE)) +#define OT_PEER_COMPARE_SIZE_FROM_PEER_SIZE(PEER_SIZE) ((PEER_SIZE) - (OT_TIME_SIZE) - (OT_FLAG_SIZE)) + +#define OT_PEER_SIZE6 ((OT_TIME_SIZE) + (OT_FLAG_SIZE) + (OT_PEER_COMPARE_SIZE6)) +#define OT_PEER_SIZE4 ((OT_TIME_SIZE) + (OT_FLAG_SIZE) + (OT_PEER_COMPARE_SIZE4)) + +typedef uint8_t ot_peer; /* Generic pointer to a v6 or v4 peer */ +typedef uint8_t ot_peer6[OT_PEER_SIZE6]; +typedef uint8_t ot_peer4[OT_PEER_SIZE4]; +static const uint8_t PEER_FLAG_SEEDING = 0x80; +static const uint8_t PEER_FLAG_COMPLETED = 0x40; +static const uint8_t PEER_FLAG_STOPPED = 0x20; +static const uint8_t PEER_FLAG_FROM_SYNC = 0x10; +static const uint8_t PEER_FLAG_LEECHING = 0x00; + +/* Takes an ot_peer6 and returns the proper pointer to the peer and sets peer_size */ +ot_peer *peer_from_peer6(ot_peer6 *peer, size_t *peer_size); +size_t peer_size_from_peer6(ot_peer6 *peer); + +/* New style */ +#define OT_SETIP(peer, ip) memcpy((uint8_t *)(peer), (ip), OT_IP_SIZE6) +#define OT_SETPORT(peer, port) memcpy(((uint8_t *)(peer)) + (OT_IP_SIZE6), (port), 2) +#define OT_PEERFLAG(peer) (((uint8_t *)(peer))[(OT_IP_SIZE6) + 2]) +#define OT_PEERFLAG_D(peer, peersize) (((uint8_t *)(peer))[(peersize) - 2]) +#define OT_PEERTIME(peer, peersize) (((uint8_t *)(peer))[(peersize) - 1]) + +#define PEERS_BENCODED6 "6:peers6" +#define PEERS_BENCODED4 "5:peers" + +#define OT_HASH_COMPARE_SIZE (sizeof(ot_hash)) + +struct ot_peerlist; +typedef struct ot_peerlist ot_peerlist; +typedef struct { + ot_hash hash; + ot_peerlist *peer_list6; + ot_peerlist *peer_list4; +} ot_torrent; + +#include "ot_vector.h" + +struct ot_peerlist { + ot_time base; + size_t seed_count; + size_t peer_count; + size_t down_count; + /* normal peers vector or + pointer to ot_vector[32] buckets if data != NULL and space == 0 + */ + ot_vector peers; +}; +#define OT_PEERLIST_HASBUCKETS(peer_list) ((peer_list)->peers.size > (peer_list)->peers.space) + +/* The http replies the tracker counts when it has to refuse a request. */ +enum { + CODE_HTTPERROR_302, + CODE_HTTPERROR_400, + CODE_HTTPERROR_400_PARAM, + CODE_HTTPERROR_400_COMPACT, + CODE_HTTPERROR_402_NOTMODEST, + CODE_HTTPERROR_402_PAYMENT_REQUIRED, + CODE_HTTPERROR_403_IP, + CODE_HTTPERROR_404, + CODE_HTTPERROR_500, + + CODE_HTTPERROR_COUNT +}; + +/* One statistic. Used when workers report stats to their workstruct */ +typedef _Atomic unsigned long long ot_counter; + +/* What each thread counts for itself. The two runs at the end are indexed by + the http error code and by the age in minutes of a renewed peer */ +typedef enum { + OT_COUNTER_TCP_CONNECTIONS, + OT_COUNTER_UDP_CONNECTIONS, + OT_COUNTER_TCP_ANNOUNCES, + OT_COUNTER_UDP_ANNOUNCES, + OT_COUNTER_TCP_SCRAPES, + OT_COUNTER_UDP_SCRAPES, + OT_COUNTER_TCP_CONNECTS, + OT_COUNTER_UDP_CONNECTS, + OT_COUNTER_UDP_CONNID_MISSMATCH, + OT_COUNTER_COMPLETED, + OT_COUNTER_FULLSCRAPES, + OT_COUNTER_FULLSCRAPE_REQUESTS, + OT_COUNTER_FULLSCRAPE_BYTES, + OT_COUNTER_SYNCS, + OT_COUNTER_STALLS, + + OT_COUNTER_FAILED, /* CODE_HTTPERROR_COUNT of them */ + OT_COUNTER_RENEWED = OT_COUNTER_FAILED + CODE_HTTPERROR_COUNT, /* OT_PEER_TIMEOUT of them */ + + OT_COUNTER_COUNT = OT_COUNTER_RENEWED + OT_PEER_TIMEOUT +} ot_counter_index; + +struct ot_workstruct { + /* Thread specific, static */ + + /* This thread's own statistics, and the link ot_stats keeps them on */ + ot_counter counters[OT_COUNTER_COUNT]; + struct ot_workstruct *counters_next; + + char *inbuf; +#define G_INBUF_SIZE 8192 + char *outbuf; +#define G_OUTBUF_SIZE 8192 +#ifdef _DEBUG_HTTPERROR + char *debugbuf; +#define G_DEBUGBUF_SIZE 8192 +#endif + + /* The peer currently in the working */ + ot_peer6 peer; /* Can fit v6 and v4 peers */ + + /* Pointers into the request buffer */ + ot_hash *hash; + char *peer_id; + + /* HTTP specific, non static */ + char *request; + ssize_t request_size; + ssize_t header_size; + char *reply; + ssize_t reply_size; + + /* Entropy state for rand48 function so that threads don't need to acquire mutexes for + global random() or arc4random() state, which causes heavy load on linuxes */ + uint16_t rand48_state[3]; + + int keep_alive; +}; + +/* Everything a thread that serves requests needs for itself. Set up and torn + down in one place so that a new kind of worker cannot miss a piece. + Returns -1 if the buffers could not be had. */ +int workstruct_init(struct ot_workstruct *ws); +void workstruct_deinit(struct ot_workstruct *ws); + +/* + Exported functions +*/ + +#ifdef WANT_SYNC_LIVE +#define WANT_SYNC +#endif + +#ifdef WANT_SYNC +#define WANT_SYNC_PARAM(param) , param +#else +#define WANT_SYNC_PARAM(param) +#endif + +void trackerlogic_init(void); +void trackerlogic_deinit(void); +void exerr(const char *message); + +/* add_peer_to_torrent does only release the torrent bucket if from_sync is set, + otherwise it is released in return_peers_for_torrent */ +size_t add_peer_to_torrent_and_return_peers(PROTO_FLAG proto, struct ot_workstruct *ws, size_t amount); +size_t remove_peer_from_torrent(PROTO_FLAG proto, struct ot_workstruct *ws); +void return_tcp_scrape_for_torrent(ot_hash const *hash_list, size_t amount, char **reply, const char *reply_end); +size_t return_udp_scrape_for_torrent(ot_hash const hash, char *reply); +void add_torrent_from_saved_state(ot_hash const hash, ot_time base, size_t down_count); +#ifdef _DEBUG_RANDOMTORRENTS +void trackerlogic_add_random_torrents(size_t amount); +#endif + +/* torrent iterator */ +void iterate_all_torrents(int (*for_each)(ot_torrent *torrent, uintptr_t data), uintptr_t data); + +/* Helper, before it moves to its own object */ +void free_peerlist(ot_peerlist *peer_list); + +#endif -- cgit v1.3