From 2dc80ac93fefd1ed7b37713243ab214812d347b3 Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 17 Oct 2017 21:32:11 -0700 Subject: [PATCH 001/138] talk to new python script to interface with relay So far uses curl and json-c to send b64-encoded data to new script which is able to echo the data. Next that script will need to open a UDP socket to the relay and return results that appear before timeout. --- xwords4/android/scripts/relay.py | 22 +++++++++ xwords4/linux/Makefile | 2 +- xwords4/linux/relaycon.c | 84 +++++++++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100755 xwords4/android/scripts/relay.py diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py new file mode 100755 index 000000000..599bf4d83 --- /dev/null +++ b/xwords4/android/scripts/relay.py @@ -0,0 +1,22 @@ +#!/usr/bin/python + +import mod_python, json + +try: + from mod_python import apache + apacheAvailable = True +except ImportError: + apacheAvailable = False + print('failed') + +def post(req, params): + jobj = json.loads(params) + jobj = {'data' : jobj['data']} + return json.dumps(jobj) + +def main(): + None + +############################################################################## +if __name__ == '__main__': + main() diff --git a/xwords4/linux/Makefile b/xwords4/linux/Makefile index 8b5499b5c..f09c514ee 100644 --- a/xwords4/linux/Makefile +++ b/xwords4/linux/Makefile @@ -226,7 +226,7 @@ OBJ = \ $(BUILD_PLAT_DIR)/relaycon.o \ $(CURSES_OBJS) $(GTK_OBJS) $(MAIN_OBJS) -LIBS = -lm -luuid $(GPROFFLAG) +LIBS = -lm -luuid -lcurl -ljson-c $(GPROFFLAG) ifdef USE_SQLITE LIBS += -lsqlite3 DEFINES += -DUSE_SQLITE diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 050e477fa..5339cbbc7 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -20,6 +20,9 @@ #include #include #include +#include +#include + #include "relaycon.h" #include "linuxmain.h" @@ -402,10 +405,88 @@ hostNameToIP( const XP_UCHAR* name ) return ip; } +typedef struct _ReadState { + gchar* ptr; + size_t curSize; +} ReadState; + +static size_t +write_callback(void *contents, size_t size, size_t nmemb, void* data) +{ + ReadState* rs = (ReadState*)data; + XP_LOGF( "%s(size=%ld, nmemb=%ld)", __func__, size, nmemb ); + // void** pp = (void**)data; + size_t oldLen = rs->curSize; + size_t newLength = size * nmemb; + rs->ptr = g_realloc( rs->ptr, oldLen + newLength ); + memcpy( rs->ptr + oldLen - 1, contents, newLength ); + rs->ptr[oldLen + newLength - 1] = '\0'; + size_t result = size * nmemb; + // XP_LOGF( "%s() => %ld: (passed: \"%s\")", __func__, result, *strp ); + return result; +} + +static ssize_t +post( const XP_U8* msgbuf, XP_U16 len ) +{ + const char* data = g_base64_encode( msgbuf, len ); + struct json_object* jobj = json_object_new_object(); + struct json_object* jstr = json_object_new_string(data); + // g_free( data ); + json_object_object_add( jobj, "data", jstr ); + const char* asStr = json_object_to_json_string( jobj ); + XP_LOGF( "%s: added str: %s", __func__, asStr ); + + ReadState rs = { + .ptr = g_malloc0(1), + .curSize = 1L + }; + + CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); + XP_ASSERT(res == CURLE_OK); + CURL* curl = curl_easy_init(); + + curl_easy_setopt(curl, CURLOPT_URL, "http://localhost/relay.py/post"); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + + char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); + char buf[4*1024]; + size_t buflen = snprintf( buf, sizeof(buf), "params=%s", curl_params); + XP_ASSERT( buflen < sizeof(buf) ); + curl_free(curl_params); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buf); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf)); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback ); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &rs ); + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + + res = curl_easy_perform(curl); + XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); + /* Check for errors */ + if (res != CURLE_OK) { + XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); + } + /* always cleanup */ + curl_easy_cleanup(curl); + curl_global_cleanup(); + + XP_LOGF( "%s(): got \"%s\"", __func__, rs.ptr ); + g_free( rs.ptr ); + + (void)json_object_put( jobj ); + return len; +} + static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) { - ssize_t nSent = sendto( storage->socket, msgbuf, len, 0, /* flags */ + ssize_t nSent; + if (1) { + nSent = post( msgbuf, len ); + } else { + nSent = sendto( storage->socket, msgbuf, len, 0, /* flags */ (struct sockaddr*)&storage->saddr, sizeof(storage->saddr) ); #ifdef COMMS_CHECKSUM @@ -415,6 +496,7 @@ sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) #else XP_LOGF( "%s()=>%zd", __func__, nSent ); #endif + } return nSent; } From b86ffeb2b9b64b0f99f327d9c39726084259b617 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 18 Oct 2017 06:53:15 -0700 Subject: [PATCH 002/138] wip: data gets to relay and response handled A device registers and a game can start. But we don't get to being able to make a move yet. --- xwords4/android/scripts/relay.py | 25 +++++++++-- xwords4/linux/relaycon.c | 73 ++++++++++++++++++++------------ 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 599bf4d83..ceb682142 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -1,6 +1,6 @@ #!/usr/bin/python -import mod_python, json +import mod_python, json, socket, base64 try: from mod_python import apache @@ -10,12 +10,31 @@ except ImportError: print('failed') def post(req, params): + err = 'none' + dataLen = 0 jobj = json.loads(params) - jobj = {'data' : jobj['data']} + data = base64.b64decode(jobj['data']) + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(3) # seconds + addr = ("127.0.0.1", 10997) + sock.sendto(data, addr) + + response = None + try: + data, server = sock.recvfrom(1024) + response = base64.b64encode(data) + except socket.timeout: + #If data is not received back from server, print it has timed out + err = 'timeout' + + jobj = {'err' : err, 'data' : response} return json.dumps(jobj) def main(): - None + params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } + params = json.dumps(params) + print(post(None, params)) ############################################################################## if __name__ == '__main__': diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 5339cbbc7..601a581f7 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -242,30 +242,9 @@ sendAckIf( RelayConStorage* storage, const MsgHeader* header ) } } -static gboolean -relaycon_receive( GIOChannel* source, GIOCondition XP_UNUSED_DBG(condition), gpointer data ) +static gboolean +process( RelayConStorage* storage, XP_U8* buf, ssize_t nRead ) { - XP_ASSERT( 0 != (G_IO_IN & condition) ); /* FIX ME */ - RelayConStorage* storage = (RelayConStorage*)data; - XP_U8 buf[512]; - struct sockaddr_in from; - socklen_t fromlen = sizeof(from); - - int socket = g_io_channel_unix_get_fd( source ); - XP_LOGF( "%s: calling recvfrom on socket %d", __func__, socket ); - - ssize_t nRead = recvfrom( socket, buf, sizeof(buf), 0, /* flags */ - (struct sockaddr*)&from, &fromlen ); - - gchar* b64 = g_base64_encode( (const guchar*)buf, - ((0 <= nRead)? nRead : 0) ); - XP_LOGF( "%s: read %zd bytes ('%s')", __func__, nRead, b64 ); - g_free( b64 ); -#ifdef COMMS_CHECKSUM - gchar* sum = g_compute_checksum_for_data( G_CHECKSUM_MD5, buf, nRead ); - XP_LOGF( "%s: read %zd bytes ('%s')(sum=%s)", __func__, nRead, b64, sum ); - g_free( sum ); -#endif if ( 0 <= nRead ) { const XP_U8* ptr = buf; const XP_U8* end = buf + nRead; @@ -369,6 +348,33 @@ relaycon_receive( GIOChannel* source, GIOCondition XP_UNUSED_DBG(condition), gpo return TRUE; } +static gboolean +relaycon_receive( GIOChannel* source, GIOCondition XP_UNUSED_DBG(condition), gpointer data ) +{ + XP_ASSERT( 0 != (G_IO_IN & condition) ); /* FIX ME */ + RelayConStorage* storage = (RelayConStorage*)data; + XP_U8 buf[512]; + struct sockaddr_in from; + socklen_t fromlen = sizeof(from); + + int socket = g_io_channel_unix_get_fd( source ); + XP_LOGF( "%s: calling recvfrom on socket %d", __func__, socket ); + + ssize_t nRead = recvfrom( socket, buf, sizeof(buf), 0, /* flags */ + (struct sockaddr*)&from, &fromlen ); + + gchar* b64 = g_base64_encode( (const guchar*)buf, + ((0 <= nRead)? nRead : 0) ); + XP_LOGF( "%s: read %zd bytes ('%s')", __func__, nRead, b64 ); + g_free( b64 ); +#ifdef COMMS_CHECKSUM + gchar* sum = g_compute_checksum_for_data( G_CHECKSUM_MD5, buf, nRead ); + XP_LOGF( "%s: read %zd bytes ('%s')(sum=%s)", __func__, nRead, b64, sum ); + g_free( sum ); +#endif + return process( storage, buf, nRead ); +} + void relaycon_cleanup( LaunchParams* params ) { @@ -427,7 +433,7 @@ write_callback(void *contents, size_t size, size_t nmemb, void* data) } static ssize_t -post( const XP_U8* msgbuf, XP_U16 len ) +post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) { const char* data = g_base64_encode( msgbuf, len ); struct json_object* jobj = json_object_new_object(); @@ -471,11 +477,26 @@ post( const XP_U8* msgbuf, XP_U16 len ) /* always cleanup */ curl_easy_cleanup(curl); curl_global_cleanup(); + (void)json_object_put( jobj ); XP_LOGF( "%s(): got \"%s\"", __func__, rs.ptr ); + + /* Now pull any data from the reply */ + // got "{"status": "ok", "dataLen": 14, "data": "AYQDiDAyMUEzQ0MyADw=", "err": "none"}" + json_object* reply = json_tokener_parse( rs.ptr ); + json_object* replyData; + if ( json_object_object_get_ex( reply, "data", &replyData ) && !!replyData ) { + const char* str = json_object_get_string( replyData ); + gsize out_len; + guchar* buf = g_base64_decode( (const gchar*)str, &out_len ); + process( storage, buf, len ); + g_free( buf ); + (void)json_object_put( replyData ); + } + (void)json_object_put( reply ); + g_free( rs.ptr ); - (void)json_object_put( jobj ); return len; } @@ -484,7 +505,7 @@ sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) { ssize_t nSent; if (1) { - nSent = post( msgbuf, len ); + nSent = post( storage, msgbuf, len ); } else { nSent = sendto( storage->socket, msgbuf, len, 0, /* flags */ (struct sockaddr*)&storage->saddr, From c08be98fda7aafc60a0008925cade9b85af31b04 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 18 Oct 2017 07:09:05 -0700 Subject: [PATCH 003/138] wip: include multiple packets in reponse json --- xwords4/android/scripts/relay.py | 18 ++++++++++-------- xwords4/linux/relaycon.c | 14 +++++++++----- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index ceb682142..0b8e3a775 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -20,15 +20,17 @@ def post(req, params): addr = ("127.0.0.1", 10997) sock.sendto(data, addr) - response = None - try: - data, server = sock.recvfrom(1024) - response = base64.b64encode(data) - except socket.timeout: - #If data is not received back from server, print it has timed out - err = 'timeout' + responses = [] + while True: + try: + data, server = sock.recvfrom(1024) + responses.append(base64.b64encode(data)) + except socket.timeout: + #If data is not received back from server, print it has timed out + err = 'timeout' + break - jobj = {'err' : err, 'data' : response} + jobj = {'err' : err, 'data' : responses} return json.dumps(jobj) def main(): diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 601a581f7..937604c5f 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -486,11 +486,15 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) json_object* reply = json_tokener_parse( rs.ptr ); json_object* replyData; if ( json_object_object_get_ex( reply, "data", &replyData ) && !!replyData ) { - const char* str = json_object_get_string( replyData ); - gsize out_len; - guchar* buf = g_base64_decode( (const gchar*)str, &out_len ); - process( storage, buf, len ); - g_free( buf ); + int len = json_object_array_length(replyData); + for ( int ii = 0; ii < len; ++ii ) { + json_object* datum = json_object_array_get_idx( replyData, ii ); + const char* str = json_object_get_string( datum ); + gsize out_len; + guchar* buf = g_base64_decode( (const gchar*)str, &out_len ); + process( storage, buf, len ); + g_free( buf ); + } (void)json_object_put( replyData ); } (void)json_object_put( reply ); From e6e93c09abfd33de8e80402f54b4dda48550bcad Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 18 Oct 2017 07:19:43 -0700 Subject: [PATCH 004/138] oops: use the right length --- xwords4/linux/relaycon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 937604c5f..3f3364f74 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -492,7 +492,7 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) const char* str = json_object_get_string( datum ); gsize out_len; guchar* buf = g_base64_decode( (const gchar*)str, &out_len ); - process( storage, buf, len ); + process( storage, buf, out_len ); g_free( buf ); } (void)json_object_put( replyData ); From fbaa1f139e533938ea776c379a7e843f549ddf72 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 18 Oct 2017 21:18:30 -0700 Subject: [PATCH 005/138] add test method and implement query() endpoint --- xwords4/android/scripts/relay.py | 74 +++++++++++++++++++++++++++++--- xwords4/linux/relaycon.c | 2 +- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 0b8e3a775..e0195f7ff 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -1,13 +1,15 @@ #!/usr/bin/python -import mod_python, json, socket, base64 +import base64, json, mod_python, socket, struct, sys + +PROTOCOL_VERSION = 0 +PRX_GET_MSGS = 4 try: from mod_python import apache apacheAvailable = True except ImportError: apacheAvailable = False - print('failed') def post(req, params): err = 'none' @@ -16,7 +18,7 @@ def post(req, params): data = base64.b64decode(jobj['data']) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(3) # seconds + sock.settimeout(1) # seconds addr = ("127.0.0.1", 10997) sock.sendto(data, addr) @@ -33,10 +35,70 @@ def post(req, params): jobj = {'err' : err, 'data' : responses} return json.dumps(jobj) +def query(req, ids): + idsLen = 0 + for id in ids: idsLen += len(id) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(5) # seconds + sock.connect(('127.0.0.1', 10998)) + + lenShort = 2 + idsLen + len(ids) + 1 + sock.send(struct.pack("hBBh", socket.htons(lenShort), + PROTOCOL_VERSION, PRX_GET_MSGS, + socket.htons(len(ids)))) + + for id in ids: sock.send(id + '\n') + + unpacker = struct.Struct('2H') # 2s f') + data = sock.recv(unpacker.size) + resLen, nameCount = unpacker.unpack(data) + resLen = socket.ntohs(resLen) + nameCount = socket.ntohs(nameCount) + print('resLen:', resLen, 'nameCount:', nameCount) + msgsLists = {} + if nameCount == len(ids): + for ii in range(nameCount): + perGame = [] + shortUnpacker = struct.Struct('H') + countsThisGame, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) + countsThisGame = socket.ntohs(countsThisGame) + print('countsThisGame:', countsThisGame) + for jj in range(countsThisGame): + msgLen, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) + msgLen = socket.ntohs(msgLen) + print('msgLen:', msgLen) + msgs = [] + if msgLen > 0: + msg = sock.recv(msgLen) + print('msg len:', len(msg)) + msg = base64.b64encode(msg) + msgs.append(msg) + perGame.append(msgs) + msgsLists[ids[ii]] = perGame + + return json.dumps(msgsLists) + + # received = sock.recv(1024*4) + # print('len:', len(received)) + # short resLen = dis.readShort(); // total message length + # short nameCount = dis.readShort(); + + +def dosend(sock, bytes): + totalsent = 0 + while totalsent < len(bytes): + sent = sock.send(bytes[totalsent:]) + if sent == 0: + raise RuntimeError("socket connection broken") + totalsent = totalsent + sent + + def main(): - params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } - params = json.dumps(params) - print(post(None, params)) + print(query(None, sys.argv[1:])) + # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } + # params = json.dumps(params) + # print(post(None, params)) ############################################################################## if __name__ == '__main__': diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 3f3364f74..72275a2eb 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -452,7 +452,7 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) XP_ASSERT(res == CURLE_OK); CURL* curl = curl_easy_init(); - curl_easy_setopt(curl, CURLOPT_URL, "http://localhost/relay.py/post"); + curl_easy_setopt(curl, CURLOPT_URL, "http://localhost/xw4/relay.py/post"); curl_easy_setopt(curl, CURLOPT_POST, 1L); char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); From 3ffef37d773d2a2bc77db6c28713e30dee22ffbe Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 18 Oct 2017 21:19:09 -0700 Subject: [PATCH 006/138] don't require myKey.py --- xwords4/android/scripts/info.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xwords4/android/scripts/info.py b/xwords4/android/scripts/info.py index 936a4dd32..80ad70d6f 100755 --- a/xwords4/android/scripts/info.py +++ b/xwords4/android/scripts/info.py @@ -7,7 +7,10 @@ import mk_for_download, mygit import xwconfig # I'm not checking my key in... -import mykey +try : + import mykey +except: + print('unable to load mykey') from stat import ST_CTIME try: From 9b486d77fb37927dff819462f1e45a05c649558a Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 18 Oct 2017 21:19:42 -0700 Subject: [PATCH 007/138] shrink column to shrink display --- xwords4/relay/scripts/showinplay.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xwords4/relay/scripts/showinplay.sh b/xwords4/relay/scripts/showinplay.sh index f4c7eeec7..a3b9955a1 100755 --- a/xwords4/relay/scripts/showinplay.sh +++ b/xwords4/relay/scripts/showinplay.sh @@ -54,12 +54,12 @@ echo "; relay pid[s]: $(pidof xwrelay)" echo "Row count:" $(psql -t xwgames -c "select count(*) FROM games $QUERY;") # Games -echo "SELECT dead as d,connname,cid,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as nPerDev,nsents as snts, seeds,devids,tokens,ack, mtimes "\ +echo "SELECT dead as d,connname,cid,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ "FROM games $QUERY ORDER BY NOT dead, ctime DESC LIMIT $LIMIT;" \ | psql xwgames # Messages -echo "SELECT * "\ +echo "SELECT id,connName,hid as h,token,ctime,stime,devid,msg64 "\ "FROM msgs WHERE connname IN (SELECT connname from games $QUERY) "\ "ORDER BY ctime DESC, connname LIMIT $LIMIT;" \ | psql xwgames From 7414e0eeae13f8bfa10e436e74e837399605f978 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 18 Oct 2017 22:02:14 -0700 Subject: [PATCH 008/138] param should be json array as string --- xwords4/android/scripts/relay.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index e0195f7ff..f04386fb0 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -36,6 +36,9 @@ def post(req, params): return json.dumps(jobj) def query(req, ids): + print(ids) + ids = json.loads(ids) + idsLen = 0 for id in ids: idsLen += len(id) @@ -95,7 +98,7 @@ def dosend(sock, bytes): def main(): - print(query(None, sys.argv[1:])) + print(query(None, json.dumps(sys.argv[1:]))) # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } # params = json.dumps(params) # print(post(None, params)) From 4a1e51b54a68cd056b3b166849d28bb049aecf4b Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 18 Oct 2017 22:03:14 -0700 Subject: [PATCH 009/138] call query from C Very rough code that fetches messages and does nothing with them. --- xwords4/linux/relaycon.c | 90 +++++++++++++++++++++++++++++++--------- xwords4/linux/relaycon.h | 2 + 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 72275a2eb..09ef9f3ae 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -63,6 +63,76 @@ static size_t writeVLI( XP_U8* out, uint32_t nn ); static size_t un2vli( int nn, uint8_t* buf ); static bool vli2un( const uint8_t** inp, uint32_t* outp ); +typedef struct _ReadState { + gchar* ptr; + size_t curSize; +} ReadState; + +static size_t +write_callback(void *contents, size_t size, size_t nmemb, void* data) +{ + ReadState* rs = (ReadState*)data; + XP_LOGF( "%s(size=%ld, nmemb=%ld)", __func__, size, nmemb ); + // void** pp = (void**)data; + size_t oldLen = rs->curSize; + size_t newLength = size * nmemb; + rs->ptr = g_realloc( rs->ptr, oldLen + newLength ); + memcpy( rs->ptr + oldLen - 1, contents, newLength ); + rs->ptr[oldLen + newLength - 1] = '\0'; + size_t result = size * nmemb; + // XP_LOGF( "%s() => %ld: (passed: \"%s\")", __func__, result, *strp ); + return result; +} + +void +checkForMsgs(const XP_UCHAR* id) +{ + ReadState rs = { + .ptr = g_malloc0(1), + .curSize = 1L + }; + + /* build a json array of relayIDs, then stringify it */ + json_object* params = json_object_new_array(); + json_object* idstr = json_object_new_string(id); + json_object_array_add(params, idstr); + const char* asStr = json_object_to_json_string( params ); + XP_LOGF( "%s: added str: %s", __func__, asStr ); + + CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); + XP_ASSERT(res == CURLE_OK); + CURL* curl = curl_easy_init(); + + curl_easy_setopt(curl, CURLOPT_URL, + "http://localhost/xw4/relay.py/query"); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + + char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); + char buf[4*1024]; + size_t buflen = snprintf( buf, sizeof(buf), "ids=%s", curl_params); + XP_ASSERT( buflen < sizeof(buf) ); + curl_free(curl_params); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buf); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf)); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback ); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &rs ); + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + + res = curl_easy_perform(curl); + + XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); + /* Check for errors */ + if (res != CURLE_OK) { + XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); + } + /* always cleanup */ + curl_easy_cleanup(curl); + curl_global_cleanup(); + + XP_LOGF( "%s(): got \"%s\"", __func__, rs.ptr ); +} void relaycon_init( LaunchParams* params, const RelayConnProcs* procs, @@ -411,26 +481,6 @@ hostNameToIP( const XP_UCHAR* name ) return ip; } -typedef struct _ReadState { - gchar* ptr; - size_t curSize; -} ReadState; - -static size_t -write_callback(void *contents, size_t size, size_t nmemb, void* data) -{ - ReadState* rs = (ReadState*)data; - XP_LOGF( "%s(size=%ld, nmemb=%ld)", __func__, size, nmemb ); - // void** pp = (void**)data; - size_t oldLen = rs->curSize; - size_t newLength = size * nmemb; - rs->ptr = g_realloc( rs->ptr, oldLen + newLength ); - memcpy( rs->ptr + oldLen - 1, contents, newLength ); - rs->ptr[oldLen + newLength - 1] = '\0'; - size_t result = size * nmemb; - // XP_LOGF( "%s() => %ld: (passed: \"%s\")", __func__, result, *strp ); - return result; -} static ssize_t post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) diff --git a/xwords4/linux/relaycon.h b/xwords4/linux/relaycon.h index a8b0ef424..3cd00e8f7 100644 --- a/xwords4/linux/relaycon.h +++ b/xwords4/linux/relaycon.h @@ -56,4 +56,6 @@ void relaycon_cleanup( LaunchParams* params ); XP_U32 makeClientToken( sqlite3_int64 rowid, XP_U16 seed ); void rowidFromToken( XP_U32 clientToken, sqlite3_int64* rowid, XP_U16* seed ); + +void checkForMsgs(const XP_UCHAR* id); #endif From 5223ccabe17a98e5e06d29595a72d42ffb3f376c Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 19 Oct 2017 20:50:33 -0700 Subject: [PATCH 010/138] add option to run forever --- xwords4/linux/scripts/discon_ok2.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index 19d914e7c..f4b1d5042 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -19,7 +19,7 @@ MAXDEVS="" ONEPER="" RESIGN_RATIO="" DROP_N="" -MINRUN=2 +MINRUN=2 # seconds ONE_PER_ROOM="" # don't run more than one device at a time per room USE_GTK="" UNDO_PCT=0 @@ -609,6 +609,7 @@ function usage() { echo " [--one-per] # force one player per device \\" >&2 echo " [--port ] \\" >&2 echo " [--resign-ratio <0 <= n <=1000 > \\" >&2 + echo " [--no-timeout] # run until all games done \\" >&2 echo " [--seed ] \\" >&2 echo " [--send-chat \\" >&2 echo " [--udp-incr ] \\" >&2 @@ -698,6 +699,9 @@ while [ "$#" -gt 0 ]; do RESIGN_RATIO=$(getArg $*) shift ;; + --no-timeout) + TIMEOUT=0x7FFFFFFF + ;; --help) usage ;; From 3045697d31ec30cc9ceefe49aeec4ced8635e5f2 Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 19 Oct 2017 21:20:14 -0700 Subject: [PATCH 011/138] wip: process moves received --- xwords4/common/game.c | 28 ++++++++ xwords4/common/game.h | 4 ++ xwords4/linux/cursesmain.c | 25 +++++++ xwords4/linux/cursesmain.h | 1 + xwords4/linux/linuxutl.c | 15 +++-- xwords4/linux/relaycon.c | 134 +++++++++++++++++++++++++------------ xwords4/linux/relaycon.h | 2 +- 7 files changed, 159 insertions(+), 50 deletions(-) diff --git a/xwords4/common/game.c b/xwords4/common/game.c index a9412b2d8..9bb55135f 100644 --- a/xwords4/common/game.c +++ b/xwords4/common/game.c @@ -338,6 +338,34 @@ game_saveSucceeded( const XWGame* game, XP_U16 saveToken ) } } +XP_Bool +game_receiveMessage( XWGame* game, XWStreamCtxt* stream, CommsAddrRec* retAddr ) +{ + ServerCtxt* server = game->server; + CommsMsgState commsState; + XP_Bool result = comms_checkIncomingStream( game->comms, stream, retAddr, + &commsState ); + if ( result ) { + (void)server_do( server ); + + result = server_receiveMessage( server, stream ); + } + comms_msgProcessed( game->comms, &commsState, !result ); + + if ( result ) { + /* in case MORE work's pending. Multiple calls are required in at + least one case, where I'm a host handling client registration *AND* + I'm a robot. Only one server_do and I'll never make that first + robot move. That's because comms can't detect a duplicate initial + packet (in validateInitialMessage()). */ + for ( int ii = 0; ii < 5; ++ii ) { + (void)server_do( server ); + } + } + + return result; +} + void game_getState( const XWGame* game, GameStateInfo* gsi ) { diff --git a/xwords4/common/game.h b/xwords4/common/game.h index d79cbc20f..6a68a9bf0 100644 --- a/xwords4/common/game.h +++ b/xwords4/common/game.h @@ -82,6 +82,10 @@ void game_saveNewGame( MPFORMAL const CurGameInfo* gi, XW_UtilCtxt* util, void game_saveToStream( const XWGame* game, const CurGameInfo* gi, XWStreamCtxt* stream, XP_U16 saveToken ); void game_saveSucceeded( const XWGame* game, XP_U16 saveToken ); + +XP_Bool game_receiveMessage( XWGame* game, XWStreamCtxt* stream, + CommsAddrRec* retAddr ); + void game_dispose( XWGame* game ); void game_getState( const XWGame* game, GameStateInfo* gsi ); diff --git a/xwords4/linux/cursesmain.c b/xwords4/linux/cursesmain.c index ea02def79..48ddeec88 100644 --- a/xwords4/linux/cursesmain.c +++ b/xwords4/linux/cursesmain.c @@ -1751,6 +1751,28 @@ cursesErrorMsgRcvd( void* closure, const XP_UCHAR* msg ) } } +static gboolean +queryTimerFired( gpointer data ) +{ + LOG_FUNC(); + CursesAppGlobals* globals = (CursesAppGlobals*)data; + + XWGame* game = &globals->cGlobals.game; + if ( !!game->comms ) { + XP_ASSERT( globals->nextQueryTimeSecs > 0 ); + if ( checkForMsgs( globals->cGlobals.params, game ) ) { + globals->nextQueryTimeSecs = 1; + } else { + globals->nextQueryTimeSecs *= 2; + } + + (void)g_timeout_add_seconds( globals->nextQueryTimeSecs, + queryTimerFired, &g_globals ); + } + + return FALSE; +} + static gboolean chatsTimerFired( gpointer data ) { @@ -1928,6 +1950,9 @@ cursesmain( XP_Bool isServer, LaunchParams* params ) (void)g_timeout_add_seconds( params->chatsInterval, chatsTimerFired, &g_globals ); } + g_globals.nextQueryTimeSecs = 1; + (void)g_timeout_add_seconds( g_globals.nextQueryTimeSecs, + queryTimerFired, &g_globals ); XP_Bool opened = XP_FALSE; initCurses( &g_globals, &width, &height ); diff --git a/xwords4/linux/cursesmain.h b/xwords4/linux/cursesmain.h index 1d859e254..59443e652 100644 --- a/xwords4/linux/cursesmain.h +++ b/xwords4/linux/cursesmain.h @@ -71,6 +71,7 @@ struct CursesAppGlobals { gchar* lastErr; XP_U16 nChatsSent; + XP_U16 nextQueryTimeSecs; union { struct { diff --git a/xwords4/linux/linuxutl.c b/xwords4/linux/linuxutl.c index 95f99666d..3f3cc6906 100644 --- a/xwords4/linux/linuxutl.c +++ b/xwords4/linux/linuxutl.c @@ -41,7 +41,7 @@ void linux_debugf( const char* format, ... ) { - char buf[1000]; + char buf[1024*8]; va_list ap; struct tm* timp; struct timeval tv; @@ -50,14 +50,17 @@ linux_debugf( const char* format, ... ) gettimeofday( &tv, &tz ); timp = localtime( &tv.tv_sec ); - snprintf( buf, sizeof(buf), "<%d>%.2d:%.2d:%.2d:", getpid(), - timp->tm_hour, timp->tm_min, timp->tm_sec ); + size_t len = snprintf( buf, sizeof(buf), "<%d>%.2d:%.2d:%.2d:", getpid(), + timp->tm_hour, timp->tm_min, timp->tm_sec ); + XP_ASSERT( len < sizeof(buf) ); va_start(ap, format); - - vsprintf(buf+strlen(buf), format, ap); - + len = vsprintf(buf+strlen(buf), format, ap); va_end(ap); + + if ( len >= sizeof(buf) ) { + buf[sizeof(buf)-1] = '\0'; + } fprintf( stderr, "%s\n", buf ); } diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 09ef9f3ae..57efe3d59 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -63,6 +63,7 @@ static size_t writeVLI( XP_U8* out, uint32_t nn ); static size_t un2vli( int nn, uint8_t* buf ); static bool vli2un( const uint8_t** inp, uint32_t* outp ); + typedef struct _ReadState { gchar* ptr; size_t curSize; @@ -75,63 +76,110 @@ write_callback(void *contents, size_t size, size_t nmemb, void* data) XP_LOGF( "%s(size=%ld, nmemb=%ld)", __func__, size, nmemb ); // void** pp = (void**)data; size_t oldLen = rs->curSize; - size_t newLength = size * nmemb; + const size_t newLength = size * nmemb; + XP_ASSERT( (oldLen + newLength) > 0 ); rs->ptr = g_realloc( rs->ptr, oldLen + newLength ); memcpy( rs->ptr + oldLen - 1, contents, newLength ); rs->ptr[oldLen + newLength - 1] = '\0'; - size_t result = size * nmemb; // XP_LOGF( "%s() => %ld: (passed: \"%s\")", __func__, result, *strp ); - return result; + return newLength; } -void -checkForMsgs(const XP_UCHAR* id) +XP_Bool +checkForMsgs( LaunchParams* params, XWGame* game ) { - ReadState rs = { - .ptr = g_malloc0(1), - .curSize = 1L - }; + XP_Bool foundAny = false; + XP_UCHAR idBuf[64]; + if ( !!game->comms ) { + XP_U16 len = VSIZE(idBuf); + if ( comms_getRelayID( game->comms, idBuf, &len ) ) { + XP_LOGF( "%s: got %s", __func__, idBuf ); + } else { + idBuf[0] = '\0'; + } + } - /* build a json array of relayIDs, then stringify it */ - json_object* params = json_object_new_array(); - json_object* idstr = json_object_new_string(id); - json_object_array_add(params, idstr); - const char* asStr = json_object_to_json_string( params ); - XP_LOGF( "%s: added str: %s", __func__, asStr ); - - CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); - XP_ASSERT(res == CURLE_OK); - CURL* curl = curl_easy_init(); - - curl_easy_setopt(curl, CURLOPT_URL, - "http://localhost/xw4/relay.py/query"); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - - char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); - char buf[4*1024]; - size_t buflen = snprintf( buf, sizeof(buf), "ids=%s", curl_params); - XP_ASSERT( buflen < sizeof(buf) ); - curl_free(curl_params); + if ( !!idBuf[0] ) { - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buf); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf)); + ReadState rs = { + .ptr = g_malloc0(1), + .curSize = 1L + }; - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &rs ); - curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + /* build a json array of relayIDs, then stringify it */ + json_object* ids = json_object_new_array(); + json_object* idstr = json_object_new_string(idBuf); + json_object_array_add(ids, idstr); + const char* asStr = json_object_to_json_string( ids ); + XP_LOGF( "%s: added str: %s", __func__, asStr ); - res = curl_easy_perform(curl); + CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); + XP_ASSERT(res == CURLE_OK); + CURL* curl = curl_easy_init(); + + curl_easy_setopt(curl, CURLOPT_URL, + "http://localhost/xw4/relay.py/query"); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + + char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); + char buf[4*1024]; + size_t buflen = snprintf( buf, sizeof(buf), "ids=%s", curl_params); + XP_ASSERT( buflen < sizeof(buf) ); + curl_free(curl_params); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buf); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf)); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback ); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &rs ); + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + + res = curl_easy_perform(curl); XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); - /* Check for errors */ - if (res != CURLE_OK) { - XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); - } - /* always cleanup */ - curl_easy_cleanup(curl); - curl_global_cleanup(); + /* Check for errors */ + if (res != CURLE_OK) { + XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); + } + /* always cleanup */ + curl_easy_cleanup(curl); + curl_global_cleanup(); - XP_LOGF( "%s(): got \"%s\"", __func__, rs.ptr ); + XP_LOGF( "%s(): got <<%s>>", __func__, rs.ptr ); + + if (res == CURLE_OK) { + json_object* reply = json_tokener_parse( rs.ptr ); + json_object_object_foreach(reply, key, val) { + int len = json_object_array_length(val); + XP_LOGF( "%s: got key: %s of len %d", __func__, key, len ); + for ( int ii = 0; ii < len; ++ii ) { + json_object* forGame = json_object_array_get_idx(val, ii); + int len2 = json_object_array_length(forGame); + foundAny = foundAny || len2 > 0; + for ( int jj = 0; jj < len2; ++jj ) { + json_object* oneMove = json_object_array_get_idx(forGame, jj); + const char* asStr = json_object_get_string(oneMove); + gsize out_len; + guchar* buf = g_base64_decode( asStr, &out_len ); + XWStreamCtxt* stream = mem_stream_make( MPPARM(params->mpool) + params->vtMgr, params, + CHANNEL_NONE, NULL ); + stream_putBytes( stream, buf, out_len ); + g_free(buf); + + CommsAddrRec addr = {0}; + addr_addType( &addr, COMMS_CONN_RELAY ); + XP_Bool handled = game_receiveMessage( game, stream, &addr ); + XP_LOGF( "%s(): game_receiveMessage() => %d", __func__, handled ); + stream_destroy( stream ); + + foundAny = XP_TRUE; + } + } + } + } + } + return foundAny; } void diff --git a/xwords4/linux/relaycon.h b/xwords4/linux/relaycon.h index 3cd00e8f7..c418b6a5a 100644 --- a/xwords4/linux/relaycon.h +++ b/xwords4/linux/relaycon.h @@ -57,5 +57,5 @@ void relaycon_cleanup( LaunchParams* params ); XP_U32 makeClientToken( sqlite3_int64 rowid, XP_U16 seed ); void rowidFromToken( XP_U32 clientToken, sqlite3_int64* rowid, XP_U16* seed ); -void checkForMsgs(const XP_UCHAR* id); +XP_Bool checkForMsgs(LaunchParams* params, XWGame* game); #endif From 816df4336c6332dcf3a6ed8f6ec4132659e7968e Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 20 Oct 2017 06:26:46 -0700 Subject: [PATCH 012/138] run post in thread since it takes time --- xwords4/linux/gtkboard.c | 11 ++++++ xwords4/linux/relaycon.c | 83 +++++++++++++++++++++++++++------------- 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/xwords4/linux/gtkboard.c b/xwords4/linux/gtkboard.c index 564ce31ac..f25622306 100644 --- a/xwords4/linux/gtkboard.c +++ b/xwords4/linux/gtkboard.c @@ -1175,6 +1175,15 @@ handle_memstats( GtkWidget* XP_UNUSED(widget), GtkGameGlobals* globals ) stream_destroy( stream ); } /* handle_memstats */ + +static void +handle_movescheck( GtkWidget* XP_UNUSED(widget), GtkGameGlobals* globals ) +{ + LaunchParams* params = globals->cGlobals.params; + if ( checkForMsgs( params, &globals->cGlobals.game ) ) { + board_draw( globals->cGlobals.game.board ); + } +} #endif #ifdef XWFEATURE_ACTIVERECT @@ -1281,6 +1290,8 @@ makeMenus( GtkGameGlobals* globals ) #ifdef MEM_DEBUG (void)createAddItem( fileMenu, "Mem stats", (GCallback)handle_memstats, globals ); + (void)createAddItem( fileMenu, "Check for moves", + (GCallback)handle_movescheck, globals ); #endif #ifdef XWFEATURE_ACTIVERECT diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 57efe3d59..fca61f912 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -529,11 +529,46 @@ hostNameToIP( const XP_UCHAR* name ) return ip; } +typedef struct _PostArgs { + RelayConStorage* storage; + ReadState rs; + const XP_U8* msgbuf; + XP_U16 len; +} PostArgs; -static ssize_t -post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) +static gboolean +onGotData(gpointer user_data) { - const char* data = g_base64_encode( msgbuf, len ); + PostArgs* pa = (PostArgs*)user_data; + /* Now pull any data from the reply */ + // got "{"status": "ok", "dataLen": 14, "data": "AYQDiDAyMUEzQ0MyADw=", "err": "none"}" + json_object* reply = json_tokener_parse( pa->rs.ptr ); + json_object* replyData; + if ( json_object_object_get_ex( reply, "data", &replyData ) && !!replyData ) { + int len = json_object_array_length(replyData); + for ( int ii = 0; ii < len; ++ii ) { + json_object* datum = json_object_array_get_idx( replyData, ii ); + const char* str = json_object_get_string( datum ); + gsize out_len; + guchar* buf = g_base64_decode( (const gchar*)str, &out_len ); + process( pa->storage, buf, out_len ); + g_free( buf ); + } + (void)json_object_put( replyData ); + } + (void)json_object_put( reply ); + + g_free( pa->rs.ptr ); + g_free( pa ); + + return FALSE; +} + +static void* +postThread( void* arg ) +{ + PostArgs* pa = (PostArgs*)arg; + const char* data = g_base64_encode( pa->msgbuf, pa->len ); struct json_object* jobj = json_object_new_object(); struct json_object* jstr = json_object_new_string(data); // g_free( data ); @@ -541,10 +576,8 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) const char* asStr = json_object_to_json_string( jobj ); XP_LOGF( "%s: added str: %s", __func__, asStr ); - ReadState rs = { - .ptr = g_malloc0(1), - .curSize = 1L - }; + pa->rs.ptr = g_malloc0(1); + pa->rs.curSize = 1L; CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); XP_ASSERT(res == CURLE_OK); @@ -563,7 +596,7 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf)); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &rs ); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &pa->rs ); curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); res = curl_easy_perform(curl); @@ -577,28 +610,24 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) curl_global_cleanup(); (void)json_object_put( jobj ); - XP_LOGF( "%s(): got \"%s\"", __func__, rs.ptr ); + XP_LOGF( "%s(): got \"%s\"", __func__, pa->rs.ptr ); - /* Now pull any data from the reply */ - // got "{"status": "ok", "dataLen": 14, "data": "AYQDiDAyMUEzQ0MyADw=", "err": "none"}" - json_object* reply = json_tokener_parse( rs.ptr ); - json_object* replyData; - if ( json_object_object_get_ex( reply, "data", &replyData ) && !!replyData ) { - int len = json_object_array_length(replyData); - for ( int ii = 0; ii < len; ++ii ) { - json_object* datum = json_object_array_get_idx( replyData, ii ); - const char* str = json_object_get_string( datum ); - gsize out_len; - guchar* buf = g_base64_decode( (const gchar*)str, &out_len ); - process( storage, buf, out_len ); - g_free( buf ); - } - (void)json_object_put( replyData ); - } - (void)json_object_put( reply ); + (void)g_idle_add(onGotData, pa); - g_free( rs.ptr ); + return NULL; +} +static ssize_t +post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) +{ + PostArgs* pa = (PostArgs*)g_malloc0(sizeof(*pa)); + pa->storage = storage; + pa->msgbuf = msgbuf; + pa->len = len; + + pthread_t thread; + (void)pthread_create( &thread, NULL, postThread, (void*)pa ); + pthread_detach( thread ); return len; } From 37162e0471abef5331a3b8176ecd128aae17e8fb Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 21 Oct 2017 12:11:26 -0700 Subject: [PATCH 013/138] fix curses commit; refactor & cleanup --- xwords4/linux/cursesmain.c | 67 ++++++++++++++++-------- xwords4/linux/linuxmain.c | 15 ++++-- xwords4/linux/main.h | 1 + xwords4/linux/relaycon.c | 102 +++++++++++++++++++------------------ 4 files changed, 109 insertions(+), 76 deletions(-) diff --git a/xwords4/linux/cursesmain.c b/xwords4/linux/cursesmain.c index 48ddeec88..a308e7847 100644 --- a/xwords4/linux/cursesmain.c +++ b/xwords4/linux/cursesmain.c @@ -279,35 +279,57 @@ curses_util_userError( XW_UtilCtxt* uc, UtilErrID id ) } } /* curses_util_userError */ +static gint +ask_move( gpointer data ) +{ + CursesAppGlobals* globals = (CursesAppGlobals*)data; + CommonGlobals* cGlobals = &globals->cGlobals; + const char* answers[] = {"Ok", "Cancel", NULL}; + + if (0 == cursesask(globals, cGlobals->question, VSIZE(answers)-1, answers) ) { + BoardCtxt* board = cGlobals->game.board; + if ( board_commitTurn( board, XP_TRUE, XP_TRUE, NULL ) ) { + board_draw( board ); + } + } + + return FALSE; +} + +/* this needs to change!!! */ static void curses_util_notifyMove( XW_UtilCtxt* uc, XWStreamCtxt* stream ) { CursesAppGlobals* globals = (CursesAppGlobals*)uc->closure; - char* question; - const char* answers[3] = {NULL}; - short numAnswers = 0; - XP_Bool freeMe = XP_FALSE; - - question = strFromStream( stream ); - freeMe = XP_TRUE; - answers[numAnswers++] = "Cancel"; - answers[numAnswers++] = "Ok"; - - // result = okIndex == - cursesask( globals, question, numAnswers, answers ); - - if ( freeMe ) { - free( question ); - } + CommonGlobals* cGlobals = &globals->cGlobals; + XP_U16 len = stream_getSize( stream ); + XP_ASSERT( len <= VSIZE(cGlobals->question) ); + stream_getBytes( stream, cGlobals->question, len ); + (void)g_idle_add( ask_move, globals ); } /* curses_util_userQuery */ +static gint +ask_trade( gpointer data ) +{ + CursesAppGlobals* globals = (CursesAppGlobals*)data; + CommonGlobals* cGlobals = &globals->cGlobals; + + const char* buttons[] = { "Ok", "Cancel" }; + if (0 == cursesask( globals, cGlobals->question, VSIZE(buttons), buttons ) ) { + BoardCtxt* board = cGlobals->game.board; + if ( board_commitTurn( board, XP_TRUE, XP_TRUE, NULL ) ) { + board_draw( board ); + } + } + return FALSE; +} + static void curses_util_notifyTrade( XW_UtilCtxt* uc, const XP_UCHAR** tiles, XP_U16 nTiles ) { CursesAppGlobals* globals = (CursesAppGlobals*)uc->closure; formatConfirmTrade( &globals->cGlobals, tiles, nTiles ); - /* const char* buttons[] = { "Cancel", "Ok" }; */ - /* cursesask( globals, question, VSIZE(buttons), buttons ); */ + (void)g_idle_add( ask_trade, globals ); } static void @@ -1950,9 +1972,12 @@ cursesmain( XP_Bool isServer, LaunchParams* params ) (void)g_timeout_add_seconds( params->chatsInterval, chatsTimerFired, &g_globals ); } - g_globals.nextQueryTimeSecs = 1; - (void)g_timeout_add_seconds( g_globals.nextQueryTimeSecs, - queryTimerFired, &g_globals ); + + if ( params->useHTTP ) { + g_globals.nextQueryTimeSecs = 1; + (void)g_timeout_add_seconds( g_globals.nextQueryTimeSecs, + queryTimerFired, &g_globals ); + } XP_Bool opened = XP_FALSE; initCurses( &g_globals, &width, &height ); diff --git a/xwords4/linux/linuxmain.c b/xwords4/linux/linuxmain.c index e3ff095e6..a677328d8 100644 --- a/xwords4/linux/linuxmain.c +++ b/xwords4/linux/linuxmain.c @@ -634,6 +634,7 @@ typedef enum { ,CMD_CHAT ,CMD_USEUDP ,CMD_NOUDP + ,CMD_USEHTTP ,CMD_DROPSENDRELAY ,CMD_DROPRCVRELAY ,CMD_DROPSENDSMS @@ -752,6 +753,7 @@ static CmdInfoRec CmdInfoRecs[] = { ,{ CMD_CHAT, true, "send-chat", "send a chat every seconds" } ,{ CMD_USEUDP, false, "use-udp", "connect to relay new-style, via udp not tcp (on by default)" } ,{ CMD_NOUDP, false, "no-use-udp", "connect to relay old-style, via tcp not udp" } + ,{ CMD_USEHTTP, false, "use-http", "use relay's new http interfaces rather than sockets" } ,{ CMD_DROPSENDRELAY, false, "drop-send-relay", "start new games with relay send disabled" } ,{ CMD_DROPRCVRELAY, false, "drop-receive-relay", "start new games with relay receive disabled" } @@ -2398,6 +2400,9 @@ main( int argc, char** argv ) case CMD_NOUDP: mainParams.useUdp = false; break; + case CMD_USEHTTP: + mainParams.useHTTP = true; + break; case CMD_DROPSENDRELAY: mainParams.commsDisableds[COMMS_CONN_RELAY][1] = XP_TRUE; @@ -2487,10 +2492,10 @@ main( int argc, char** argv ) mainParams.dictDirs = g_slist_append( mainParams.dictDirs, "./" ); } - if ( isServer ) { - if ( mainParams.info.serverInfo.nRemotePlayers == 0 ) { - mainParams.pgi.serverRole = SERVER_STANDALONE; - } else { + if ( mainParams.info.serverInfo.nRemotePlayers == 0 ) { + mainParams.pgi.serverRole = SERVER_STANDALONE; + } else if ( isServer ) { + if ( mainParams.info.serverInfo.nRemotePlayers > 0 ) { mainParams.pgi.serverRole = SERVER_ISSERVER; } } else { @@ -2646,7 +2651,7 @@ main( int argc, char** argv ) if ( mainParams.useCurses ) { if ( mainParams.needsNewGame ) { /* curses doesn't have newgame dialog */ - usage( argv[0], "game params required for curses version" ); + usage( argv[0], "game params required for curses version, e.g. --name Eric --remote-player"); } else { #if defined PLATFORM_NCURSES cursesmain( isServer, &mainParams ); diff --git a/xwords4/linux/main.h b/xwords4/linux/main.h index 93605dc93..deab8b036 100644 --- a/xwords4/linux/main.h +++ b/xwords4/linux/main.h @@ -105,6 +105,7 @@ typedef struct LaunchParams { XP_Bool closeStdin; XP_Bool useCurses; XP_Bool useUdp; + XP_Bool useHTTP; XP_U16 splitPackets; XP_U16 chatsInterval; /* 0 means disabled */ XP_U16 askTimeout; diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index fca61f912..a56389de0 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -74,7 +74,6 @@ write_callback(void *contents, size_t size, size_t nmemb, void* data) { ReadState* rs = (ReadState*)data; XP_LOGF( "%s(size=%ld, nmemb=%ld)", __func__, size, nmemb ); - // void** pp = (void**)data; size_t oldLen = rs->curSize; const size_t newLength = size * nmemb; XP_ASSERT( (oldLen + newLength) > 0 ); @@ -85,6 +84,26 @@ write_callback(void *contents, size_t size, size_t nmemb, void* data) return newLength; } +static void +addJsonParams( CURL* curl, const char* name, json_object* param ) +{ + const char* asStr = json_object_to_json_string( param ); + XP_LOGF( "%s: added str: %s", __func__, asStr ); + + char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); + // char buf[4*1024]; + gchar* buf = g_strdup_printf( "%s=%s", name, curl_params ); + + curl_easy_setopt( curl, CURLOPT_POSTFIELDS, buf ); + curl_easy_setopt( curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf) ); + + g_free( buf ); + // size_t buflen = snprintf( buf, sizeof(buf), "ids=%s", curl_params); + // XP_ASSERT( buflen < sizeof(buf) ); + curl_free( curl_params ); + json_object_put( param ); +} + XP_Bool checkForMsgs( LaunchParams* params, XWGame* game ) { @@ -100,7 +119,6 @@ checkForMsgs( LaunchParams* params, XWGame* game ) } if ( !!idBuf[0] ) { - ReadState rs = { .ptr = g_malloc0(1), .curSize = 1L @@ -110,8 +128,6 @@ checkForMsgs( LaunchParams* params, XWGame* game ) json_object* ids = json_object_new_array(); json_object* idstr = json_object_new_string(idBuf); json_object_array_add(ids, idstr); - const char* asStr = json_object_to_json_string( ids ); - XP_LOGF( "%s: added str: %s", __func__, asStr ); CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); XP_ASSERT(res == CURLE_OK); @@ -121,15 +137,8 @@ checkForMsgs( LaunchParams* params, XWGame* game ) "http://localhost/xw4/relay.py/query"); curl_easy_setopt(curl, CURLOPT_POST, 1L); - char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); - char buf[4*1024]; - size_t buflen = snprintf( buf, sizeof(buf), "ids=%s", curl_params); - XP_ASSERT( buflen < sizeof(buf) ); - curl_free(curl_params); + addJsonParams( curl, "ids", ids ); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buf); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf)); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback ); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &rs ); curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); @@ -149,31 +158,33 @@ checkForMsgs( LaunchParams* params, XWGame* game ) if (res == CURLE_OK) { json_object* reply = json_tokener_parse( rs.ptr ); - json_object_object_foreach(reply, key, val) { - int len = json_object_array_length(val); - XP_LOGF( "%s: got key: %s of len %d", __func__, key, len ); - for ( int ii = 0; ii < len; ++ii ) { - json_object* forGame = json_object_array_get_idx(val, ii); - int len2 = json_object_array_length(forGame); - foundAny = foundAny || len2 > 0; - for ( int jj = 0; jj < len2; ++jj ) { - json_object* oneMove = json_object_array_get_idx(forGame, jj); - const char* asStr = json_object_get_string(oneMove); - gsize out_len; - guchar* buf = g_base64_decode( asStr, &out_len ); - XWStreamCtxt* stream = mem_stream_make( MPPARM(params->mpool) - params->vtMgr, params, - CHANNEL_NONE, NULL ); - stream_putBytes( stream, buf, out_len ); - g_free(buf); + if ( !!reply ) { + json_object_object_foreach(reply, key, val) { + int len = json_object_array_length(val); + XP_LOGF( "%s: got key: %s of len %d", __func__, key, len ); + for ( int ii = 0; ii < len; ++ii ) { + json_object* forGame = json_object_array_get_idx(val, ii); + int len2 = json_object_array_length(forGame); + foundAny = foundAny || len2 > 0; + for ( int jj = 0; jj < len2; ++jj ) { + json_object* oneMove = json_object_array_get_idx(forGame, jj); + const char* asStr = json_object_get_string(oneMove); + gsize out_len; + guchar* buf = g_base64_decode( asStr, &out_len ); + XWStreamCtxt* stream = mem_stream_make( MPPARM(params->mpool) + params->vtMgr, params, + CHANNEL_NONE, NULL ); + stream_putBytes( stream, buf, out_len ); + g_free(buf); - CommsAddrRec addr = {0}; - addr_addType( &addr, COMMS_CONN_RELAY ); - XP_Bool handled = game_receiveMessage( game, stream, &addr ); - XP_LOGF( "%s(): game_receiveMessage() => %d", __func__, handled ); - stream_destroy( stream ); + CommsAddrRec addr = {0}; + addr_addType( &addr, COMMS_CONN_RELAY ); + XP_Bool handled = game_receiveMessage( game, stream, &addr ); + XP_LOGF( "%s(): game_receiveMessage() => %d", __func__, handled ); + stream_destroy( stream ); - foundAny = XP_TRUE; + foundAny = XP_TRUE; + } } } } @@ -568,13 +579,11 @@ static void* postThread( void* arg ) { PostArgs* pa = (PostArgs*)arg; - const char* data = g_base64_encode( pa->msgbuf, pa->len ); + char* data = g_base64_encode( pa->msgbuf, pa->len ); struct json_object* jobj = json_object_new_object(); struct json_object* jstr = json_object_new_string(data); - // g_free( data ); + g_free( data ); json_object_object_add( jobj, "data", jstr ); - const char* asStr = json_object_to_json_string( jobj ); - XP_LOGF( "%s: added str: %s", __func__, asStr ); pa->rs.ptr = g_malloc0(1); pa->rs.curSize = 1L; @@ -586,14 +595,7 @@ postThread( void* arg ) curl_easy_setopt(curl, CURLOPT_URL, "http://localhost/xw4/relay.py/post"); curl_easy_setopt(curl, CURLOPT_POST, 1L); - char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); - char buf[4*1024]; - size_t buflen = snprintf( buf, sizeof(buf), "params=%s", curl_params); - XP_ASSERT( buflen < sizeof(buf) ); - curl_free(curl_params); - - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, buf); - curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf)); + addJsonParams( curl, "params", jobj ); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback ); curl_easy_setopt(curl, CURLOPT_WRITEDATA, &pa->rs ); @@ -608,11 +610,11 @@ postThread( void* arg ) /* always cleanup */ curl_easy_cleanup(curl); curl_global_cleanup(); - (void)json_object_put( jobj ); XP_LOGF( "%s(): got \"%s\"", __func__, pa->rs.ptr ); - (void)g_idle_add(onGotData, pa); + // Put the data on the main thread for processing + (void)g_idle_add( onGotData, pa ); return NULL; } @@ -635,7 +637,7 @@ static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) { ssize_t nSent; - if (1) { + if ( storage->params->useHTTP ) { nSent = post( storage, msgbuf, len ); } else { nSent = sendto( storage->socket, msgbuf, len, 0, /* flags */ From 284888df44aac13c266e6b0c6f9bc75dade593fd Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 21 Oct 2017 12:13:19 -0700 Subject: [PATCH 014/138] add timeoutSecs param --- xwords4/android/scripts/relay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index f04386fb0..b1a18ef8f 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -11,14 +11,14 @@ try: except ImportError: apacheAvailable = False -def post(req, params): +def post(req, params, timeoutSecs = 1): err = 'none' dataLen = 0 jobj = json.loads(params) data = base64.b64decode(jobj['data']) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(1) # seconds + sock.settimeout(timeoutSecs) # seconds addr = ("127.0.0.1", 10997) sock.sendto(data, addr) From a6602fabe0e3cd9dc2618f67bfe63e97c4854700 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 21 Oct 2017 14:24:38 -0700 Subject: [PATCH 015/138] hide menuitem when not in http mode --- xwords4/linux/gtkboard.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xwords4/linux/gtkboard.c b/xwords4/linux/gtkboard.c index f25622306..bbfb79595 100644 --- a/xwords4/linux/gtkboard.c +++ b/xwords4/linux/gtkboard.c @@ -1290,8 +1290,10 @@ makeMenus( GtkGameGlobals* globals ) #ifdef MEM_DEBUG (void)createAddItem( fileMenu, "Mem stats", (GCallback)handle_memstats, globals ); - (void)createAddItem( fileMenu, "Check for moves", - (GCallback)handle_movescheck, globals ); + if ( globals->cGlobals.params->useHTTP ) { + (void)createAddItem( fileMenu, "Check for moves", + (GCallback)handle_movescheck, globals ); + } #endif #ifdef XWFEATURE_ACTIVERECT From ec7fde3b6276979a623ddd8886fef760523bf05b Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 21 Oct 2017 14:59:10 -0700 Subject: [PATCH 016/138] add relayID to DB and to table --- xwords4/linux/gamesdb.c | 59 +++++++++++++++++++++++----------------- xwords4/linux/gamesdb.h | 1 + xwords4/linux/gtkboard.c | 2 +- xwords4/linux/gtkmain.c | 5 +++- 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/xwords4/linux/gamesdb.c b/xwords4/linux/gamesdb.c index 6b599e2bc..9e623239f 100644 --- a/xwords4/linux/gamesdb.c +++ b/xwords4/linux/gamesdb.c @@ -52,6 +52,7 @@ openGamesDB( const char* dbName ) ",inviteInfo BLOB" ",room VARCHAR(32)" ",connvia VARCHAR(32)" + ",relayid VARCHAR(32)" ",ended INT(1)" ",turn INT(2)" ",local INT(1)" @@ -199,11 +200,12 @@ addSnapshot( CommonGlobals* cGlobals ) void summarize( CommonGlobals* cGlobals ) { - XP_S16 nMoves = model_getNMoves( cGlobals->game.model ); - XP_Bool gameOver = server_getGameIsOver( cGlobals->game.server ); + const XWGame* game = &cGlobals->game; + XP_S16 nMoves = model_getNMoves( game->model ); + XP_Bool gameOver = server_getGameIsOver( game->server ); XP_Bool isLocal; - XP_S16 turn = server_getCurrentTurn( cGlobals->game.server, &isLocal ); - XP_U32 lastMoveTime = server_getLastMoveTime( cGlobals->game.server ); + XP_S16 turn = server_getCurrentTurn( game->server, &isLocal ); + XP_U32 lastMoveTime = server_getLastMoveTime( game->server ); XP_U16 seed = 0; XP_S16 nMissing = 0; XP_U16 nTotal = cGlobals->gi->nPlayers; @@ -214,10 +216,11 @@ summarize( CommonGlobals* cGlobals ) // gchar* connvia = "local"; gchar connvia[128] = {0}; + XP_UCHAR relayID[32] = {0}; - if ( !!cGlobals->game.comms ) { - nMissing = server_getMissingPlayers( cGlobals->game.server ); - comms_getAddr( cGlobals->game.comms, &addr ); + if ( !!game->comms ) { + nMissing = server_getMissingPlayers( game->server ); + comms_getAddr( game->comms, &addr ); CommsConnType typ; for ( XP_U32 st = 0; addr_iter( &addr, &typ, &st ); ) { if ( !!connvia[0] ) { @@ -242,18 +245,21 @@ summarize( CommonGlobals* cGlobals ) break; } } - seed = comms_getChannelSeed( cGlobals->game.comms ); + seed = comms_getChannelSeed( game->comms ); + XP_U16 len = VSIZE(relayID); + (void)comms_getRelayID( game->comms, relayID, &len ); } else { strcat( connvia, "local" ); } const char* fmt = "UPDATE games " - " SET room='%s', ended=%d, turn=%d, local=%d, ntotal=%d, nmissing=%d, " - " nmoves=%d, seed=%d, gameid=%d, connvia='%s', lastMoveTime=%d" + " SET room='%s', ended=%d, turn=%d, local=%d, ntotal=%d, " + " nmissing=%d, nmoves=%d, seed=%d, gameid=%d, connvia='%s', " + " relayid='%s', lastMoveTime=%d" " WHERE rowid=%lld"; XP_UCHAR buf[256]; snprintf( buf, sizeof(buf), fmt, room, gameOver?1:0, turn, isLocal?1:0, - nTotal, nMissing, nMoves, seed, gameID, connvia, lastMoveTime, + nTotal, nMissing, nMoves, seed, gameID, connvia, relayID, lastMoveTime, cGlobals->selRow ); XP_LOGF( "query: %s", buf ); sqlite3_stmt* stmt = NULL; @@ -310,7 +316,7 @@ getGameInfo( sqlite3* pDb, sqlite3_int64 rowid, GameInfo* gib ) { XP_Bool success = XP_FALSE; const char* fmt = "SELECT room, ended, turn, local, nmoves, ntotal, nmissing, " - "seed, connvia, gameid, lastMoveTime, snap " + "seed, connvia, gameid, lastMoveTime, relayid, snap " "FROM games WHERE rowid = %lld"; XP_UCHAR query[256]; snprintf( query, sizeof(query), fmt, rowid ); @@ -321,25 +327,28 @@ getGameInfo( sqlite3* pDb, sqlite3_int64 rowid, GameInfo* gib ) result = sqlite3_step( ppStmt ); if ( SQLITE_ROW == result ) { success = XP_TRUE; - getColumnText( ppStmt, 0, gib->room, sizeof(gib->room) ); - gib->gameOver = 1 == sqlite3_column_int( ppStmt, 1 ); - gib->turn = sqlite3_column_int( ppStmt, 2 ); - gib->turnLocal = 1 == sqlite3_column_int( ppStmt, 3 ); - gib->nMoves = sqlite3_column_int( ppStmt, 4 ); - gib->nTotal = sqlite3_column_int( ppStmt, 5 ); - gib->nMissing = sqlite3_column_int( ppStmt, 6 ); - gib->seed = sqlite3_column_int( ppStmt, 7 ); - getColumnText( ppStmt, 8, gib->conn, sizeof(gib->conn) ); - gib->gameID = sqlite3_column_int( ppStmt, 9 ); - gib->lastMoveTime = sqlite3_column_int( ppStmt, 10 ); + int col = 0; + getColumnText( ppStmt, col++, gib->room, sizeof(gib->room) ); + gib->gameOver = 1 == sqlite3_column_int( ppStmt, col++ ); + gib->turn = sqlite3_column_int( ppStmt, col++ ); + gib->turnLocal = 1 == sqlite3_column_int( ppStmt, col++ ); + gib->nMoves = sqlite3_column_int( ppStmt, col++ ); + gib->nTotal = sqlite3_column_int( ppStmt, col++ ); + gib->nMissing = sqlite3_column_int( ppStmt, col++ ); + gib->seed = sqlite3_column_int( ppStmt, col++ ); + getColumnText( ppStmt, col++, gib->conn, sizeof(gib->conn) ); + gib->gameID = sqlite3_column_int( ppStmt, col++ ); + gib->lastMoveTime = sqlite3_column_int( ppStmt, col++ ); + getColumnText( ppStmt, col++, gib->relayID, sizeof(gib->relayID) ); snprintf( gib->name, sizeof(gib->name), "Game %lld", rowid ); #ifdef PLATFORM_GTK /* Load the snapshot */ GdkPixbuf* snap = NULL; - const XP_U8* ptr = sqlite3_column_blob( ppStmt, 11 ); + int snapCol = col++; + const XP_U8* ptr = sqlite3_column_blob( ppStmt, snapCol ); if ( !!ptr ) { - int size = sqlite3_column_bytes( ppStmt, 11 ); + int size = sqlite3_column_bytes( ppStmt, snapCol ); /* Skip the version that's written in */ ptr += sizeof(XP_U16); size -= sizeof(XP_U16); GInputStream* istr = g_memory_input_stream_new_from_data( ptr, size, NULL ); diff --git a/xwords4/linux/gamesdb.h b/xwords4/linux/gamesdb.h index b7744cdc8..f8ac3316a 100644 --- a/xwords4/linux/gamesdb.h +++ b/xwords4/linux/gamesdb.h @@ -31,6 +31,7 @@ typedef struct _GameInfo { XP_UCHAR name[128]; XP_UCHAR room[128]; XP_UCHAR conn[128]; + XP_UCHAR relayID[32]; #ifdef PLATFORM_GTK GdkPixbuf* snap; #endif diff --git a/xwords4/linux/gtkboard.c b/xwords4/linux/gtkboard.c index bbfb79595..505724679 100644 --- a/xwords4/linux/gtkboard.c +++ b/xwords4/linux/gtkboard.c @@ -1315,7 +1315,7 @@ static void disenable_buttons( GtkGameGlobals* globals ) { XP_U16 nPending = server_getPendingRegs( globals->cGlobals.game.server ); - if ( !globals->invite_button && 0 < nPending ) { + if ( !globals->invite_button && 0 < nPending && !!globals->buttons_hbox ) { globals->invite_button = addButton( globals->buttons_hbox, "Invite", G_CALLBACK(handle_invite_button), globals ); diff --git a/xwords4/linux/gtkmain.c b/xwords4/linux/gtkmain.c index 63ec72f3a..913a26007 100644 --- a/xwords4/linux/gtkmain.c +++ b/xwords4/linux/gtkmain.c @@ -76,7 +76,7 @@ findOpenGame( const GtkAppGlobals* apg, sqlite3_int64 rowid ) } enum { ROW_ITEM, ROW_THUMB, NAME_ITEM, ROOM_ITEM, GAMEID_ITEM, SEED_ITEM, - CONN_ITEM, OVER_ITEM, TURN_ITEM, LOCAL_ITEM, NMOVES_ITEM, NTOTAL_ITEM, + CONN_ITEM, RELAYID_ITEM, OVER_ITEM, TURN_ITEM, LOCAL_ITEM, NMOVES_ITEM, NTOTAL_ITEM, MISSING_ITEM, LASTTURN_ITEM, N_ITEMS }; static void @@ -167,6 +167,7 @@ init_games_list( GtkAppGlobals* apg ) addTextColumn( list, "GameID", GAMEID_ITEM ); addTextColumn( list, "Seed", SEED_ITEM ); addTextColumn( list, "Conn. via", CONN_ITEM ); + addTextColumn( list, "RelayID", RELAYID_ITEM ); addTextColumn( list, "Ended", OVER_ITEM ); addTextColumn( list, "Turn", TURN_ITEM ); addTextColumn( list, "Local", LOCAL_ITEM ); @@ -183,6 +184,7 @@ init_games_list( GtkAppGlobals* apg ) G_TYPE_INT, /* GAMEID_ITEM */ G_TYPE_INT, /* SEED_ITEM */ G_TYPE_STRING, /* CONN_ITEM */ + G_TYPE_STRING, /*RELAYID_ITEM */ G_TYPE_BOOLEAN, /* OVER_ITEM */ G_TYPE_INT, /* TURN_ITEM */ G_TYPE_STRING, /* LOCAL_ITEM */ @@ -239,6 +241,7 @@ add_to_list( GtkWidget* list, sqlite3_int64 rowid, XP_Bool isNew, GAMEID_ITEM, gib->gameID, SEED_ITEM, gib->seed, CONN_ITEM, gib->conn, + RELAYID_ITEM, gib->relayID, TURN_ITEM, gib->turn, OVER_ITEM, gib->gameOver, LOCAL_ITEM, localString, From a65af7995305bc037e530572cb7d79235ca71d3b Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 21 Oct 2017 15:23:46 -0700 Subject: [PATCH 017/138] add check for being on main thread --- xwords4/linux/relaycon.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index a56389de0..491d1a3af 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -29,6 +29,7 @@ #include "comtypes.h" typedef struct _RelayConStorage { + pthread_t mainThread; int socket; RelayConnProcs procs; void* procsClosure; @@ -202,6 +203,8 @@ relaycon_init( LaunchParams* params, const RelayConnProcs* procs, XP_MEMCPY( &storage->procs, procs, sizeof(storage->procs) ); storage->procsClosure = procsClosure; + storage->mainThread = pthread_self(); + storage->socket = socket( AF_INET, SOCK_DGRAM, IPPROTO_UDP ); (*procs->socketAdded)( storage, storage->socket, relaycon_receive ); @@ -360,6 +363,12 @@ relaycon_deleted( LaunchParams* params, const XP_UCHAR* devID, sendIt( storage, tmpbuf, indx ); } +static XP_Bool +onMainThread( RelayConStorage* storage ) +{ + return storage->mainThread = pthread_self(); +} + static void sendAckIf( RelayConStorage* storage, const MsgHeader* header ) { @@ -579,6 +588,7 @@ static void* postThread( void* arg ) { PostArgs* pa = (PostArgs*)arg; + XP_ASSERT( !onMainThread(pa->storage) ); char* data = g_base64_encode( pa->msgbuf, pa->len ); struct json_object* jobj = json_object_new_object(); struct json_object* jstr = json_object_new_string(data); From 47a048d553352306f0d8c3febe95cab77d0e9670 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 21 Oct 2017 16:05:54 -0700 Subject: [PATCH 018/138] fetch relayIDs from db --- xwords4/linux/gamesdb.c | 34 +++++++++++++++++++++++++ xwords4/linux/gamesdb.h | 2 ++ xwords4/linux/relaycon.c | 54 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) diff --git a/xwords4/linux/gamesdb.c b/xwords4/linux/gamesdb.c index 9e623239f..33f27c2db 100644 --- a/xwords4/linux/gamesdb.c +++ b/xwords4/linux/gamesdb.c @@ -311,6 +311,40 @@ listGames( sqlite3* pDb ) return list; } +GHashTable* +getRowsToRelayIDsMap( sqlite3* pDb ) +{ + GHashTable* table = g_hash_table_new( g_int64_hash, g_int64_equal ); + sqlite3_stmt *ppStmt; + int result = sqlite3_prepare_v2( pDb, "SELECT rowid, relayid FROM games", + -1, &ppStmt, NULL ); + assertPrintResult( pDb, result, SQLITE_OK ); + XP_USE( result ); + while ( NULL != ppStmt ) { + switch( sqlite3_step( ppStmt ) ) { + case SQLITE_ROW: /* have data */ + { + sqlite3_int64* key = g_malloc( sizeof( *key ) ); + *key = sqlite3_column_int64( ppStmt, 0 ); + XP_UCHAR relayID[32]; + getColumnText( ppStmt, 1, relayID, VSIZE(relayID) ); + gpointer value = g_strdup( relayID ); + g_hash_table_insert( table, key, value ); + } + break; + case SQLITE_DONE: + sqlite3_finalize( ppStmt ); + ppStmt = NULL; + break; + default: + XP_ASSERT( 0 ); + break; + } + } + + return table; +} + XP_Bool getGameInfo( sqlite3* pDb, sqlite3_int64 rowid, GameInfo* gib ) { diff --git a/xwords4/linux/gamesdb.h b/xwords4/linux/gamesdb.h index f8ac3316a..214f324f9 100644 --- a/xwords4/linux/gamesdb.h +++ b/xwords4/linux/gamesdb.h @@ -56,6 +56,8 @@ void summarize( CommonGlobals* cGlobals ); /* Return GSList whose data is (ptrs to) rowids */ GSList* listGames( sqlite3* dbp ); +GHashTable *getRowsToRelayIDsMap(sqlite3* dbp); + XP_Bool getGameInfo( sqlite3* dbp, sqlite3_int64 rowid, GameInfo* gib ); void getRowsForGameID( sqlite3* dbp, XP_U32 gameID, sqlite3_int64* rowids, int* nRowIDs ); diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 491d1a3af..2d011277e 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -27,9 +27,15 @@ #include "relaycon.h" #include "linuxmain.h" #include "comtypes.h" +#include "gamesdb.h" + +#define MAX_MOVE_CHECK_SECS ((XP_U16)(60 * 60 * 24)) typedef struct _RelayConStorage { pthread_t mainThread; + guint moveCheckerID; + XP_U16 nextMoveCheckSecs; + int socket; RelayConnProcs procs; void* procsClosure; @@ -48,6 +54,7 @@ static RelayConStorage* getStorage( LaunchParams* params ); static XP_U32 hostNameToIP( const XP_UCHAR* name ); static gboolean relaycon_receive( GIOChannel *source, GIOCondition condition, gpointer data ); +static void scheule_next_check( RelayConStorage* storage ); static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ); static size_t addVLIStr( XP_U8* buf, size_t len, const XP_UCHAR* str ); static void getNetString( const XP_U8** ptr, XP_U16 len, XP_UCHAR* buf ); @@ -216,6 +223,10 @@ relaycon_init( LaunchParams* params, const RelayConnProcs* procs, storage->params = params; storage->proto = XWPDEV_PROTO_VERSION_1; + + if ( params->useHTTP ) { + scheule_next_check( storage ); + } } /* Send existing relay-assigned rDevID to relay, or empty string if we have @@ -643,6 +654,49 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) return len; } +static gboolean +checkForMoves( gpointer user_data ) +{ + LOG_FUNC(); + RelayConStorage* storage = (RelayConStorage*)user_data; + XP_ASSERT( onMainThread(storage) ); + + sqlite3* dbp = storage->params->pDb; + GHashTable* map = getRowsToRelayIDsMap( dbp ); + GList* ids = g_hash_table_get_values( map ); + for ( GList* iter = ids; !!iter; iter = iter->next ) { + gpointer data = iter->data; + XP_LOGF( "checkForMoves: got id: %s", (char*)data ); + } + g_list_free( ids ); + g_hash_table_destroy( map ); + + scheule_next_check( storage ); + return FALSE; +} + +static void +scheule_next_check( RelayConStorage* storage ) +{ + XP_ASSERT( onMainThread(storage) ); + + if ( storage->moveCheckerID != 0 ) { + g_source_remove( storage->moveCheckerID ); + storage->moveCheckerID = 0; + } + + storage->nextMoveCheckSecs *= 2; + if ( storage->nextMoveCheckSecs > MAX_MOVE_CHECK_SECS ) { + storage->nextMoveCheckSecs = MAX_MOVE_CHECK_SECS; + } else if ( storage->nextMoveCheckSecs == 0 ) { + storage->nextMoveCheckSecs = 1; + } + + storage->moveCheckerID = g_timeout_add( 1000 * storage->nextMoveCheckSecs, + checkForMoves, storage ); + XP_ASSERT( storage->moveCheckerID != 0 ); +} + static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) { From 43ffb156fcfc20e3a153265d5ab2addc05f4f928 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 22 Oct 2017 07:30:14 -0700 Subject: [PATCH 019/138] wip: successfully get list of moves --- xwords4/linux/relaycon.c | 74 +++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 2d011277e..3c514cd59 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -96,19 +96,17 @@ static void addJsonParams( CURL* curl, const char* name, json_object* param ) { const char* asStr = json_object_to_json_string( param ); - XP_LOGF( "%s: added str: %s", __func__, asStr ); char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); - // char buf[4*1024]; gchar* buf = g_strdup_printf( "%s=%s", name, curl_params ); + XP_LOGF( "%s: added param: %s", __func__, buf ); + curl_free( curl_params ); curl_easy_setopt( curl, CURLOPT_POSTFIELDS, buf ); curl_easy_setopt( curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf) ); - g_free( buf ); - // size_t buflen = snprintf( buf, sizeof(buf), "ids=%s", curl_params); - // XP_ASSERT( buflen < sizeof(buf) ); - curl_free( curl_params ); + // Can't free the buf!! Well, maybe after the send... + // g_free( buf ); json_object_put( param ); } @@ -654,6 +652,56 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) return len; } +typedef struct _QueryArgs { + RelayConStorage* storage; + GSList* ids; + ReadState rs; +} QueryArgs; + +static void* +queryThread( void* arg ) +{ + QueryArgs* qa = (QueryArgs*)arg; + GSList* ids = qa->ids; + qa->rs.ptr = g_malloc0(1); + qa->rs.curSize = 1L; + + json_object* jIds = json_object_new_array(); + for ( GSList* iter = ids; !!iter; iter = iter->next ) { + json_object* idstr = json_object_new_string( iter->data ); + json_object_array_add(jIds, idstr); + } + + CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); + XP_ASSERT(res == CURLE_OK); + CURL* curl = curl_easy_init(); + + curl_easy_setopt(curl, CURLOPT_URL, + "http://localhost/xw4/relay.py/query"); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + + addJsonParams( curl, "ids", jIds ); + + curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); + curl_easy_setopt( curl, CURLOPT_WRITEDATA, &qa->rs ); + curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); + + res = curl_easy_perform( curl ); + + XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); + /* Check for errors */ + if (res != CURLE_OK) { + XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); + } + /* always cleanup */ + curl_easy_cleanup(curl); + curl_global_cleanup(); + + XP_LOGF( "%s(): got <<%s>>", __func__, qa->rs.ptr ); + + return NULL; +} + static gboolean checkForMoves( gpointer user_data ) { @@ -661,16 +709,24 @@ checkForMoves( gpointer user_data ) RelayConStorage* storage = (RelayConStorage*)user_data; XP_ASSERT( onMainThread(storage) ); + QueryArgs* qa = (QueryArgs*)g_malloc0(sizeof(*qa)); + qa->storage = storage; + sqlite3* dbp = storage->params->pDb; GHashTable* map = getRowsToRelayIDsMap( dbp ); - GList* ids = g_hash_table_get_values( map ); - for ( GList* iter = ids; !!iter; iter = iter->next ) { + GList* values = g_hash_table_get_values( map ); + for ( GList* iter = values; !!iter; iter = iter->next ) { gpointer data = iter->data; XP_LOGF( "checkForMoves: got id: %s", (char*)data ); + qa->ids = g_slist_prepend( qa->ids, g_strdup(data) ); } - g_list_free( ids ); + g_list_free( values ); g_hash_table_destroy( map ); + pthread_t thread; + (void)pthread_create( &thread, NULL, queryThread, (void*)qa ); + pthread_detach( thread ); + scheule_next_check( storage ); return FALSE; } From f49c81462c12afe10030736900dc087f42dcc332 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 22 Oct 2017 09:29:15 -0700 Subject: [PATCH 020/138] wip: received messages dispatched to games --- xwords4/linux/cursesmain.c | 28 ------ xwords4/linux/gamesdb.c | 15 +-- xwords4/linux/gamesdb.h | 3 +- xwords4/linux/gtkboard.c | 7 +- xwords4/linux/gtkmain.c | 12 +++ xwords4/linux/relaycon.c | 191 +++++++++++++++++-------------------- xwords4/linux/relaycon.h | 4 +- 7 files changed, 114 insertions(+), 146 deletions(-) diff --git a/xwords4/linux/cursesmain.c b/xwords4/linux/cursesmain.c index a308e7847..8c7922a99 100644 --- a/xwords4/linux/cursesmain.c +++ b/xwords4/linux/cursesmain.c @@ -1773,28 +1773,6 @@ cursesErrorMsgRcvd( void* closure, const XP_UCHAR* msg ) } } -static gboolean -queryTimerFired( gpointer data ) -{ - LOG_FUNC(); - CursesAppGlobals* globals = (CursesAppGlobals*)data; - - XWGame* game = &globals->cGlobals.game; - if ( !!game->comms ) { - XP_ASSERT( globals->nextQueryTimeSecs > 0 ); - if ( checkForMsgs( globals->cGlobals.params, game ) ) { - globals->nextQueryTimeSecs = 1; - } else { - globals->nextQueryTimeSecs *= 2; - } - - (void)g_timeout_add_seconds( globals->nextQueryTimeSecs, - queryTimerFired, &g_globals ); - } - - return FALSE; -} - static gboolean chatsTimerFired( gpointer data ) { @@ -1973,12 +1951,6 @@ cursesmain( XP_Bool isServer, LaunchParams* params ) &g_globals ); } - if ( params->useHTTP ) { - g_globals.nextQueryTimeSecs = 1; - (void)g_timeout_add_seconds( g_globals.nextQueryTimeSecs, - queryTimerFired, &g_globals ); - } - XP_Bool opened = XP_FALSE; initCurses( &g_globals, &width, &height ); diff --git a/xwords4/linux/gamesdb.c b/xwords4/linux/gamesdb.c index 33f27c2db..7d8f1e6d9 100644 --- a/xwords4/linux/gamesdb.c +++ b/xwords4/linux/gamesdb.c @@ -312,11 +312,11 @@ listGames( sqlite3* pDb ) } GHashTable* -getRowsToRelayIDsMap( sqlite3* pDb ) +getRelayIDsToRowsMap( sqlite3* pDb ) { - GHashTable* table = g_hash_table_new( g_int64_hash, g_int64_equal ); + GHashTable* table = g_hash_table_new( g_str_hash, g_str_equal ); sqlite3_stmt *ppStmt; - int result = sqlite3_prepare_v2( pDb, "SELECT rowid, relayid FROM games", + int result = sqlite3_prepare_v2( pDb, "SELECT relayid, rowid FROM games", -1, &ppStmt, NULL ); assertPrintResult( pDb, result, SQLITE_OK ); XP_USE( result ); @@ -324,12 +324,13 @@ getRowsToRelayIDsMap( sqlite3* pDb ) switch( sqlite3_step( ppStmt ) ) { case SQLITE_ROW: /* have data */ { - sqlite3_int64* key = g_malloc( sizeof( *key ) ); - *key = sqlite3_column_int64( ppStmt, 0 ); XP_UCHAR relayID[32]; - getColumnText( ppStmt, 1, relayID, VSIZE(relayID) ); - gpointer value = g_strdup( relayID ); + getColumnText( ppStmt, 0, relayID, VSIZE(relayID) ); + gpointer key = g_strdup( relayID ); + sqlite3_int64* value = g_malloc( sizeof( *key ) ); + *value = sqlite3_column_int64( ppStmt, 1 ); g_hash_table_insert( table, key, value ); + /* XP_LOGF( "%s(): added map %s => %lld", __func__, (char*)key, *value ); */ } break; case SQLITE_DONE: diff --git a/xwords4/linux/gamesdb.h b/xwords4/linux/gamesdb.h index 214f324f9..086301570 100644 --- a/xwords4/linux/gamesdb.h +++ b/xwords4/linux/gamesdb.h @@ -56,7 +56,8 @@ void summarize( CommonGlobals* cGlobals ); /* Return GSList whose data is (ptrs to) rowids */ GSList* listGames( sqlite3* dbp ); -GHashTable *getRowsToRelayIDsMap(sqlite3* dbp); +/* Mapping of relayID -> rowid */ +GHashTable* getRelayIDsToRowsMap( sqlite3* pDb ); XP_Bool getGameInfo( sqlite3* dbp, sqlite3_int64 rowid, GameInfo* gib ); void getRowsForGameID( sqlite3* dbp, XP_U32 gameID, sqlite3_int64* rowids, diff --git a/xwords4/linux/gtkboard.c b/xwords4/linux/gtkboard.c index 505724679..eb6c9b573 100644 --- a/xwords4/linux/gtkboard.c +++ b/xwords4/linux/gtkboard.c @@ -341,6 +341,8 @@ relay_connd_gtk( void* closure, XP_UCHAR* const room, char buf[256]; if ( allHere ) { + /* disable for now. Seeing this too often */ + skip = XP_TRUE; snprintf( buf, sizeof(buf), "All expected players have joined in %s. Play!", room ); } else { @@ -1179,10 +1181,7 @@ handle_memstats( GtkWidget* XP_UNUSED(widget), GtkGameGlobals* globals ) static void handle_movescheck( GtkWidget* XP_UNUSED(widget), GtkGameGlobals* globals ) { - LaunchParams* params = globals->cGlobals.params; - if ( checkForMsgs( params, &globals->cGlobals.game ) ) { - board_draw( globals->cGlobals.game.board ); - } + checkForMsgsNow( globals->cGlobals.params ); } #endif diff --git a/xwords4/linux/gtkmain.c b/xwords4/linux/gtkmain.c index 913a26007..ad3119513 100644 --- a/xwords4/linux/gtkmain.c +++ b/xwords4/linux/gtkmain.c @@ -696,6 +696,17 @@ gtkGotBuf( void* closure, const CommsAddrRec* from, XP_USE( seed ); } +static void +gtkGotMsgForRow( void* closure, const CommsAddrRec* from, + sqlite3_int64 rowid, const XP_U8* buf, XP_U16 len ) +{ + XP_LOGF( "%s(): got msg of len %d for row %lld", __func__, len, rowid ); + GtkAppGlobals* apg = (GtkAppGlobals*)closure; + // LaunchParams* params = apg->params; + (void)feedBufferGTK( apg, rowid, buf, len, from ); + LOG_RETURN_VOID(); +} + static gint requestMsgs( gpointer data ) { @@ -850,6 +861,7 @@ gtkmain( LaunchParams* params ) if ( params->useUdp ) { RelayConnProcs procs = { .msgReceived = gtkGotBuf, + .msgForRow = gtkGotMsgForRow, .msgNoticeReceived = gtkNoticeRcvd, .devIDReceived = gtkDevIDReceived, .msgErrorMsg = gtkErrorMsgRcvd, diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 3c514cd59..caf1777e8 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -51,10 +51,12 @@ typedef struct _MsgHeader { } MsgHeader; static RelayConStorage* getStorage( LaunchParams* params ); +static XP_Bool onMainThread( RelayConStorage* storage ); static XP_U32 hostNameToIP( const XP_UCHAR* name ); static gboolean relaycon_receive( GIOChannel *source, GIOCondition condition, gpointer data ); -static void scheule_next_check( RelayConStorage* storage ); +static void schedule_next_check( RelayConStorage* storage ); +static void reset_schedule_check_interval( RelayConStorage* storage ); static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ); static size_t addVLIStr( XP_U8* buf, size_t len, const XP_UCHAR* str ); static void getNetString( const XP_U8** ptr, XP_U16 len, XP_UCHAR* buf ); @@ -96,7 +98,8 @@ static void addJsonParams( CURL* curl, const char* name, json_object* param ) { const char* asStr = json_object_to_json_string( param ); - + XP_LOGF( "%s: adding param: %s", __func__, asStr ); + char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); gchar* buf = g_strdup_printf( "%s=%s", name, curl_params ); XP_LOGF( "%s: added param: %s", __func__, buf ); @@ -110,93 +113,12 @@ addJsonParams( CURL* curl, const char* name, json_object* param ) json_object_put( param ); } -XP_Bool -checkForMsgs( LaunchParams* params, XWGame* game ) +void +checkForMsgsNow( LaunchParams* params ) { - XP_Bool foundAny = false; - XP_UCHAR idBuf[64]; - if ( !!game->comms ) { - XP_U16 len = VSIZE(idBuf); - if ( comms_getRelayID( game->comms, idBuf, &len ) ) { - XP_LOGF( "%s: got %s", __func__, idBuf ); - } else { - idBuf[0] = '\0'; - } - } - - if ( !!idBuf[0] ) { - ReadState rs = { - .ptr = g_malloc0(1), - .curSize = 1L - }; - - /* build a json array of relayIDs, then stringify it */ - json_object* ids = json_object_new_array(); - json_object* idstr = json_object_new_string(idBuf); - json_object_array_add(ids, idstr); - - CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); - XP_ASSERT(res == CURLE_OK); - CURL* curl = curl_easy_init(); - - curl_easy_setopt(curl, CURLOPT_URL, - "http://localhost/xw4/relay.py/query"); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - - addJsonParams( curl, "ids", ids ); - - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &rs ); - curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); - - res = curl_easy_perform(curl); - - XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); - /* Check for errors */ - if (res != CURLE_OK) { - XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); - } - /* always cleanup */ - curl_easy_cleanup(curl); - curl_global_cleanup(); - - XP_LOGF( "%s(): got <<%s>>", __func__, rs.ptr ); - - if (res == CURLE_OK) { - json_object* reply = json_tokener_parse( rs.ptr ); - if ( !!reply ) { - json_object_object_foreach(reply, key, val) { - int len = json_object_array_length(val); - XP_LOGF( "%s: got key: %s of len %d", __func__, key, len ); - for ( int ii = 0; ii < len; ++ii ) { - json_object* forGame = json_object_array_get_idx(val, ii); - int len2 = json_object_array_length(forGame); - foundAny = foundAny || len2 > 0; - for ( int jj = 0; jj < len2; ++jj ) { - json_object* oneMove = json_object_array_get_idx(forGame, jj); - const char* asStr = json_object_get_string(oneMove); - gsize out_len; - guchar* buf = g_base64_decode( asStr, &out_len ); - XWStreamCtxt* stream = mem_stream_make( MPPARM(params->mpool) - params->vtMgr, params, - CHANNEL_NONE, NULL ); - stream_putBytes( stream, buf, out_len ); - g_free(buf); - - CommsAddrRec addr = {0}; - addr_addType( &addr, COMMS_CONN_RELAY ); - XP_Bool handled = game_receiveMessage( game, stream, &addr ); - XP_LOGF( "%s(): game_receiveMessage() => %d", __func__, handled ); - stream_destroy( stream ); - - foundAny = XP_TRUE; - } - } - } - } - } - } - return foundAny; + RelayConStorage* storage = getStorage( params ); + XP_ASSERT( onMainThread(storage) ); + XP_ASSERT(0); /* FIX ME */ } void @@ -223,7 +145,7 @@ relaycon_init( LaunchParams* params, const RelayConnProcs* procs, storage->proto = XWPDEV_PROTO_VERSION_1; if ( params->useHTTP ) { - scheule_next_check( storage ); + schedule_next_check( storage ); } } @@ -566,7 +488,7 @@ typedef struct _PostArgs { } PostArgs; static gboolean -onGotData(gpointer user_data) +onGotPostData(gpointer user_data) { PostArgs* pa = (PostArgs*)user_data; /* Now pull any data from the reply */ @@ -633,7 +555,7 @@ postThread( void* arg ) XP_LOGF( "%s(): got \"%s\"", __func__, pa->rs.ptr ); // Put the data on the main thread for processing - (void)g_idle_add( onGotData, pa ); + (void)g_idle_add( onGotPostData, pa ); return NULL; } @@ -654,23 +576,72 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) typedef struct _QueryArgs { RelayConStorage* storage; - GSList* ids; + /* GSList* ids; */ ReadState rs; + GHashTable* map; } QueryArgs; +static gboolean +onGotQueryData( gpointer user_data ) +{ + QueryArgs* qa = (QueryArgs*)user_data; + XP_Bool foundAny = false; + json_object* reply = json_tokener_parse( qa->rs.ptr ); + if ( !!reply ) { + CommsAddrRec addr = {0}; + addr_addType( &addr, COMMS_CONN_RELAY ); + + /* Currently there's an array of arrays for each relayID (value) */ + json_object_object_foreach(reply, relayID, arrOfArrOfMoves) { + int len1 = json_object_array_length( arrOfArrOfMoves ); + XP_LOGF( "%s: got key: %s of len %d", __func__, relayID, len1 ); + if ( len1 > 0 ) { + sqlite3_int64 rowid = *(sqlite3_int64*)g_hash_table_lookup( qa->map, relayID ); + for ( int ii = 0; ii < len1; ++ii ) { + json_object* forGameArray = json_object_array_get_idx( arrOfArrOfMoves, ii ); + int len2 = json_object_array_length( forGameArray ); + for ( int jj = 0; jj < len2; ++jj ) { + json_object* oneMove = json_object_array_get_idx( forGameArray, jj ); + const char* asStr = json_object_get_string( oneMove ); + gsize out_len; + guchar* buf = g_base64_decode( asStr, &out_len ); + (*qa->storage->procs.msgForRow)( qa->storage->procsClosure, &addr, + rowid, buf, out_len ); + g_free(buf); + foundAny = XP_TRUE; + } + } + } + } + json_object_put( reply ); + } + + if ( foundAny ) { + /* Reschedule. If we got anything this time, check again sooner! */ + reset_schedule_check_interval( qa->storage ); + } + schedule_next_check( qa->storage ); + + g_hash_table_destroy( qa->map ); + g_free( qa ); + + return FALSE; +} + static void* queryThread( void* arg ) { QueryArgs* qa = (QueryArgs*)arg; - GSList* ids = qa->ids; + GList* ids = g_hash_table_get_keys( qa->map ); qa->rs.ptr = g_malloc0(1); qa->rs.curSize = 1L; json_object* jIds = json_object_new_array(); - for ( GSList* iter = ids; !!iter; iter = iter->next ) { + for ( GList* iter = ids; !!iter; iter = iter->next ) { json_object* idstr = json_object_new_string( iter->data ); json_object_array_add(jIds, idstr); } + g_list_free( ids ); CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); XP_ASSERT(res == CURLE_OK); @@ -699,6 +670,9 @@ queryThread( void* arg ) XP_LOGF( "%s(): got <<%s>>", __func__, qa->rs.ptr ); + /* Put processing back on the main thread */ + g_idle_add( onGotQueryData, qa ); + return NULL; } @@ -713,26 +687,33 @@ checkForMoves( gpointer user_data ) qa->storage = storage; sqlite3* dbp = storage->params->pDb; - GHashTable* map = getRowsToRelayIDsMap( dbp ); - GList* values = g_hash_table_get_values( map ); - for ( GList* iter = values; !!iter; iter = iter->next ) { - gpointer data = iter->data; - XP_LOGF( "checkForMoves: got id: %s", (char*)data ); - qa->ids = g_slist_prepend( qa->ids, g_strdup(data) ); - } - g_list_free( values ); - g_hash_table_destroy( map ); + // qa->map = getRowsToRelayIDsMap( dbp ); + qa->map = getRelayIDsToRowsMap( dbp ); + // qa->ids = g_hash_table_get_values( qa->map ); + /* for ( GList* iter = values; !!iter; iter = iter->next ) { */ + /* gpointer data = iter->data; */ + /* XP_LOGF( "checkForMoves: got id: %s", (char*)data ); */ + /* qa->ids = g_slist_prepend( qa->ids, g_strdup(data) ); */ + /* } */ + /* g_list_free( values ); */ pthread_t thread; (void)pthread_create( &thread, NULL, queryThread, (void*)qa ); pthread_detach( thread ); - scheule_next_check( storage ); + schedule_next_check( storage ); return FALSE; } static void -scheule_next_check( RelayConStorage* storage ) +reset_schedule_check_interval( RelayConStorage* storage ) +{ + XP_ASSERT( onMainThread(storage) ); + storage->nextMoveCheckSecs = 0; +} + +static void +schedule_next_check( RelayConStorage* storage ) { XP_ASSERT( onMainThread(storage) ); diff --git a/xwords4/linux/relaycon.h b/xwords4/linux/relaycon.h index c418b6a5a..fc57e1a79 100644 --- a/xwords4/linux/relaycon.h +++ b/xwords4/linux/relaycon.h @@ -27,6 +27,8 @@ typedef struct _Procs { void (*msgReceived)( void* closure, const CommsAddrRec* from, const XP_U8* buf, XP_U16 len ); + void (*msgForRow)( void* closure, const CommsAddrRec* from, + sqlite3_int64 rowid, const XP_U8* buf, XP_U16 len ); void (*msgNoticeReceived)( void* closure ); void (*devIDReceived)( void* closure, const XP_UCHAR* devID, XP_U16 maxInterval ); @@ -57,5 +59,5 @@ void relaycon_cleanup( LaunchParams* params ); XP_U32 makeClientToken( sqlite3_int64 rowid, XP_U16 seed ); void rowidFromToken( XP_U32 clientToken, sqlite3_int64* rowid, XP_U16* seed ); -XP_Bool checkForMsgs(LaunchParams* params, XWGame* game); +void checkForMsgsNow( LaunchParams* params ); #endif From 523fd26eee3d1e40efb3cf7a9f1156bbe4c639dc Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 22 Oct 2017 09:58:01 -0700 Subject: [PATCH 021/138] make relay hostname configurable --- xwords4/linux/relaycon.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index caf1777e8..d8c20930e 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -30,6 +30,7 @@ #include "gamesdb.h" #define MAX_MOVE_CHECK_SECS ((XP_U16)(60 * 60 * 24)) +#define RELAY_API_PROTO "http" typedef struct _RelayConStorage { pthread_t mainThread; @@ -43,6 +44,7 @@ typedef struct _RelayConStorage { uint32_t nextID; XWPDevProto proto; LaunchParams* params; + XP_UCHAR host[64]; } RelayConStorage; typedef struct _MsgHeader { @@ -140,6 +142,9 @@ relaycon_init( LaunchParams* params, const RelayConnProcs* procs, storage->saddr.sin_addr.s_addr = htonl( hostNameToIP(host) ); storage->saddr.sin_port = htons(port); + XP_ASSERT( XP_STRLEN(host) < VSIZE(storage->host) ); + XP_MEMCPY( storage->host, host, XP_STRLEN(host) + 1 ); + storage->params = params; storage->proto = XWPDEV_PROTO_VERSION_1; @@ -533,14 +538,17 @@ postThread( void* arg ) XP_ASSERT(res == CURLE_OK); CURL* curl = curl_easy_init(); - curl_easy_setopt(curl, CURLOPT_URL, "http://localhost/xw4/relay.py/post"); - curl_easy_setopt(curl, CURLOPT_POST, 1L); + char url[128]; + snprintf( url, sizeof(url), "%s://%s/xw4/relay.py/post", + RELAY_API_PROTO, pa->storage->host ); + curl_easy_setopt( curl, CURLOPT_URL, url ); + curl_easy_setopt( curl, CURLOPT_POST, 1L ); addJsonParams( curl, "params", jobj ); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &pa->rs ); - curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); + curl_easy_setopt( curl, CURLOPT_WRITEDATA, &pa->rs ); + curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); res = curl_easy_perform(curl); XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); @@ -558,7 +566,7 @@ postThread( void* arg ) (void)g_idle_add( onGotPostData, pa ); return NULL; -} +} /* postThread */ static ssize_t post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) @@ -632,6 +640,7 @@ static void* queryThread( void* arg ) { QueryArgs* qa = (QueryArgs*)arg; + XP_ASSERT( !onMainThread(qa->storage) ); GList* ids = g_hash_table_get_keys( qa->map ); qa->rs.ptr = g_malloc0(1); qa->rs.curSize = 1L; @@ -647,8 +656,10 @@ queryThread( void* arg ) XP_ASSERT(res == CURLE_OK); CURL* curl = curl_easy_init(); - curl_easy_setopt(curl, CURLOPT_URL, - "http://localhost/xw4/relay.py/query"); + char url[128]; + snprintf( url, sizeof(url), "%s://%s/xw4/relay.py/query", + RELAY_API_PROTO, qa->storage->host ); + curl_easy_setopt(curl, CURLOPT_URL, url ); curl_easy_setopt(curl, CURLOPT_POST, 1L); addJsonParams( curl, "ids", jIds ); @@ -674,7 +685,7 @@ queryThread( void* arg ) g_idle_add( onGotQueryData, qa ); return NULL; -} +} /* queryThread */ static gboolean checkForMoves( gpointer user_data ) From 16ee3e9439f793d96ed8438e9e5c65f4753b1e6f Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 22 Oct 2017 10:02:39 -0700 Subject: [PATCH 022/138] rename and make self-initing --- xwords4/linux/relaycon.c | 43 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index d8c20930e..cad18fe06 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -76,22 +76,28 @@ static size_t un2vli( int nn, uint8_t* buf ); static bool vli2un( const uint8_t** inp, uint32_t* outp ); -typedef struct _ReadState { +typedef struct _WriteState { gchar* ptr; size_t curSize; -} ReadState; +} WriteState; static size_t write_callback(void *contents, size_t size, size_t nmemb, void* data) { - ReadState* rs = (ReadState*)data; + WriteState* ws = (WriteState*)data; + + if ( !ws->ptr ) { + ws->ptr = g_malloc0(1); + ws->curSize = 1L; + } + XP_LOGF( "%s(size=%ld, nmemb=%ld)", __func__, size, nmemb ); - size_t oldLen = rs->curSize; + size_t oldLen = ws->curSize; const size_t newLength = size * nmemb; XP_ASSERT( (oldLen + newLength) > 0 ); - rs->ptr = g_realloc( rs->ptr, oldLen + newLength ); - memcpy( rs->ptr + oldLen - 1, contents, newLength ); - rs->ptr[oldLen + newLength - 1] = '\0'; + ws->ptr = g_realloc( ws->ptr, oldLen + newLength ); + memcpy( ws->ptr + oldLen - 1, contents, newLength ); + ws->ptr[oldLen + newLength - 1] = '\0'; // XP_LOGF( "%s() => %ld: (passed: \"%s\")", __func__, result, *strp ); return newLength; } @@ -487,7 +493,7 @@ hostNameToIP( const XP_UCHAR* name ) typedef struct _PostArgs { RelayConStorage* storage; - ReadState rs; + WriteState ws; const XP_U8* msgbuf; XP_U16 len; } PostArgs; @@ -498,7 +504,7 @@ onGotPostData(gpointer user_data) PostArgs* pa = (PostArgs*)user_data; /* Now pull any data from the reply */ // got "{"status": "ok", "dataLen": 14, "data": "AYQDiDAyMUEzQ0MyADw=", "err": "none"}" - json_object* reply = json_tokener_parse( pa->rs.ptr ); + json_object* reply = json_tokener_parse( pa->ws.ptr ); json_object* replyData; if ( json_object_object_get_ex( reply, "data", &replyData ) && !!replyData ) { int len = json_object_array_length(replyData); @@ -514,7 +520,7 @@ onGotPostData(gpointer user_data) } (void)json_object_put( reply ); - g_free( pa->rs.ptr ); + g_free( pa->ws.ptr ); g_free( pa ); return FALSE; @@ -531,9 +537,6 @@ postThread( void* arg ) g_free( data ); json_object_object_add( jobj, "data", jstr ); - pa->rs.ptr = g_malloc0(1); - pa->rs.curSize = 1L; - CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); XP_ASSERT(res == CURLE_OK); CURL* curl = curl_easy_init(); @@ -547,7 +550,7 @@ postThread( void* arg ) addJsonParams( curl, "params", jobj ); curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt( curl, CURLOPT_WRITEDATA, &pa->rs ); + curl_easy_setopt( curl, CURLOPT_WRITEDATA, &pa->ws ); curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); res = curl_easy_perform(curl); @@ -560,7 +563,7 @@ postThread( void* arg ) curl_easy_cleanup(curl); curl_global_cleanup(); - XP_LOGF( "%s(): got \"%s\"", __func__, pa->rs.ptr ); + XP_LOGF( "%s(): got \"%s\"", __func__, pa->ws.ptr ); // Put the data on the main thread for processing (void)g_idle_add( onGotPostData, pa ); @@ -585,7 +588,7 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) typedef struct _QueryArgs { RelayConStorage* storage; /* GSList* ids; */ - ReadState rs; + WriteState ws; GHashTable* map; } QueryArgs; @@ -594,7 +597,7 @@ onGotQueryData( gpointer user_data ) { QueryArgs* qa = (QueryArgs*)user_data; XP_Bool foundAny = false; - json_object* reply = json_tokener_parse( qa->rs.ptr ); + json_object* reply = json_tokener_parse( qa->ws.ptr ); if ( !!reply ) { CommsAddrRec addr = {0}; addr_addType( &addr, COMMS_CONN_RELAY ); @@ -642,8 +645,6 @@ queryThread( void* arg ) QueryArgs* qa = (QueryArgs*)arg; XP_ASSERT( !onMainThread(qa->storage) ); GList* ids = g_hash_table_get_keys( qa->map ); - qa->rs.ptr = g_malloc0(1); - qa->rs.curSize = 1L; json_object* jIds = json_object_new_array(); for ( GList* iter = ids; !!iter; iter = iter->next ) { @@ -665,7 +666,7 @@ queryThread( void* arg ) addJsonParams( curl, "ids", jIds ); curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt( curl, CURLOPT_WRITEDATA, &qa->rs ); + curl_easy_setopt( curl, CURLOPT_WRITEDATA, &qa->ws ); curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); res = curl_easy_perform( curl ); @@ -679,7 +680,7 @@ queryThread( void* arg ) curl_easy_cleanup(curl); curl_global_cleanup(); - XP_LOGF( "%s(): got <<%s>>", __func__, qa->rs.ptr ); + XP_LOGF( "%s(): got <<%s>>", __func__, qa->ws.ptr ); /* Put processing back on the main thread */ g_idle_add( onGotQueryData, qa ); From 4c15723f903ae6edf288a70ebb55079e8fa2f798 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 22 Oct 2017 11:43:56 -0700 Subject: [PATCH 023/138] add http option to test script --- xwords4/linux/scripts/discon_ok2.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index f4b1d5042..8efe77d4d 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -31,6 +31,7 @@ NAMES=(UNUSED Brynn Ariela Kati Eric) SEND_CHAT='' CORE_COUNT=$(ls core.* 2>/dev/null | wc -l) DUP_PACKETS='' +HTTP_PCT=0 declare -A PIDS declare -A APPS @@ -220,6 +221,9 @@ build_cmds() { PARAMS="$PARAMS --slow-robot 1:3 --skip-confirm" PARAMS="$PARAMS --db $FILE" PARAMS="$PARAMS --drop-nth-packet $DROP_N $PLAT_PARMS" + if [ $((${RANDOM}%100)) -lt $HTTP_PCT ]; then + PARAMS="$PARAMS --use-http" + fi # PARAMS="$PARAMS --split-packets 2" if [ -n "$SEND_CHAT" ]; then PARAMS="$PARAMS --send-chat $SEND_CHAT" @@ -615,6 +619,7 @@ function usage() { echo " [--udp-incr ] \\" >&2 echo " [--udp-start ] # default: $UDP_PCT_START \\" >&2 echo " [--undo-pct ] \\" >&2 + echo " [--http-pct <0 <= n <=100>] \\" >&2 exit 1 } @@ -691,6 +696,11 @@ while [ "$#" -gt 0 ]; do UNDO_PCT=$(getArg $*) shift ;; + --http-pct) + HTTP_PCT=$(getArg $*) + [ $HTTP_PCT -ge 0 -a $HTTP_PCT -le 100 ] || usage "n must be 0 <= n <= 100" + shift + ;; --send-chat) SEND_CHAT=$(getArg $*) shift From 4583b5d88d9836b827971e50c2af4ec192c73943 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 22 Oct 2017 15:47:01 -0700 Subject: [PATCH 024/138] use ! with struct instead of ntohs --- xwords4/android/scripts/relay.py | 37 ++++++++------------------------ 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index b1a18ef8f..5cfa04144 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -36,7 +36,7 @@ def post(req, params, timeoutSecs = 1): return json.dumps(jobj) def query(req, ids): - print(ids) + print('ids', ids) ids = json.loads(ids) idsLen = 0 @@ -47,29 +47,25 @@ def query(req, ids): sock.connect(('127.0.0.1', 10998)) lenShort = 2 + idsLen + len(ids) + 1 - sock.send(struct.pack("hBBh", socket.htons(lenShort), - PROTOCOL_VERSION, PRX_GET_MSGS, - socket.htons(len(ids)))) + print(lenShort, PROTOCOL_VERSION, PRX_GET_MSGS, len(ids)) + header = struct.Struct("!hBBh") + assert header.size == 6 + sock.send(header.pack(lenShort, PROTOCOL_VERSION, PRX_GET_MSGS, len(ids))) for id in ids: sock.send(id + '\n') - unpacker = struct.Struct('2H') # 2s f') - data = sock.recv(unpacker.size) - resLen, nameCount = unpacker.unpack(data) - resLen = socket.ntohs(resLen) - nameCount = socket.ntohs(nameCount) + unpacker = struct.Struct('!2H') # 2s f') + resLen, nameCount = unpacker.unpack(sock.recv(unpacker.size)) # problem when ids empty print('resLen:', resLen, 'nameCount:', nameCount) msgsLists = {} if nameCount == len(ids): for ii in range(nameCount): perGame = [] - shortUnpacker = struct.Struct('H') - countsThisGame, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) - countsThisGame = socket.ntohs(countsThisGame) + shortUnpacker = struct.Struct('!H') + countsThisGame, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) # problem print('countsThisGame:', countsThisGame) for jj in range(countsThisGame): msgLen, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) - msgLen = socket.ntohs(msgLen) print('msgLen:', msgLen) msgs = [] if msgLen > 0: @@ -82,21 +78,6 @@ def query(req, ids): return json.dumps(msgsLists) - # received = sock.recv(1024*4) - # print('len:', len(received)) - # short resLen = dis.readShort(); // total message length - # short nameCount = dis.readShort(); - - -def dosend(sock, bytes): - totalsent = 0 - while totalsent < len(bytes): - sent = sock.send(bytes[totalsent:]) - if sent == 0: - raise RuntimeError("socket connection broken") - totalsent = totalsent + sent - - def main(): print(query(None, json.dumps(sys.argv[1:]))) # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } From 6787fe1406d41f97be6f1cfbfd529951e86fadb7 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 22 Oct 2017 21:29:16 -0700 Subject: [PATCH 025/138] fix length byte --- xwords4/android/scripts/relay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 5cfa04144..c8e6d3ef9 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -46,9 +46,9 @@ def query(req, ids): sock.settimeout(5) # seconds sock.connect(('127.0.0.1', 10998)) - lenShort = 2 + idsLen + len(ids) + 1 + lenShort = 2 + idsLen + len(ids) + 2 print(lenShort, PROTOCOL_VERSION, PRX_GET_MSGS, len(ids)) - header = struct.Struct("!hBBh") + header = struct.Struct('!hBBh') assert header.size == 6 sock.send(header.pack(lenShort, PROTOCOL_VERSION, PRX_GET_MSGS, len(ids))) From c3887b9c77a8168ea0c4720aa3cf909842259f02 Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 23 Oct 2017 21:07:05 -0700 Subject: [PATCH 026/138] use a single thread and a protected queue I don't want race conditions between threads talking to the server. --- xwords4/linux/relaycon.c | 344 +++++++++++++++++++++++---------------- 1 file changed, 207 insertions(+), 137 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index cad18fe06..bfb28af9c 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -36,6 +36,9 @@ typedef struct _RelayConStorage { pthread_t mainThread; guint moveCheckerID; XP_U16 nextMoveCheckSecs; + pthread_cond_t relayCondVar; + pthread_mutex_t relayMutex; + GSList* relayTaskList; int socket; RelayConnProcs procs; @@ -75,12 +78,36 @@ static size_t writeVLI( XP_U8* out, uint32_t nn ); static size_t un2vli( int nn, uint8_t* buf ); static bool vli2un( const uint8_t** inp, uint32_t* outp ); +static void* relayThread( void* arg ); typedef struct _WriteState { gchar* ptr; size_t curSize; } WriteState; +typedef enum { POST, QUERY, } TaskType; + +typedef struct _RelayTask { + TaskType typ; + RelayConStorage* storage; + WriteState ws; + union { + struct { + XP_U8* msgbuf; + XP_U16 len; + } post; + struct { + GHashTable* map; + } query; + } u; +} RelayTask; + +static RelayTask* makeRelayTask(RelayConStorage* storage, TaskType typ); +static void freeRelayTask(RelayTask* task); +static void handlePost( RelayTask* task ); +static void handleQuery( RelayTask* task ); + + static size_t write_callback(void *contents, size_t size, size_t nmemb, void* data) { @@ -138,19 +165,26 @@ relaycon_init( LaunchParams* params, const RelayConnProcs* procs, XP_MEMCPY( &storage->procs, procs, sizeof(storage->procs) ); storage->procsClosure = procsClosure; - storage->mainThread = pthread_self(); + if ( params->useHTTP ) { + storage->mainThread = pthread_self(); + pthread_mutex_init ( &storage->relayMutex, NULL ); + pthread_cond_init( &storage->relayCondVar, NULL ); + pthread_t thread; + (void)pthread_create( &thread, NULL, relayThread, storage ); + pthread_detach( thread ); - storage->socket = socket( AF_INET, SOCK_DGRAM, IPPROTO_UDP ); - (*procs->socketAdded)( storage, storage->socket, relaycon_receive ); + XP_ASSERT( XP_STRLEN(host) < VSIZE(storage->host) ); + XP_MEMCPY( storage->host, host, XP_STRLEN(host) + 1 ); + } else { + storage->socket = socket( AF_INET, SOCK_DGRAM, IPPROTO_UDP ); + (*procs->socketAdded)( storage, storage->socket, relaycon_receive ); - XP_MEMSET( &storage->saddr, 0, sizeof(storage->saddr) ); - storage->saddr.sin_family = PF_INET; - storage->saddr.sin_addr.s_addr = htonl( hostNameToIP(host) ); - storage->saddr.sin_port = htons(port); - - XP_ASSERT( XP_STRLEN(host) < VSIZE(storage->host) ); - XP_MEMCPY( storage->host, host, XP_STRLEN(host) + 1 ); + XP_MEMSET( &storage->saddr, 0, sizeof(storage->saddr) ); + storage->saddr.sin_family = PF_INET; + storage->saddr.sin_addr.s_addr = htonl( hostNameToIP(host) ); + storage->saddr.sin_port = htons(port); + } storage->params = params; storage->proto = XWPDEV_PROTO_VERSION_1; @@ -311,6 +345,59 @@ onMainThread( RelayConStorage* storage ) return storage->mainThread = pthread_self(); } +static void* +relayThread( void* arg ) +{ + RelayConStorage* storage = (RelayConStorage*)arg; + for ( ; ; ) { + pthread_mutex_lock( &storage->relayMutex ); + while ( !storage->relayTaskList ) { + pthread_cond_wait( &storage->relayCondVar, &storage->relayMutex ); + } + + RelayTask* task = storage->relayTaskList->data; + storage->relayTaskList = storage->relayTaskList->next; + pthread_mutex_unlock( &storage->relayMutex ); + + switch ( task->typ ) { + case POST: + handlePost( task ); + break; + case QUERY: + handleQuery( task ); + break; + default: + XP_ASSERT(0); + } + } + return NULL; +} + +static void +addTask( RelayConStorage* storage, RelayTask* task ) +{ + pthread_mutex_lock( &storage->relayMutex ); + storage->relayTaskList = g_slist_append( storage->relayTaskList, task ); + pthread_cond_signal( &storage->relayCondVar ); + pthread_mutex_unlock( &storage->relayMutex ); +} + +static RelayTask* +makeRelayTask( RelayConStorage* storage, TaskType typ ) +{ + RelayTask* task = (RelayTask*)g_malloc0(sizeof(*task)); + task->typ = typ; + task->storage = storage; + return task; +} + +static void +freeRelayTask( RelayTask* task ) +{ + g_free( task->ws.ptr ); + g_free( task ); +} + static void sendAckIf( RelayConStorage* storage, const MsgHeader* header ) { @@ -331,6 +418,7 @@ process( RelayConStorage* storage, XP_U8* buf, ssize_t nRead ) MsgHeader header; if ( readHeader( &ptr, &header ) ) { sendAckIf( storage, &header ); + switch( header.cmd ) { case XWPDEV_REGRSP: { uint32_t len; @@ -433,6 +521,7 @@ relaycon_receive( GIOChannel* source, GIOCondition XP_UNUSED_DBG(condition), gpo { XP_ASSERT( 0 != (G_IO_IN & condition) ); /* FIX ME */ RelayConStorage* storage = (RelayConStorage*)data; + XP_ASSERT( !storage->params->useHTTP ); XP_U8 buf[512]; struct sockaddr_in from; socklen_t fromlen = sizeof(from); @@ -494,44 +583,47 @@ hostNameToIP( const XP_UCHAR* name ) typedef struct _PostArgs { RelayConStorage* storage; WriteState ws; - const XP_U8* msgbuf; + XP_U8* msgbuf; XP_U16 len; } PostArgs; static gboolean onGotPostData(gpointer user_data) { - PostArgs* pa = (PostArgs*)user_data; + RelayTask* task = (RelayTask*)user_data; /* Now pull any data from the reply */ // got "{"status": "ok", "dataLen": 14, "data": "AYQDiDAyMUEzQ0MyADw=", "err": "none"}" - json_object* reply = json_tokener_parse( pa->ws.ptr ); - json_object* replyData; - if ( json_object_object_get_ex( reply, "data", &replyData ) && !!replyData ) { - int len = json_object_array_length(replyData); - for ( int ii = 0; ii < len; ++ii ) { - json_object* datum = json_object_array_get_idx( replyData, ii ); - const char* str = json_object_get_string( datum ); - gsize out_len; - guchar* buf = g_base64_decode( (const gchar*)str, &out_len ); - process( pa->storage, buf, out_len ); - g_free( buf ); + if ( !!task->ws.ptr ) { + json_object* reply = json_tokener_parse( task->ws.ptr ); + json_object* replyData; + if ( json_object_object_get_ex( reply, "data", &replyData ) && !!replyData ) { + int len = json_object_array_length(replyData); + for ( int ii = 0; ii < len; ++ii ) { + json_object* datum = json_object_array_get_idx( replyData, ii ); + const char* str = json_object_get_string( datum ); + gsize out_len; + guchar* buf = g_base64_decode( (const gchar*)str, &out_len ); + process( task->storage, buf, out_len ); + g_free( buf ); + } + (void)json_object_put( replyData ); } - (void)json_object_put( replyData ); + (void)json_object_put( reply ); } - (void)json_object_put( reply ); - g_free( pa->ws.ptr ); - g_free( pa ); + g_free( task->u.post.msgbuf ); + + freeRelayTask( task ); return FALSE; } -static void* -postThread( void* arg ) +static void +handlePost( RelayTask* task ) { - PostArgs* pa = (PostArgs*)arg; - XP_ASSERT( !onMainThread(pa->storage) ); - char* data = g_base64_encode( pa->msgbuf, pa->len ); + XP_LOGF( "%s(len=%d)", __func__, task->u.post.len ); + XP_ASSERT( !onMainThread(task->storage) ); + char* data = g_base64_encode( task->u.post.msgbuf, task->u.post.len ); struct json_object* jobj = json_object_new_object(); struct json_object* jstr = json_object_new_string(data); g_free( data ); @@ -543,14 +635,14 @@ postThread( void* arg ) char url[128]; snprintf( url, sizeof(url), "%s://%s/xw4/relay.py/post", - RELAY_API_PROTO, pa->storage->host ); + RELAY_API_PROTO, task->storage->host ); curl_easy_setopt( curl, CURLOPT_URL, url ); curl_easy_setopt( curl, CURLOPT_POST, 1L ); addJsonParams( curl, "params", jobj ); curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt( curl, CURLOPT_WRITEDATA, &pa->ws ); + curl_easy_setopt( curl, CURLOPT_WRITEDATA, &task->ws ); curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); res = curl_easy_perform(curl); @@ -563,130 +655,121 @@ postThread( void* arg ) curl_easy_cleanup(curl); curl_global_cleanup(); - XP_LOGF( "%s(): got \"%s\"", __func__, pa->ws.ptr ); + XP_LOGF( "%s(): got \"%s\"", __func__, task->ws.ptr ); // Put the data on the main thread for processing - (void)g_idle_add( onGotPostData, pa ); - - return NULL; -} /* postThread */ + (void)g_idle_add( onGotPostData, task ); +} /* handlePost */ static ssize_t post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) { - PostArgs* pa = (PostArgs*)g_malloc0(sizeof(*pa)); - pa->storage = storage; - pa->msgbuf = msgbuf; - pa->len = len; - - pthread_t thread; - (void)pthread_create( &thread, NULL, postThread, (void*)pa ); - pthread_detach( thread ); + XP_LOGF( "%s(len=%d)", __func__, len ); + RelayTask* task = makeRelayTask( storage, POST ); + task->u.post.msgbuf = g_malloc(len); + XP_MEMCPY( task->u.post.msgbuf, msgbuf, len ); + task->u.post.len = len; + addTask( storage, task ); return len; } -typedef struct _QueryArgs { - RelayConStorage* storage; - /* GSList* ids; */ - WriteState ws; - GHashTable* map; -} QueryArgs; - static gboolean onGotQueryData( gpointer user_data ) { - QueryArgs* qa = (QueryArgs*)user_data; + RelayTask* task = (RelayTask*)user_data; + RelayConStorage* storage = task->storage; XP_Bool foundAny = false; - json_object* reply = json_tokener_parse( qa->ws.ptr ); - if ( !!reply ) { - CommsAddrRec addr = {0}; - addr_addType( &addr, COMMS_CONN_RELAY ); + if ( !!task->ws.ptr ) { + json_object* reply = json_tokener_parse( task->ws.ptr ); + if ( !!reply ) { + CommsAddrRec addr = {0}; + addr_addType( &addr, COMMS_CONN_RELAY ); - /* Currently there's an array of arrays for each relayID (value) */ - json_object_object_foreach(reply, relayID, arrOfArrOfMoves) { - int len1 = json_object_array_length( arrOfArrOfMoves ); - XP_LOGF( "%s: got key: %s of len %d", __func__, relayID, len1 ); - if ( len1 > 0 ) { - sqlite3_int64 rowid = *(sqlite3_int64*)g_hash_table_lookup( qa->map, relayID ); - for ( int ii = 0; ii < len1; ++ii ) { - json_object* forGameArray = json_object_array_get_idx( arrOfArrOfMoves, ii ); - int len2 = json_object_array_length( forGameArray ); - for ( int jj = 0; jj < len2; ++jj ) { - json_object* oneMove = json_object_array_get_idx( forGameArray, jj ); - const char* asStr = json_object_get_string( oneMove ); - gsize out_len; - guchar* buf = g_base64_decode( asStr, &out_len ); - (*qa->storage->procs.msgForRow)( qa->storage->procsClosure, &addr, + /* Currently there's an array of arrays for each relayID (value) */ + json_object_object_foreach(reply, relayID, arrOfArrOfMoves) { + int len1 = json_object_array_length( arrOfArrOfMoves ); + XP_LOGF( "%s: got key: %s of len %d", __func__, relayID, len1 ); + if ( len1 > 0 ) { + sqlite3_int64 rowid = *(sqlite3_int64*)g_hash_table_lookup( task->u.query.map, relayID ); + for ( int ii = 0; ii < len1; ++ii ) { + json_object* forGameArray = json_object_array_get_idx( arrOfArrOfMoves, ii ); + int len2 = json_object_array_length( forGameArray ); + for ( int jj = 0; jj < len2; ++jj ) { + json_object* oneMove = json_object_array_get_idx( forGameArray, jj ); + const char* asStr = json_object_get_string( oneMove ); + gsize out_len; + guchar* buf = g_base64_decode( asStr, &out_len ); + (*storage->procs.msgForRow)( storage->procsClosure, &addr, rowid, buf, out_len ); - g_free(buf); - foundAny = XP_TRUE; + g_free(buf); + foundAny = XP_TRUE; + } } } } + json_object_put( reply ); } - json_object_put( reply ); } if ( foundAny ) { /* Reschedule. If we got anything this time, check again sooner! */ - reset_schedule_check_interval( qa->storage ); + reset_schedule_check_interval( storage ); } - schedule_next_check( qa->storage ); + schedule_next_check( storage ); - g_hash_table_destroy( qa->map ); - g_free( qa ); + g_hash_table_destroy( task->u.query.map ); + freeRelayTask(task); return FALSE; } -static void* -queryThread( void* arg ) +static void +handleQuery( RelayTask* task ) { - QueryArgs* qa = (QueryArgs*)arg; - XP_ASSERT( !onMainThread(qa->storage) ); - GList* ids = g_hash_table_get_keys( qa->map ); + XP_ASSERT( !onMainThread(task->storage) ); - json_object* jIds = json_object_new_array(); - for ( GList* iter = ids; !!iter; iter = iter->next ) { - json_object* idstr = json_object_new_string( iter->data ); - json_object_array_add(jIds, idstr); - } - g_list_free( ids ); + if ( g_hash_table_size( task->u.query.map ) > 0 ) { + GList* ids = g_hash_table_get_keys( task->u.query.map ); - CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); - XP_ASSERT(res == CURLE_OK); - CURL* curl = curl_easy_init(); + json_object* jIds = json_object_new_array(); + for ( GList* iter = ids; !!iter; iter = iter->next ) { + json_object* idstr = json_object_new_string( iter->data ); + json_object_array_add(jIds, idstr); + } + g_list_free( ids ); - char url[128]; - snprintf( url, sizeof(url), "%s://%s/xw4/relay.py/query", - RELAY_API_PROTO, qa->storage->host ); - curl_easy_setopt(curl, CURLOPT_URL, url ); - curl_easy_setopt(curl, CURLOPT_POST, 1L); + CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); + XP_ASSERT(res == CURLE_OK); + CURL* curl = curl_easy_init(); - addJsonParams( curl, "ids", jIds ); + char url[128]; + snprintf( url, sizeof(url), "%s://%s/xw4/relay.py/query", + RELAY_API_PROTO, task->storage->host ); + curl_easy_setopt(curl, CURLOPT_URL, url ); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + + addJsonParams( curl, "ids", jIds ); - curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt( curl, CURLOPT_WRITEDATA, &qa->ws ); - curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); + curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); + curl_easy_setopt( curl, CURLOPT_WRITEDATA, &task->ws ); + curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); - res = curl_easy_perform( curl ); + res = curl_easy_perform( curl ); + + /* Check for errors */ + if (res != CURLE_OK) { + XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); + } + /* always cleanup */ + curl_easy_cleanup(curl); + curl_global_cleanup(); + + XP_LOGF( "%s(): got <<%s>>", __func__, task->ws.ptr ); - XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); - /* Check for errors */ - if (res != CURLE_OK) { - XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); } - /* always cleanup */ - curl_easy_cleanup(curl); - curl_global_cleanup(); - - XP_LOGF( "%s(): got <<%s>>", __func__, qa->ws.ptr ); - /* Put processing back on the main thread */ - g_idle_add( onGotQueryData, qa ); - - return NULL; -} /* queryThread */ + g_idle_add( onGotQueryData, task ); +} /* handleQuery */ static gboolean checkForMoves( gpointer user_data ) @@ -695,23 +778,10 @@ checkForMoves( gpointer user_data ) RelayConStorage* storage = (RelayConStorage*)user_data; XP_ASSERT( onMainThread(storage) ); - QueryArgs* qa = (QueryArgs*)g_malloc0(sizeof(*qa)); - qa->storage = storage; - + RelayTask* task = makeRelayTask( storage, QUERY ); sqlite3* dbp = storage->params->pDb; - // qa->map = getRowsToRelayIDsMap( dbp ); - qa->map = getRelayIDsToRowsMap( dbp ); - // qa->ids = g_hash_table_get_values( qa->map ); - /* for ( GList* iter = values; !!iter; iter = iter->next ) { */ - /* gpointer data = iter->data; */ - /* XP_LOGF( "checkForMoves: got id: %s", (char*)data ); */ - /* qa->ids = g_slist_prepend( qa->ids, g_strdup(data) ); */ - /* } */ - /* g_list_free( values ); */ - - pthread_t thread; - (void)pthread_create( &thread, NULL, queryThread, (void*)qa ); - pthread_detach( thread ); + task->u.query.map = getRelayIDsToRowsMap( dbp ); + addTask( storage, task ); schedule_next_check( storage ); return FALSE; @@ -754,8 +824,8 @@ sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) nSent = post( storage, msgbuf, len ); } else { nSent = sendto( storage->socket, msgbuf, len, 0, /* flags */ - (struct sockaddr*)&storage->saddr, - sizeof(storage->saddr) ); + (struct sockaddr*)&storage->saddr, + sizeof(storage->saddr) ); #ifdef COMMS_CHECKSUM gchar* sum = g_compute_checksum_for_data( G_CHECKSUM_MD5, msgbuf, len ); XP_LOGF( "%s: sent %d bytes with sum %s", __func__, len, sum ); From 2cfac68f91e7acc779f2a3bafde96a8455232032 Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 23 Oct 2017 21:08:14 -0700 Subject: [PATCH 027/138] leave out entries without relayIDs SQL is the easiest place to filter --- xwords4/linux/gamesdb.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/xwords4/linux/gamesdb.c b/xwords4/linux/gamesdb.c index 7d8f1e6d9..fd968a22b 100644 --- a/xwords4/linux/gamesdb.c +++ b/xwords4/linux/gamesdb.c @@ -316,11 +316,10 @@ getRelayIDsToRowsMap( sqlite3* pDb ) { GHashTable* table = g_hash_table_new( g_str_hash, g_str_equal ); sqlite3_stmt *ppStmt; - int result = sqlite3_prepare_v2( pDb, "SELECT relayid, rowid FROM games", - -1, &ppStmt, NULL ); + int result = sqlite3_prepare_v2( pDb, "SELECT relayid, rowid FROM games " + "where NOT relayid = ''", -1, &ppStmt, NULL ); assertPrintResult( pDb, result, SQLITE_OK ); - XP_USE( result ); - while ( NULL != ppStmt ) { + while ( result == SQLITE_OK && NULL != ppStmt ) { switch( sqlite3_step( ppStmt ) ) { case SQLITE_ROW: /* have data */ { From 28bec6d456cf70622b2511c98c0ab2a56a2805e5 Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 24 Oct 2017 06:20:08 -0700 Subject: [PATCH 028/138] implement checkForMoves menu again --- xwords4/linux/relaycon.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index bfb28af9c..3e1b407bb 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -62,6 +62,8 @@ static gboolean relaycon_receive( GIOChannel *source, GIOCondition condition, gpointer data ); static void schedule_next_check( RelayConStorage* storage ); static void reset_schedule_check_interval( RelayConStorage* storage ); +static void checkForMovesOnce( RelayConStorage* storage ); + static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ); static size_t addVLIStr( XP_U8* buf, size_t len, const XP_UCHAR* str ); static void getNetString( const XP_U8** ptr, XP_U16 len, XP_UCHAR* buf ); @@ -153,7 +155,7 @@ checkForMsgsNow( LaunchParams* params ) { RelayConStorage* storage = getStorage( params ); XP_ASSERT( onMainThread(storage) ); - XP_ASSERT(0); /* FIX ME */ + checkForMovesOnce( storage ); } void @@ -688,7 +690,6 @@ onGotQueryData( gpointer user_data ) /* Currently there's an array of arrays for each relayID (value) */ json_object_object_foreach(reply, relayID, arrOfArrOfMoves) { int len1 = json_object_array_length( arrOfArrOfMoves ); - XP_LOGF( "%s: got key: %s of len %d", __func__, relayID, len1 ); if ( len1 > 0 ) { sqlite3_int64 rowid = *(sqlite3_int64*)g_hash_table_lookup( task->u.query.map, relayID ); for ( int ii = 0; ii < len1; ++ii ) { @@ -771,18 +772,23 @@ handleQuery( RelayTask* task ) g_idle_add( onGotQueryData, task ); } /* handleQuery */ -static gboolean -checkForMoves( gpointer user_data ) +static void +checkForMovesOnce( RelayConStorage* storage ) { LOG_FUNC(); - RelayConStorage* storage = (RelayConStorage*)user_data; XP_ASSERT( onMainThread(storage) ); RelayTask* task = makeRelayTask( storage, QUERY ); sqlite3* dbp = storage->params->pDb; task->u.query.map = getRelayIDsToRowsMap( dbp ); addTask( storage, task ); +} +static gboolean +checkForMoves( gpointer user_data ) +{ + RelayConStorage* storage = (RelayConStorage*)user_data; + checkForMovesOnce( storage ); schedule_next_check( storage ); return FALSE; } From 342d229be1acc877c8952e361b22755651630f39 Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 24 Oct 2017 07:07:22 -0700 Subject: [PATCH 029/138] refactor --- xwords4/linux/relaycon.c | 92 ++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 55 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 3e1b407bb..860fecc45 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -139,7 +139,6 @@ addJsonParams( CURL* curl, const char* name, json_object* param ) char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); gchar* buf = g_strdup_printf( "%s=%s", name, curl_params ); - XP_LOGF( "%s: added param: %s", __func__, buf ); curl_free( curl_params ); curl_easy_setopt( curl, CURLOPT_POSTFIELDS, buf ); @@ -150,6 +149,41 @@ addJsonParams( CURL* curl, const char* name, json_object* param ) json_object_put( param ); } +static XP_Bool +runWitCurl( RelayTask* task, const gchar* proc, const gchar* key, + json_object* jVal ) +{ + CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); + XP_ASSERT(res == CURLE_OK); + CURL* curl = curl_easy_init(); + + char url[128]; + snprintf( url, sizeof(url), "%s://%s/xw4/relay.py/%s", + RELAY_API_PROTO, task->storage->host, proc ); + curl_easy_setopt( curl, CURLOPT_URL, url ); + curl_easy_setopt( curl, CURLOPT_POST, 1L ); + + addJsonParams( curl, key, jVal ); + + curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); + curl_easy_setopt( curl, CURLOPT_WRITEDATA, &task->ws ); + // curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); + + res = curl_easy_perform(curl); + XP_Bool success = res == CURLE_OK; + XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); + /* Check for errors */ + if ( ! success ) { + XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); + } else { + XP_LOGF( "%s(): got for %s: \"%s\"", __func__, proc, task->ws.ptr ); + } + /* always cleanup */ + curl_easy_cleanup(curl); + curl_global_cleanup(); + return success; +} + void checkForMsgsNow( LaunchParams* params ) { @@ -631,33 +665,8 @@ handlePost( RelayTask* task ) g_free( data ); json_object_object_add( jobj, "data", jstr ); - CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); - XP_ASSERT(res == CURLE_OK); - CURL* curl = curl_easy_init(); + runWitCurl( task, "post", "params", jobj ); - char url[128]; - snprintf( url, sizeof(url), "%s://%s/xw4/relay.py/post", - RELAY_API_PROTO, task->storage->host ); - curl_easy_setopt( curl, CURLOPT_URL, url ); - curl_easy_setopt( curl, CURLOPT_POST, 1L ); - - addJsonParams( curl, "params", jobj ); - - curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt( curl, CURLOPT_WRITEDATA, &task->ws ); - curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); - - res = curl_easy_perform(curl); - XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); - /* Check for errors */ - if (res != CURLE_OK) { - XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); - } - /* always cleanup */ - curl_easy_cleanup(curl); - curl_global_cleanup(); - - XP_LOGF( "%s(): got \"%s\"", __func__, task->ws.ptr ); // Put the data on the main thread for processing (void)g_idle_add( onGotPostData, task ); @@ -739,34 +748,7 @@ handleQuery( RelayTask* task ) } g_list_free( ids ); - CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); - XP_ASSERT(res == CURLE_OK); - CURL* curl = curl_easy_init(); - - char url[128]; - snprintf( url, sizeof(url), "%s://%s/xw4/relay.py/query", - RELAY_API_PROTO, task->storage->host ); - curl_easy_setopt(curl, CURLOPT_URL, url ); - curl_easy_setopt(curl, CURLOPT_POST, 1L); - - addJsonParams( curl, "ids", jIds ); - - curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); - curl_easy_setopt( curl, CURLOPT_WRITEDATA, &task->ws ); - curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); - - res = curl_easy_perform( curl ); - - /* Check for errors */ - if (res != CURLE_OK) { - XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); - } - /* always cleanup */ - curl_easy_cleanup(curl); - curl_global_cleanup(); - - XP_LOGF( "%s(): got <<%s>>", __func__, task->ws.ptr ); - + runWitCurl( task, "query", "ids", jIds ); } /* Put processing back on the main thread */ g_idle_add( onGotQueryData, task ); From 6ff13e6e23d4b0d291e5c338a97eea156fdaba90 Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 24 Oct 2017 19:14:23 -0700 Subject: [PATCH 030/138] move 'check moves' menu to main window --- xwords4/linux/gtkboard.c | 15 +++------------ xwords4/linux/gtkboard.h | 4 ++++ xwords4/linux/gtkmain.c | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/xwords4/linux/gtkboard.c b/xwords4/linux/gtkboard.c index eb6c9b573..52eb7c820 100644 --- a/xwords4/linux/gtkboard.c +++ b/xwords4/linux/gtkboard.c @@ -1178,11 +1178,6 @@ handle_memstats( GtkWidget* XP_UNUSED(widget), GtkGameGlobals* globals ) } /* handle_memstats */ -static void -handle_movescheck( GtkWidget* XP_UNUSED(widget), GtkGameGlobals* globals ) -{ - checkForMsgsNow( globals->cGlobals.params ); -} #endif #ifdef XWFEATURE_ACTIVERECT @@ -1207,15 +1202,15 @@ frame_active( GtkWidget* XP_UNUSED(widget), GtkGameGlobals* globals ) } #endif -static GtkWidget* +GtkWidget* createAddItem( GtkWidget* parent, gchar* label, - GCallback handlerFunc, GtkGameGlobals* globals ) + GCallback handlerFunc, gpointer closure ) { GtkWidget* item = gtk_menu_item_new_with_label( label ); if ( handlerFunc != NULL ) { g_signal_connect( item, "activate", G_CALLBACK(handlerFunc), - globals ); + closure ); } gtk_menu_shell_append( GTK_MENU_SHELL(parent), item ); @@ -1289,10 +1284,6 @@ makeMenus( GtkGameGlobals* globals ) #ifdef MEM_DEBUG (void)createAddItem( fileMenu, "Mem stats", (GCallback)handle_memstats, globals ); - if ( globals->cGlobals.params->useHTTP ) { - (void)createAddItem( fileMenu, "Check for moves", - (GCallback)handle_movescheck, globals ); - } #endif #ifdef XWFEATURE_ACTIVERECT diff --git a/xwords4/linux/gtkboard.h b/xwords4/linux/gtkboard.h index d46f26ba6..491be224f 100644 --- a/xwords4/linux/gtkboard.h +++ b/xwords4/linux/gtkboard.h @@ -187,6 +187,10 @@ XP_Bool loadGameNoDraw( GtkGameGlobals* globals, LaunchParams* params, sqlite3_int64 rowid ); void destroy_board_window( GtkWidget* widget, GtkGameGlobals* globals ); +GtkWidget* makeAddSubmenu( GtkWidget* menubar, gchar* label ); +GtkWidget* createAddItem( GtkWidget* parent, gchar* label, + GCallback handlerFunc, gpointer closure ); + #endif /* PLATFORM_GTK */ #endif diff --git a/xwords4/linux/gtkmain.c b/xwords4/linux/gtkmain.c index ad3119513..51da3f2ce 100644 --- a/xwords4/linux/gtkmain.c +++ b/xwords4/linux/gtkmain.c @@ -509,6 +509,13 @@ trySetWinConfig( GtkAppGlobals* apg ) gtk_window_move (GTK_WINDOW(apg->window), xx, yy ); } +static void +handle_movescheck( GtkWidget* XP_UNUSED(widget), GtkAppGlobals* apg ) +{ + LaunchParams* params = apg->params; + checkForMsgsNow( params ); +} + static void makeGamesWindow( GtkAppGlobals* apg ) { @@ -532,6 +539,17 @@ makeGamesWindow( GtkAppGlobals* apg ) GtkWidget* vbox = gtk_box_new(GTK_ORIENTATION_VERTICAL, 0); gtk_container_add( GTK_CONTAINER(swin), vbox ); gtk_widget_show( vbox ); + + // add menubar here + GtkWidget* menubar = gtk_menu_bar_new(); + GtkWidget* netMenu = makeAddSubmenu( menubar, "Network" ); + if ( params->useHTTP ) { + (void)createAddItem( netMenu, "Check for moves", + (GCallback)handle_movescheck, apg ); + } + gtk_widget_show( menubar ); + gtk_box_pack_start( GTK_BOX(vbox), menubar, FALSE, TRUE, 0 ); + GtkWidget* list = init_games_list( apg ); gtk_container_add( GTK_CONTAINER(vbox), list ); From c68a06700945b39ab1d1eb60fe46acf1c2980a86 Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 24 Oct 2017 19:18:19 -0700 Subject: [PATCH 031/138] add option to not check moves except manually --- xwords4/linux/cursesmain.c | 15 +++++++++++++++ xwords4/linux/linuxmain.c | 10 +++++++++- xwords4/linux/main.h | 1 + xwords4/linux/relaycon.c | 29 +++++++++++++++-------------- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/xwords4/linux/cursesmain.c b/xwords4/linux/cursesmain.c index 8c7922a99..f76d8fe1f 100644 --- a/xwords4/linux/cursesmain.c +++ b/xwords4/linux/cursesmain.c @@ -1681,6 +1681,7 @@ static void cursesGotBuf( void* closure, const CommsAddrRec* addr, const XP_U8* buf, XP_U16 len ) { + LOG_FUNC(); CursesAppGlobals* globals = (CursesAppGlobals*)closure; XP_U32 clientToken; XP_ASSERT( sizeof(clientToken) < len ); @@ -1698,6 +1699,19 @@ cursesGotBuf( void* closure, const CommsAddrRec* addr, XP_LOGF( "%s: dropping packet; meant for a different device", __func__ ); } + LOG_RETURN_VOID(); +} + +static void +cursesGotForRow( void* closure, const CommsAddrRec* from, + sqlite3_int64 rowid, const XP_U8* buf, + XP_U16 len ) +{ + LOG_FUNC(); + CursesAppGlobals* globals = (CursesAppGlobals*)closure; + XP_ASSERT( globals->cGlobals.selRow == rowid ); + gameGotBuf( &globals->cGlobals, XP_TRUE, buf, len, from ); + LOG_RETURN_VOID(); } static gint @@ -1971,6 +1985,7 @@ cursesmain( XP_Bool isServer, LaunchParams* params ) if ( params->useUdp ) { RelayConnProcs procs = { .msgReceived = cursesGotBuf, + .msgForRow = cursesGotForRow, .msgNoticeReceived = cursesNoticeRcvd, .devIDReceived = cursesDevIDReceived, .msgErrorMsg = cursesErrorMsgRcvd, diff --git a/xwords4/linux/linuxmain.c b/xwords4/linux/linuxmain.c index a677328d8..f39459140 100644 --- a/xwords4/linux/linuxmain.c +++ b/xwords4/linux/linuxmain.c @@ -635,6 +635,7 @@ typedef enum { ,CMD_USEUDP ,CMD_NOUDP ,CMD_USEHTTP + ,CMD_NOHTTPAUTO ,CMD_DROPSENDRELAY ,CMD_DROPRCVRELAY ,CMD_DROPSENDSMS @@ -754,6 +755,7 @@ static CmdInfoRec CmdInfoRecs[] = { ,{ CMD_USEUDP, false, "use-udp", "connect to relay new-style, via udp not tcp (on by default)" } ,{ CMD_NOUDP, false, "no-use-udp", "connect to relay old-style, via tcp not udp" } ,{ CMD_USEHTTP, false, "use-http", "use relay's new http interfaces rather than sockets" } + ,{ CMD_NOHTTPAUTO, false, "no-http-auto", "When http's on, don't periodically connect to relay (manual only)" } ,{ CMD_DROPSENDRELAY, false, "drop-send-relay", "start new games with relay send disabled" } ,{ CMD_DROPRCVRELAY, false, "drop-receive-relay", "start new games with relay receive disabled" } @@ -975,6 +977,7 @@ linux_setupDevidParams( LaunchParams* params ) static int linux_init_relay_socket( CommonGlobals* cGlobals, const CommsAddrRec* addrRec ) { + XP_ASSERT( !cGlobals->params->useHTTP ); struct sockaddr_in to_sock; struct hostent* host; int sock = cGlobals->relaySocket; @@ -1176,6 +1179,7 @@ linux_relay_send( CommonGlobals* cGlobals, const XP_U8* buf, XP_U16 buflen, result = relaycon_send( cGlobals->params, buf, buflen, clientToken, addrRec ); } else { + XP_ASSERT( !cGlobals->params->useHTTP ); int sock = cGlobals->relaySocket; if ( sock == -1 ) { @@ -2403,6 +2407,9 @@ main( int argc, char** argv ) case CMD_USEHTTP: mainParams.useHTTP = true; break; + case CMD_NOHTTPAUTO: + mainParams.noHTTPAuto = true; + break; case CMD_DROPSENDRELAY: mainParams.commsDisableds[COMMS_CONN_RELAY][1] = XP_TRUE; @@ -2651,7 +2658,8 @@ main( int argc, char** argv ) if ( mainParams.useCurses ) { if ( mainParams.needsNewGame ) { /* curses doesn't have newgame dialog */ - usage( argv[0], "game params required for curses version, e.g. --name Eric --remote-player"); + usage( argv[0], "game params required for curses version, e.g. --name Eric " + "--remote-player --dict-dir ../ --game-dict dict.xwd"); } else { #if defined PLATFORM_NCURSES cursesmain( isServer, &mainParams ); diff --git a/xwords4/linux/main.h b/xwords4/linux/main.h index deab8b036..5f90e6999 100644 --- a/xwords4/linux/main.h +++ b/xwords4/linux/main.h @@ -106,6 +106,7 @@ typedef struct LaunchParams { XP_Bool useCurses; XP_Bool useUdp; XP_Bool useHTTP; + XP_Bool noHTTPAuto; XP_U16 splitPackets; XP_U16 chatsInterval; /* 0 means disabled */ XP_U16 askTimeout; diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 860fecc45..8e152b562 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -786,22 +786,23 @@ static void schedule_next_check( RelayConStorage* storage ) { XP_ASSERT( onMainThread(storage) ); + if ( !storage->params->noHTTPAuto ) { + if ( storage->moveCheckerID != 0 ) { + g_source_remove( storage->moveCheckerID ); + storage->moveCheckerID = 0; + } - if ( storage->moveCheckerID != 0 ) { - g_source_remove( storage->moveCheckerID ); - storage->moveCheckerID = 0; + storage->nextMoveCheckSecs *= 2; + if ( storage->nextMoveCheckSecs > MAX_MOVE_CHECK_SECS ) { + storage->nextMoveCheckSecs = MAX_MOVE_CHECK_SECS; + } else if ( storage->nextMoveCheckSecs == 0 ) { + storage->nextMoveCheckSecs = 1; + } + + storage->moveCheckerID = g_timeout_add( 1000 * storage->nextMoveCheckSecs, + checkForMoves, storage ); + XP_ASSERT( storage->moveCheckerID != 0 ); } - - storage->nextMoveCheckSecs *= 2; - if ( storage->nextMoveCheckSecs > MAX_MOVE_CHECK_SECS ) { - storage->nextMoveCheckSecs = MAX_MOVE_CHECK_SECS; - } else if ( storage->nextMoveCheckSecs == 0 ) { - storage->nextMoveCheckSecs = 1; - } - - storage->moveCheckerID = g_timeout_add( 1000 * storage->nextMoveCheckSecs, - checkForMoves, storage ); - XP_ASSERT( storage->moveCheckerID != 0 ); } static ssize_t From cc54621e45dfa903d7990dd8346ccd75ea072001 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 25 Oct 2017 05:51:45 -0700 Subject: [PATCH 032/138] store message first, remove on ack --- xwords4/relay/cref.cpp | 34 +++++++++++----------------------- xwords4/relay/cref.h | 3 +-- xwords4/relay/dbmgr.cpp | 28 ++++++++++++++++++++++------ xwords4/relay/dbmgr.h | 8 ++++---- xwords4/relay/xwrelay.cpp | 26 +++++++++++++++++--------- 5 files changed, 55 insertions(+), 44 deletions(-) diff --git a/xwords4/relay/cref.cpp b/xwords4/relay/cref.cpp index 86c8c8e8f..07dfa354e 100644 --- a/xwords4/relay/cref.cpp +++ b/xwords4/relay/cref.cpp @@ -875,13 +875,13 @@ putNetShort( uint8_t** bufpp, unsigned short s ) *bufpp += sizeof(s); } -void +int CookieRef::store_message( HostID dest, const uint8_t* buf, unsigned int len ) { logf( XW_LOGVERBOSE0, "%s: storing msg size %d for dest %d", __func__, len, dest ); - DBMgr::Get()->StoreMessage( ConnName(), dest, buf, len ); + return DBMgr::Get()->StoreMessage( ConnName(), dest, buf, len ); } void @@ -1044,6 +1044,7 @@ CookieRef::postCheckAllHere() void CookieRef::postDropDevice( HostID hostID ) { + logf( XW_LOGINFO, "%s(hostID=%d)", __func__, hostID ); CRefEvent evt( XWE_ACKTIMEOUT ); evt.u.ack.srcID = hostID; m_eventQueue.push_back( evt ); @@ -1192,21 +1193,16 @@ CookieRef::sendAnyStored( const CRefEvent* evt ) } typedef struct _StoreData { - string connName; - HostID dest; - uint8_t* buf; - int buflen; + int msgID; } StoreData; void CookieRef::storeNoAck( bool acked, uint32_t packetID, void* data ) { StoreData* sdata = (StoreData*)data; - if ( !acked ) { - DBMgr::Get()->StoreMessage( sdata->connName.c_str(), sdata->dest, - sdata->buf, sdata->buflen ); + if ( acked ) { + DBMgr::Get()->RemoveStoredMessages( &sdata->msgID, 1 ); } - free( sdata->buf ); delete sdata; } @@ -1237,17 +1233,13 @@ CookieRef::forward_or_store( const CRefEvent* evt ) } uint32_t packetID = 0; + int msgID = store_message( dest, buf, buflen ); if ( (NULL == destAddr) || !send_with_length( destAddr, dest, buf, buflen, true, &packetID ) ) { - store_message( dest, buf, buflen ); - } else if ( 0 != packetID ) { // sent via UDP + } else if ( 0 != msgID && 0 != packetID ) { // sent via UDP StoreData* data = new StoreData; - data->connName = m_connName; - data->dest = dest; - data->buf = (uint8_t*)malloc( buflen ); - memcpy( data->buf, buf, buflen ); - data->buflen = buflen; + data->msgID = msgID; UDPAckTrack::setOnAck( storeNoAck, packetID, data ); } @@ -1376,20 +1368,16 @@ CookieRef::sendAllHere( bool initial ) through the vector each time. */ HostID dest; for ( dest = 1; dest <= m_nPlayersSought; ++dest ) { - bool sent = false; *idLoc = dest; /* write in this target's hostId */ { RWReadLock rrl( &m_socketsRWLock ); HostRec* hr = m_sockets[dest-1]; if ( !!hr ) { - sent = send_with_length( &hr->m_addr, dest, buf, - bufp-buf, true ); + (void)send_with_length( &hr->m_addr, dest, buf, bufp-buf, true ); } } - if ( !sent ) { - store_message( dest, buf, bufp-buf ); - } + (void)store_message( dest, buf, bufp-buf ); } } /* sendAllHere */ diff --git a/xwords4/relay/cref.h b/xwords4/relay/cref.h index 508a4485c..61a677976 100644 --- a/xwords4/relay/cref.h +++ b/xwords4/relay/cref.h @@ -275,8 +275,7 @@ class CookieRef { bool notInUse(void) { return m_cid == 0; } - void store_message( HostID dest, const uint8_t* buf, - unsigned int len ); + int store_message( HostID dest, const uint8_t* buf, unsigned int len ); void send_stored_messages( HostID dest, const AddrInfo* addr ); void printSeeds( const char* caller ); diff --git a/xwords4/relay/dbmgr.cpp b/xwords4/relay/dbmgr.cpp index 95625a9b6..66ae8f43f 100644 --- a/xwords4/relay/dbmgr.cpp +++ b/xwords4/relay/dbmgr.cpp @@ -1014,15 +1014,16 @@ DBMgr::CountStoredMessages( DevIDRelay relayID ) return getCountWhere( MSGS_TABLE, test ); } -void +int DBMgr::StoreMessage( DevIDRelay destDevID, const uint8_t* const buf, int len ) { + int msgID = 0; clearHasNoMessages( destDevID ); size_t newLen; const char* fmt = "INSERT INTO " MSGS_TABLE " " - "(devid, %s, msglen) VALUES(%d, %s'%s', %d)"; + "(devid, %s, msglen) VALUES(%d, %s'%s', %d) RETURNING id"; StrWPF query; if ( m_useB64 ) { @@ -1038,13 +1039,20 @@ DBMgr::StoreMessage( DevIDRelay destDevID, const uint8_t* const buf, } logf( XW_LOGINFO, "%s: query: %s", __func__, query.c_str() ); - execSql( query ); + + PGresult* result = PQexec( getThreadConn(), query.c_str() ); + if ( 1 == PQntuples( result ) ) { + msgID = atoi( PQgetvalue( result, 0, 0 ) ); + } + PQclear( result ); + return msgID; } -void +int DBMgr::StoreMessage( const char* const connName, int destHid, const uint8_t* buf, int len ) { + int msgID = 0; clearHasNoMessages( connName, destHid ); DevIDRelay devID = getDevID( connName, destHid ); @@ -1074,7 +1082,7 @@ DBMgr::StoreMessage( const char* const connName, int destHid, #ifdef HAVE_STIME " AND stime='epoch'" #endif - " );", connName, destHid, b64 ); + " )", connName, destHid, b64 ); g_free( b64 ); } else { uint8_t* bytes = PQescapeByteaConn( getThreadConn(), buf, @@ -1085,9 +1093,17 @@ DBMgr::StoreMessage( const char* const connName, int destHid, "E", bytes, len ); PQfreemem( bytes ); } + query.catf(" RETURNING id;"); logf( XW_LOGINFO, "%s: query: %s", __func__, query.c_str() ); - execSql( query ); + PGresult* result = PQexec( getThreadConn(), query.c_str() ); + if ( 1 == PQntuples( result ) ) { + msgID = atoi( PQgetvalue( result, 0, 0 ) ); + } else { + logf( XW_LOGINFO, "Not stored; duplicate?" ); + } + PQclear( result ); + return msgID; } void diff --git a/xwords4/relay/dbmgr.h b/xwords4/relay/dbmgr.h index 690ca5c39..22d3ba8ce 100644 --- a/xwords4/relay/dbmgr.h +++ b/xwords4/relay/dbmgr.h @@ -137,10 +137,10 @@ class DBMgr { /* message storage -- different DB */ int CountStoredMessages( const char* const connName ); int CountStoredMessages( DevIDRelay relayID ); - void StoreMessage( DevIDRelay destRelayID, const uint8_t* const buf, - int len ); - void StoreMessage( const char* const connName, int destHid, - const uint8_t* const buf, int len ); + int StoreMessage( DevIDRelay destRelayID, const uint8_t* const buf, + int len ); + int StoreMessage( const char* const connName, int destHid, + const uint8_t* const buf, int len ); void GetStoredMessages( DevIDRelay relayID, vector& msgs ); void GetStoredMessages( const char* const connName, HostID hid, vector& msgs ); diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index 666bee3f3..1f91dea70 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -761,13 +761,17 @@ send_havemsgs( const AddrInfo* addr ) class MsgClosure { public: MsgClosure( DevIDRelay dest, const vector* packet, - OnMsgAckProc proc, void* procClosure ) + int msgID, OnMsgAckProc proc, void* procClosure ) { + assert(m_msgID != 0); m_destDevID = dest; m_packet = *packet; m_proc = proc; m_procClosure = procClosure; + m_msgID = msgID; } + int getMsgID() { return m_msgID; } + int m_msgID; DevIDRelay m_destDevID; vector m_packet; OnMsgAckProc m_proc; @@ -778,9 +782,14 @@ static void onPostedMsgAcked( bool acked, uint32_t packetID, void* data ) { MsgClosure* mc = (MsgClosure*)data; - if ( !acked ) { - DBMgr::Get()->StoreMessage( mc->m_destDevID, mc->m_packet.data(), - mc->m_packet.size() ); + int msgID = mc->getMsgID(); + if ( acked ) { + DBMgr::Get()->RemoveStoredMessages( &msgID, 1 ); + } else { + assert( msgID != 0 ); + // So we only store after ack fails? Change that!!! + // DBMgr::Get()->StoreMessage( mc->m_destDevID, mc->m_packet.data(), + // mc->m_packet.size() ); } if ( NULL != mc->m_proc ) { (*mc->m_proc)( acked, mc->m_destDevID, packetID, mc->m_procClosure ); @@ -793,6 +802,8 @@ static bool post_or_store( DevIDRelay destDevID, vector& packet, uint32_t packetID, OnMsgAckProc proc, void* procClosure ) { + int msgID = DBMgr::Get()->StoreMessage( destDevID, packet.data(), packet.size() ); + const AddrInfo::AddrUnion* addru = DevMgr::Get()->get( destDevID ); bool canSendNow = !!addru; @@ -804,16 +815,13 @@ post_or_store( DevIDRelay destDevID, vector& packet, uint32_t packetID, if ( get_addr_info_if( &addr, &sock, &dest_addr ) ) { sent = 0 < send_packet_via_udp_impl( packet, sock, dest_addr ); - if ( sent ) { - MsgClosure* mc = new MsgClosure( destDevID, &packet, + if ( sent && msgID != 0 ) { + MsgClosure* mc = new MsgClosure( destDevID, &packet, msgID, proc, procClosure ); UDPAckTrack::setOnAck( onPostedMsgAcked, packetID, (void*)mc ); } } } - if ( !sent ) { - DBMgr::Get()->StoreMessage( destDevID, packet.data(), packet.size() ); - } return sent; } From d05a67ed4bb4c5ca0a2aab1d0f5159f61371db5e Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 25 Oct 2017 06:15:46 -0700 Subject: [PATCH 033/138] switch to milliseconds --- xwords4/linux/relaycon.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 8e152b562..ae38c7f6f 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -29,13 +29,13 @@ #include "comtypes.h" #include "gamesdb.h" -#define MAX_MOVE_CHECK_SECS ((XP_U16)(60 * 60 * 24)) +#define MAX_MOVE_CHECK_MS ((XP_U16)(1000 * 60 * 60 * 24)) #define RELAY_API_PROTO "http" typedef struct _RelayConStorage { pthread_t mainThread; guint moveCheckerID; - XP_U16 nextMoveCheckSecs; + XP_U32 nextMoveCheckMS; pthread_cond_t relayCondVar; pthread_mutex_t relayMutex; GSList* relayTaskList; @@ -779,7 +779,7 @@ static void reset_schedule_check_interval( RelayConStorage* storage ) { XP_ASSERT( onMainThread(storage) ); - storage->nextMoveCheckSecs = 0; + storage->nextMoveCheckMS = 0; } static void @@ -792,14 +792,14 @@ schedule_next_check( RelayConStorage* storage ) storage->moveCheckerID = 0; } - storage->nextMoveCheckSecs *= 2; - if ( storage->nextMoveCheckSecs > MAX_MOVE_CHECK_SECS ) { - storage->nextMoveCheckSecs = MAX_MOVE_CHECK_SECS; - } else if ( storage->nextMoveCheckSecs == 0 ) { - storage->nextMoveCheckSecs = 1; + storage->nextMoveCheckMS *= 2; + if ( storage->nextMoveCheckMS > MAX_MOVE_CHECK_MS ) { + storage->nextMoveCheckMS = MAX_MOVE_CHECK_MS; + } else if ( storage->nextMoveCheckMS == 0 ) { + storage->nextMoveCheckMS = 250; } - storage->moveCheckerID = g_timeout_add( 1000 * storage->nextMoveCheckSecs, + storage->moveCheckerID = g_timeout_add( storage->nextMoveCheckMS, checkForMoves, storage ); XP_ASSERT( storage->moveCheckerID != 0 ); } From 81df91c4db8555c427cf70feb036fa61f389a84d Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 25 Oct 2017 06:34:55 -0700 Subject: [PATCH 034/138] add commandline option to spec log dir location --- xwords4/linux/scripts/discon_ok2.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index 8efe77d4d..a658b22b7 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -1,7 +1,7 @@ #!/bin/bash set -u -e -LOGDIR=$(basename $0)_logs +LOGDIR=./$(basename $0)_logs APP_NEW="" DO_CLEAN="" APP_NEW_PARAMS="" @@ -598,6 +598,7 @@ function getArg() { function usage() { [ $# -gt 0 ] && echo "Error: $1" >&2 echo "Usage: $(basename $0) \\" >&2 + echo " [--log-root] # default: . \\" >&2 echo " [--dup-packets] # send all packets twice \\" >&2 echo " [--clean-start] \\" >&2 echo " [--game-dict ]* \\" >&2 @@ -653,6 +654,11 @@ while [ "$#" -gt 0 ]; do APPS_OLD[${#APPS_OLD[@]}]=$(getArg $*) shift ;; + --log-root) + [ -d $2 ] || usage "$1: no such directory $2" + LOGDIR=$2/$(basename $0)_logs + shift + ;; --dup-packets) DUP_PACKETS=1 ;; @@ -761,7 +767,8 @@ for FILE in $(ls $LOGDIR/*.{xwg,txt} 2>/dev/null); do done if [ -z "$RESUME" -a -d $LOGDIR ]; then - mv $LOGDIR /tmp/${LOGDIR}_$$ + NEWNAME="$(basename $LOGDIR)_$$" + (cd $(dirname $LOGDIR) && mv $(basename $LOGDIR) /tmp/${NEWNAME}) fi mkdir -p $LOGDIR From 14d9063ad06ebbd8713efb0bf71eff6b845523fc Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 25 Oct 2017 06:43:10 -0700 Subject: [PATCH 035/138] add --room to error message/hint --- xwords4/linux/linuxmain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xwords4/linux/linuxmain.c b/xwords4/linux/linuxmain.c index f39459140..5edeea05f 100644 --- a/xwords4/linux/linuxmain.c +++ b/xwords4/linux/linuxmain.c @@ -2658,8 +2658,8 @@ main( int argc, char** argv ) if ( mainParams.useCurses ) { if ( mainParams.needsNewGame ) { /* curses doesn't have newgame dialog */ - usage( argv[0], "game params required for curses version, e.g. --name Eric " - "--remote-player --dict-dir ../ --game-dict dict.xwd"); + usage( argv[0], "game params required for curses version, e.g. --name Eric --room MyRoom" + " --remote-player --dict-dir ../ --game-dict dict.xwd"); } else { #if defined PLATFORM_NCURSES cursesmain( isServer, &mainParams ); From 6e06fdea07d5267ec075c0a91df3857f950da0e1 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 25 Oct 2017 19:56:56 -0700 Subject: [PATCH 036/138] stop storing cid in database It's a runtime-only thing, explicitly removed from db on boot. So add a map from connname->cid to the dbmgr class and modify that rather than a column. Passes discon_ok2.sh tests as long as use-http stuff isn't on. --- xwords4/relay/dbmgr.cpp | 112 ++++++++++++++-------------- xwords4/relay/dbmgr.h | 2 + xwords4/relay/scripts/showinplay.sh | 2 +- 3 files changed, 57 insertions(+), 59 deletions(-) diff --git a/xwords4/relay/dbmgr.cpp b/xwords4/relay/dbmgr.cpp index 66ae8f43f..86073552a 100644 --- a/xwords4/relay/dbmgr.cpp +++ b/xwords4/relay/dbmgr.cpp @@ -69,21 +69,8 @@ DBMgr::DBMgr() pthread_key_create( &m_conn_key, destr_function ); pthread_mutex_init( &m_haveNoMessagesMutex, NULL ); + pthread_mutex_init( &m_cidsMutex, NULL ); - /* Now figure out what the largest cid currently is. There must be a way - to get postgres to do this for me.... */ - /* const char* query = "SELECT cid FROM games ORDER BY cid DESC LIMIT 1"; */ - /* PGresult* result = PQexec( m_pgconn, query ); */ - /* if ( 0 == PQntuples( result ) ) { */ - /* m_nextCID = 1; */ - /* } else { */ - /* char* value = PQgetvalue( result, 0, 0 ); */ - /* m_nextCID = 1 + atoi( value ); */ - /* } */ - /* PQclear(result); */ - /* logf( XW_LOGINFO, "%s: m_nextCID=%d", __func__, m_nextCID ); */ - - // I've seen rand returning the same series several times.... srand( time( NULL ) ); } @@ -103,11 +90,13 @@ DBMgr::AddNew( const char* cookie, const char* connName, CookieID cid, if ( !cookie ) cookie = ""; if ( !connName ) connName = ""; + MutexLock ml( &m_cidsMutex ); + m_cidsMap[ connName ] = cid; + QueryBuilder qb; qb.appendQueryf( "INSERT INTO " GAMES_TABLE - " (cid, room, connName, nTotal, lang, pub)" - " VALUES( $$, $$, $$, $$, $$, $$ )" ) - .appendParam(cid) + " (room, connName, nTotal, lang, pub)" + " VALUES( $$, $$, $$, $$, $$ )" ) .appendParam(cookie) .appendParam(connName) .appendParam(nPlayersT) @@ -136,7 +125,7 @@ DBMgr::FindGameFor( const char* connName, char* cookieBuf, int bufLen, { bool found = false; - const char* fmt = "SELECT cid, room, lang, nPerDevice, dead FROM " + const char* fmt = "SELECT room, lang, dead FROM " GAMES_TABLE " WHERE connName = '%s' AND nTotal = %d " "AND %d = seeds[%d] AND 'A' = ack[%d] " ; @@ -144,14 +133,18 @@ DBMgr::FindGameFor( const char* connName, char* cookieBuf, int bufLen, query.catf( fmt, connName, nPlayersS, seed, hid, hid ); logf( XW_LOGINFO, "query: %s", query.c_str() ); + MutexLock ml( &m_cidsMutex ); // not sure I need this.... + PGresult* result = PQexec( getThreadConn(), query.c_str() ); assert( 1 >= PQntuples( result ) ); found = 1 == PQntuples( result ); if ( found ) { - *cidp = atoi( PQgetvalue( result, 0, 0 ) ); - snprintf( cookieBuf, bufLen, "%s", PQgetvalue( result, 0, 1 ) ); - *langP = atoi( PQgetvalue( result, 0, 2 ) ); - *isDead = 't' == PQgetvalue( result, 0, 4 )[0]; + int col = 0; + snprintf( cookieBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); + *langP = atoi( PQgetvalue( result, 0, col++ ) ); + *isDead = 't' == PQgetvalue( result, 0, col++ )[0]; + + *cidp = m_cidsMap[connName]; } PQclear( result ); @@ -163,9 +156,11 @@ CookieID DBMgr::FindGame( const char* connName, char* cookieBuf, int bufLen, int* langP, int* nPlayersTP, int* nPlayersHP, bool* isDead ) { + MutexLock ml( &m_cidsMutex ); + CookieID cid = 0; - const char* fmt = "SELECT cid, room, lang, nTotal, nPerDevice, dead FROM " + const char* fmt = "SELECT room, lang, nTotal, nPerDevice, dead FROM " GAMES_TABLE " WHERE connName = '%s'" // " LIMIT 1" ; @@ -176,12 +171,13 @@ DBMgr::FindGame( const char* connName, char* cookieBuf, int bufLen, PGresult* result = PQexec( getThreadConn(), query.c_str() ); assert( 1 >= PQntuples( result ) ); if ( 1 == PQntuples( result ) ) { - cid = atoi( PQgetvalue( result, 0, 0 ) ); - snprintf( cookieBuf, bufLen, "%s", PQgetvalue( result, 0, 1 ) ); - *langP = atoi( PQgetvalue( result, 0, 2 ) ); - *nPlayersTP = atoi( PQgetvalue( result, 0, 3 ) ); - *nPlayersHP = atoi( PQgetvalue( result, 0, 4 ) ); - *isDead = 't' == PQgetvalue( result, 0, 5 )[0]; + int col = 0; + cid = m_cidsMap[connName]; + snprintf( cookieBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); + *langP = atoi( PQgetvalue( result, 0, col++ ) ); + *nPlayersTP = atoi( PQgetvalue( result, 0, col++ ) ); + *nPlayersHP = atoi( PQgetvalue( result, 0, col++ ) ); + *isDead = 't' == PQgetvalue( result, 0, col++ )[0]; } PQclear( result ); @@ -271,7 +267,7 @@ DBMgr::SeenSeed( const char* cookie, unsigned short seed, CookieID* cid ) { QueryBuilder qb; - qb.appendQueryf( "SELECT cid, connName, seeds, sum_array(nPerDevice) FROM " + qb.appendQueryf( "SELECT connName, seeds, sum_array(nPerDevice) FROM " GAMES_TABLE " WHERE NOT dead" " AND room ILIKE $$" @@ -288,17 +284,21 @@ DBMgr::SeenSeed( const char* cookie, unsigned short seed, .appendParam(wantsPublic?"TRUE":"FALSE" ) .finish(); + MutexLock ml( &m_cidsMutex ); + PGresult* result = PQexecParams( getThreadConn(), qb.c_str(), qb.paramCount(), NULL, qb.paramValues(), NULL, NULL, 0 ); bool found = 1 == PQntuples( result ); if ( found ) { - *cid = atoi( PQgetvalue( result, 0, 0 ) ); - *nPlayersHP = here_less_seed( PQgetvalue( result, 0, 2 ), - atoi( PQgetvalue( result, 0, 3 ) ), - seed ); - snprintf( connNameBuf, bufLen, "%s", PQgetvalue( result, 0, 1 ) ); + int col = 0; + snprintf( connNameBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); + *cid = m_cidsMap[connNameBuf]; + + const char* seeds = PQgetvalue( result, 0, col++ ); + int perDeviceSum = atoi( PQgetvalue( result, 0, col++ ) ); + *nPlayersHP = here_less_seed( seeds, perDeviceSum, seed ); } PQclear( result ); logf( XW_LOGINFO, "%s(%4X)=>%s", __func__, seed, found?"true":"false" ); @@ -311,7 +311,7 @@ DBMgr::FindOpen( const char* cookie, int lang, int nPlayersT, int nPlayersH, int* nPlayersHP ) { QueryBuilder qb; - qb.appendQueryf("SELECT cid, connName, sum_array(nPerDevice) FROM " + qb.appendQueryf("SELECT connName, sum_array(nPerDevice) FROM " GAMES_TABLE " WHERE NOT dead" " AND room ILIKE $$" @@ -327,15 +327,18 @@ DBMgr::FindOpen( const char* cookie, int lang, int nPlayersT, int nPlayersH, .appendParam(wantsPublic?"TRUE":"FALSE" ) .finish(); + MutexLock ml( &m_cidsMutex ); + PGresult* result = PQexecParams( getThreadConn(), qb.c_str(), qb.paramCount(), NULL, qb.paramValues(), NULL, NULL, 0 ); CookieID cid = 0; if ( 1 == PQntuples( result ) ) { - cid = atoi( PQgetvalue( result, 0, 0 ) ); - snprintf( connNameBuf, bufLen, "%s", PQgetvalue( result, 0, 1 ) ); - *nPlayersHP = atoi( PQgetvalue( result, 0, 2 ) ); + int col = 0; + cid = m_cidsMap[connNameBuf]; + snprintf( connNameBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); + *nPlayersHP = atoi( PQgetvalue( result, 0, col++ ) ); /* cid may be 0, but should use game anyway */ } PQclear( result ); @@ -641,27 +644,17 @@ DBMgr::HaveDevice( const char* connName, HostID hid, int seed ) bool DBMgr::AddCID( const char* const connName, CookieID cid ) { - const char* fmt = "UPDATE " GAMES_TABLE " SET cid = %d " - " WHERE connName = '%s' AND cid IS NULL"; - StrWPF query; - query.catf( fmt, cid, connName ); - logf( XW_LOGINFO, "%s: query: %s", __func__, query.c_str() ); + MutexLock ml( &m_cidsMutex ); + m_cidsMap[ connName ] = cid; - bool result = execSql( query ); - logf( XW_LOGINFO, "%s(cid=%d)=>%d", __func__, cid, result ); - return result; + return TRUE; } void DBMgr::ClearCID( const char* connName ) { - const char* fmt = "UPDATE " GAMES_TABLE " SET cid = null " - "WHERE connName = '%s'"; - StrWPF query; - query.catf( fmt, connName ); - logf( XW_LOGINFO, "%s: query: %s", __func__, query.c_str() ); - - execSql( query ); + MutexLock ml( &m_cidsMutex ); + m_cidsMap.erase( connName ); } void @@ -699,9 +692,11 @@ DBMgr::RecordSent( const int* msgIDs, int nMsgIDs ) if ( PGRES_TUPLES_OK == PQresultStatus( result ) ) { int ntuples = PQntuples( result ); for ( int ii = 0; ii < ntuples; ++ii ) { - RecordSent( PQgetvalue( result, ii, 0 ), - atoi( PQgetvalue( result, ii, 1 ) ), - atoi( PQgetvalue( result, ii, 2 ) ) ); + int col = 0; + const char* const connName = PQgetvalue( result, ii, col++ ); + HostID hid = atoi( PQgetvalue( result, ii, col++ ) ); + int nBytes = atoi( PQgetvalue( result, ii, col++ ) ); + RecordSent( connName, hid, nBytes ); } } PQclear( result ); @@ -781,7 +776,8 @@ DBMgr::WaitDBConn( void ) void DBMgr::ClearCIDs( void ) { - execSql( "UPDATE " GAMES_TABLE " set cid = null" ); + MutexLock ml( &m_cidsMutex ); + m_cidsMap.clear(); } void diff --git a/xwords4/relay/dbmgr.h b/xwords4/relay/dbmgr.h index 22d3ba8ce..325b88de2 100644 --- a/xwords4/relay/dbmgr.h +++ b/xwords4/relay/dbmgr.h @@ -190,6 +190,8 @@ class DBMgr { pthread_mutex_t m_haveNoMessagesMutex; set m_haveNoMessagesDevID; set m_haveNoMessagesConnname; + pthread_mutex_t m_cidsMutex; + map m_cidsMap; }; /* DBMgr */ diff --git a/xwords4/relay/scripts/showinplay.sh b/xwords4/relay/scripts/showinplay.sh index a3b9955a1..24e906ada 100755 --- a/xwords4/relay/scripts/showinplay.sh +++ b/xwords4/relay/scripts/showinplay.sh @@ -54,7 +54,7 @@ echo "; relay pid[s]: $(pidof xwrelay)" echo "Row count:" $(psql -t xwgames -c "select count(*) FROM games $QUERY;") # Games -echo "SELECT dead as d,connname,cid,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ +echo "SELECT dead as d,connname,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ "FROM games $QUERY ORDER BY NOT dead, ctime DESC LIMIT $LIMIT;" \ | psql xwgames From 051b6a722073e4e418980944b092d211ae513838 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 25 Oct 2017 20:00:55 -0700 Subject: [PATCH 037/138] change resign-ratio to resign-pct To match the way other pct stuff works, in part because it seemed it didn't. --- xwords4/linux/scripts/discon_ok2.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index a658b22b7..9929da583 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -17,7 +17,7 @@ SAVE_GOOD="" MINDEVS="" MAXDEVS="" ONEPER="" -RESIGN_RATIO="" +RESIGN_PCT=0 DROP_N="" MINRUN=2 # seconds ONE_PER_ROOM="" # don't run more than one device at a time per room @@ -357,11 +357,11 @@ kill_from_log() { } maybe_resign() { - if [ "$RESIGN_RATIO" -gt 0 ]; then + if [ "$RESIGN_PCT" -gt 0 ]; then KEY=$1 LOG=${LOGS[$KEY]} if grep -aq XWRELAY_ALLHERE $LOG; then - if [ 0 -eq $(($RANDOM % $RESIGN_RATIO)) ]; then + if [ $((${RANDOM}%100)) -lt $RESIGN_PCT ]; then echo "making $LOG $(connName $LOG) resign..." kill_from_log $LOG && close_device $KEY $DEADDIR "resignation forced" || /bin/true fi @@ -613,7 +613,7 @@ function usage() { echo " [--old-app &2 echo " [--one-per] # force one player per device \\" >&2 echo " [--port ] \\" >&2 - echo " [--resign-ratio <0 <= n <=1000 > \\" >&2 + echo " [--resign-pct <0 <= n <=100 > \\" >&2 echo " [--no-timeout] # run until all games done \\" >&2 echo " [--seed ] \\" >&2 echo " [--send-chat \\" >&2 @@ -704,15 +704,16 @@ while [ "$#" -gt 0 ]; do ;; --http-pct) HTTP_PCT=$(getArg $*) - [ $HTTP_PCT -ge 0 -a $HTTP_PCT -le 100 ] || usage "n must be 0 <= n <= 100" + [ $HTTP_PCT -ge 0 -a $HTTP_PCT -le 100 ] || usage "$1: n must be 0 <= n <= 100" shift ;; --send-chat) SEND_CHAT=$(getArg $*) shift ;; - --resign-ratio) - RESIGN_RATIO=$(getArg $*) + --resign-pct) + RESIGN_PCT=$(getArg $*) + [ $RESIGN_PCT -ge 0 -a $RESIGN_PCT -le 100 ] || usage "$1: n must be 0 <= n <= 100" shift ;; --no-timeout) @@ -739,7 +740,7 @@ done [ -z "$PORT" ] && PORT=10997 [ -z "$TIMEOUT" ] && TIMEOUT=$((NGAMES*60+500)) [ -z "$SAVE_GOOD" ] && SAVE_GOOD=YES -[ -z "$RESIGN_RATIO" -a "$NGAMES" -gt 1 ] && RESIGN_RATIO=1000 || RESIGN_RATIO=0 +# [ -z "$RESIGN_PCT" -a "$NGAMES" -gt 1 ] && RESIGN_RATIO=1000 || RESIGN_RATIO=0 [ -z "$DROP_N" ] && DROP_N=0 [ -z "$USE_GTK" ] && USE_GTK=FALSE [ -z "$UPGRADE_ODDS" ] && UPGRADE_ODDS=10 @@ -780,7 +781,7 @@ DEADDIR=$LOGDIR/dead mkdir -p $DEADDIR for VAR in NGAMES NROOMS USE_GTK TIMEOUT HOST PORT SAVE_GOOD \ - MINDEVS MAXDEVS ONEPER RESIGN_RATIO DROP_N ALL_VIA_RQ SEED \ + MINDEVS MAXDEVS ONEPER RESIGN_PCT DROP_N ALL_VIA_RQ SEED \ APP_NEW; do echo "$VAR:" $(eval "echo \$${VAR}") 1>&2 done From 364533106134408d373575f0498050cca5a88657 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 25 Oct 2017 20:33:25 -0700 Subject: [PATCH 038/138] pass hid in to db function getting per-dev info Somehow I've been failing to treat this column as an array for some time. I don't have any test cases that failed but it was clearly wrong. Test cases still pass.... --- xwords4/relay/crefmgr.cpp | 8 ++++---- xwords4/relay/crefmgr.h | 4 ++-- xwords4/relay/dbmgr.cpp | 8 ++++---- xwords4/relay/dbmgr.h | 2 +- xwords4/relay/xwrelay.cpp | 8 ++++---- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/xwords4/relay/crefmgr.cpp b/xwords4/relay/crefmgr.cpp index c08a0d2c9..43d09b618 100644 --- a/xwords4/relay/crefmgr.cpp +++ b/xwords4/relay/crefmgr.cpp @@ -337,7 +337,7 @@ CRefMgr::getMakeCookieRef( const char* connName, const char* cookie, } /* getMakeCookieRef */ CidInfo* -CRefMgr::getMakeCookieRef( const char* const connName, bool* isDead ) +CRefMgr::getMakeCookieRef( const char* const connName, HostID hid, bool* isDead ) { CookieRef* cref = NULL; CidInfo* cinfo = NULL; @@ -347,7 +347,7 @@ CRefMgr::getMakeCookieRef( const char* const connName, bool* isDead ) int nAlreadyHere = 0; for ( ; ; ) { /* for: see comment above */ - CookieID cid = m_db->FindGame( connName, curCookie, sizeof(curCookie), + CookieID cid = m_db->FindGame( connName, hid, curCookie, sizeof(curCookie), &curLangCode, &nPlayersT, &nAlreadyHere, isDead ); if ( 0 != cid ) { /* already open */ @@ -672,13 +672,13 @@ SafeCref::SafeCref( const char* connName, const char* cookie, HostID hid, } /* ConnName case -- must exist (unless DB record's been removed */ -SafeCref::SafeCref( const char* const connName ) +SafeCref::SafeCref( const char* const connName, HostID hid ) : m_cinfo( NULL ) , m_mgr( CRefMgr::Get() ) , m_isValid( false ) { bool isDead = false; - CidInfo* cinfo = m_mgr->getMakeCookieRef( connName, &isDead ); + CidInfo* cinfo = m_mgr->getMakeCookieRef( connName, hid, &isDead ); if ( NULL != cinfo && NULL != cinfo->GetRef() ) { assert( cinfo->GetCid() == cinfo->GetRef()->GetCid() ); m_locked = cinfo->GetRef()->Lock(); diff --git a/xwords4/relay/crefmgr.h b/xwords4/relay/crefmgr.h index 315bce9c8..809dea796 100644 --- a/xwords4/relay/crefmgr.h +++ b/xwords4/relay/crefmgr.h @@ -128,7 +128,7 @@ class CRefMgr { int nPlayersS, int seed, int langCode, bool isPublic, bool* isDead ); - CidInfo* getMakeCookieRef( const char* const connName, bool* isDead ); + CidInfo* getMakeCookieRef( const char* const connName, HostID hid, bool* isDead ); CidInfo* getCookieRef( CookieID cid, bool failOk = false ); CidInfo* getCookieRef( const AddrInfo* addr ); @@ -179,7 +179,7 @@ class SafeCref { const AddrInfo* addr, int clientVersion, DevID* devID, int nPlayersH, int nPlayersS, unsigned short gameSeed, int clientIndx, int langCode, bool wantsPublic, bool makePublic ); - SafeCref( const char* const connName ); + SafeCref( const char* const connName, HostID hid ); SafeCref( CookieID cid, bool failOk = false ); SafeCref( const AddrInfo* addr ); /* SafeCref( CookieRef* cref ); */ diff --git a/xwords4/relay/dbmgr.cpp b/xwords4/relay/dbmgr.cpp index 86073552a..4978f5b9e 100644 --- a/xwords4/relay/dbmgr.cpp +++ b/xwords4/relay/dbmgr.cpp @@ -153,19 +153,19 @@ DBMgr::FindGameFor( const char* connName, char* cookieBuf, int bufLen, } /* FindGameFor */ CookieID -DBMgr::FindGame( const char* connName, char* cookieBuf, int bufLen, +DBMgr::FindGame( const char* connName, HostID hid, char* roomBuf, int roomBufLen, int* langP, int* nPlayersTP, int* nPlayersHP, bool* isDead ) { MutexLock ml( &m_cidsMutex ); CookieID cid = 0; - const char* fmt = "SELECT room, lang, nTotal, nPerDevice, dead FROM " + const char* fmt = "SELECT room, lang, nTotal, nPerDevice[%d], dead FROM " GAMES_TABLE " WHERE connName = '%s'" // " LIMIT 1" ; StrWPF query; - query.catf( fmt, connName ); + query.catf( fmt, hid, connName ); logf( XW_LOGINFO, "query: %s", query.c_str() ); PGresult* result = PQexec( getThreadConn(), query.c_str() ); @@ -173,7 +173,7 @@ DBMgr::FindGame( const char* connName, char* cookieBuf, int bufLen, if ( 1 == PQntuples( result ) ) { int col = 0; cid = m_cidsMap[connName]; - snprintf( cookieBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); + snprintf( roomBuf, roomBufLen, "%s", PQgetvalue( result, 0, col++ ) ); *langP = atoi( PQgetvalue( result, 0, col++ ) ); *nPlayersTP = atoi( PQgetvalue( result, 0, col++ ) ); *nPlayersHP = atoi( PQgetvalue( result, 0, col++ ) ); diff --git a/xwords4/relay/dbmgr.h b/xwords4/relay/dbmgr.h index 325b88de2..5550d90df 100644 --- a/xwords4/relay/dbmgr.h +++ b/xwords4/relay/dbmgr.h @@ -75,7 +75,7 @@ class DBMgr { bool FindRelayIDFor( const char* connName, HostID hid, unsigned short seed, const DevID* host, DevIDRelay* devID ); - CookieID FindGame( const char* connName, char* cookieBuf, int bufLen, + CookieID FindGame( const char* connName, HostID hid, char* cookieBuf, int bufLen, int* langP, int* nPlayersTP, int* nPlayersHP, bool* isDead ); diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index 1f91dea70..bb260dabe 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -1446,7 +1446,7 @@ handleProxyMsgs( int sock, const AddrInfo* addr, const uint8_t* bufp, } unsigned short nMsgs; if ( getNetShort( &bufp, end, &nMsgs ) ) { - SafeCref scr( connName ); + SafeCref scr( connName, hid ); while ( scr.IsValid() && nMsgs-- > 0 ) { unsigned short len; if ( getNetShort( &bufp, end, &len ) ) { @@ -1536,7 +1536,7 @@ proxy_thread_proc( UdpThreadClosure* utc ) sizeof( connName ), &hid ) ) { break; } - SafeCref scr( connName ); + SafeCref scr( connName, hid ); scr.DeviceGone( hid, seed ); } } @@ -1774,7 +1774,7 @@ handle_udp_packet( UdpThreadClosure* utc ) logf( XW_LOGERROR, "parse failed!!!" ); break; } - SafeCref scr( connName ); + SafeCref scr( connName, hid ); if ( scr.IsValid() ) { AddrInfo addr( g_udpsock, clientToken, utc->saddr() ); handlePutMessage( scr, hid, &addr, end - ptr, &ptr, end ); @@ -1841,7 +1841,7 @@ handle_udp_packet( UdpThreadClosure* utc ) string connName; if ( DBMgr::Get()->FindPlayer( devID.asRelayID(), clientToken, connName, &hid, &seed ) ) { - SafeCref scr( connName.c_str() ); + SafeCref scr( connName.c_str(), hid ); scr.DeviceGone( hid, seed ); } } From a8f06b53e25897bc9ee981f958504398528018de Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 27 Oct 2017 05:57:40 -0700 Subject: [PATCH 039/138] test script; log threadid --- xwords4/linux/linuxutl.c | 4 ++-- xwords4/linux/scripts/start-pair.sh | 37 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100755 xwords4/linux/scripts/start-pair.sh diff --git a/xwords4/linux/linuxutl.c b/xwords4/linux/linuxutl.c index 3f3cc6906..dd9e87f9b 100644 --- a/xwords4/linux/linuxutl.c +++ b/xwords4/linux/linuxutl.c @@ -50,8 +50,8 @@ linux_debugf( const char* format, ... ) gettimeofday( &tv, &tz ); timp = localtime( &tv.tv_sec ); - size_t len = snprintf( buf, sizeof(buf), "<%d>%.2d:%.2d:%.2d:", getpid(), - timp->tm_hour, timp->tm_min, timp->tm_sec ); + size_t len = snprintf( buf, sizeof(buf), "<%d:%lu>%.2d:%.2d:%.2d:", getpid(), + pthread_self(), timp->tm_hour, timp->tm_min, timp->tm_sec ); XP_ASSERT( len < sizeof(buf) ); va_start(ap, format); diff --git a/xwords4/linux/scripts/start-pair.sh b/xwords4/linux/scripts/start-pair.sh new file mode 100755 index 000000000..68175be81 --- /dev/null +++ b/xwords4/linux/scripts/start-pair.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +set -e -u + +DB_TMPLATE=_cursesdb_ +LOG_TMPLATE=_curseslog_ +ROOM_TMPLATE=cursesRoom + +echo "delete from msgs;" | psql xwgames +echo "delete from games where room like '$ROOM_TMPLATE%';" | psql xwgames + +rm -f ${DB_TMPLATE}*.sqldb +rm -f ${LOG_TMPLATE}* + +PIDS='' +for GAME in $(seq 1); do + ROOM=${ROOM_TMPLATE}${GAME} + # for N in $(seq 2); do + for N in $(seq 1); do + DB=$DB_TMPLATE${GAME}_${N}.sqldb + LOG=$LOG_TMPLATE${GAME}_${N}.log + exec ./obj_linux_memdbg/xwords --server --curses --remote-player --name Player \ + --room $ROOM --game-dict dict.xwd \ + --skip-confirm --db $DB --close-stdin --server \ + >/dev/null 2>>$LOG & + PIDS="$PIDS $!" + done +done +echo "launched $PIDS" + +# --use-http \ + +sleep 10 + +for PID in $PIDS; do + kill $PID +done From 952272172a3d338a7f6be8f7f8cb6a43036c69f7 Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 27 Oct 2017 05:58:35 -0700 Subject: [PATCH 040/138] always log bytes sent --- xwords4/linux/relaycon.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index ae38c7f6f..555db4120 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -135,7 +135,7 @@ static void addJsonParams( CURL* curl, const char* name, json_object* param ) { const char* asStr = json_object_to_json_string( param ); - XP_LOGF( "%s: adding param: %s", __func__, asStr ); + XP_LOGF( "%s: adding param (with name %s): %s", __func__, name, asStr ); char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); gchar* buf = g_strdup_printf( "%s=%s", name, curl_params ); @@ -667,7 +667,6 @@ handlePost( RelayTask* task ) runWitCurl( task, "post", "params", jobj ); - // Put the data on the main thread for processing (void)g_idle_add( onGotPostData, task ); } /* handlePost */ @@ -815,6 +814,7 @@ sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) nSent = sendto( storage->socket, msgbuf, len, 0, /* flags */ (struct sockaddr*)&storage->saddr, sizeof(storage->saddr) ); + } #ifdef COMMS_CHECKSUM gchar* sum = g_compute_checksum_for_data( G_CHECKSUM_MD5, msgbuf, len ); XP_LOGF( "%s: sent %d bytes with sum %s", __func__, len, sum ); @@ -822,7 +822,6 @@ sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) #else XP_LOGF( "%s()=>%zd", __func__, nSent ); #endif - } return nSent; } From e3b2a0e4e10c39e6d72e456c0b6ca5792b725cc4 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 28 Oct 2017 15:01:31 -0700 Subject: [PATCH 041/138] fix misbehavior by new in-memory map I don't know why what I was doing didn't work, but removing access via [] and using iterators passes all the asserts I could throw at it. --- xwords4/relay/dbmgr.cpp | 47 ++++++++++++++++++++++++++------------- xwords4/relay/dbmgr.h | 5 +++-- xwords4/relay/xwrelay.cpp | 2 +- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/xwords4/relay/dbmgr.cpp b/xwords4/relay/dbmgr.cpp index 4978f5b9e..87bb669c6 100644 --- a/xwords4/relay/dbmgr.cpp +++ b/xwords4/relay/dbmgr.cpp @@ -91,7 +91,7 @@ DBMgr::AddNew( const char* cookie, const char* connName, CookieID cid, if ( !connName ) connName = ""; MutexLock ml( &m_cidsMutex ); - m_cidsMap[ connName ] = cid; + AddCIDImpl( connName, cid ); QueryBuilder qb; qb.appendQueryf( "INSERT INTO " GAMES_TABLE @@ -144,7 +144,7 @@ DBMgr::FindGameFor( const char* connName, char* cookieBuf, int bufLen, *langP = atoi( PQgetvalue( result, 0, col++ ) ); *isDead = 't' == PQgetvalue( result, 0, col++ )[0]; - *cidp = m_cidsMap[connName]; + *cidp = GetCIDImpl(connName); } PQclear( result ); @@ -172,7 +172,7 @@ DBMgr::FindGame( const char* connName, HostID hid, char* roomBuf, int roomBufLen assert( 1 >= PQntuples( result ) ); if ( 1 == PQntuples( result ) ) { int col = 0; - cid = m_cidsMap[connName]; + cid = GetCIDImpl(connName); snprintf( roomBuf, roomBufLen, "%s", PQgetvalue( result, 0, col++ ) ); *langP = atoi( PQgetvalue( result, 0, col++ ) ); *nPlayersTP = atoi( PQgetvalue( result, 0, col++ ) ); @@ -294,7 +294,7 @@ DBMgr::SeenSeed( const char* cookie, unsigned short seed, if ( found ) { int col = 0; snprintf( connNameBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); - *cid = m_cidsMap[connNameBuf]; + *cid = GetCIDImpl(connNameBuf); const char* seeds = PQgetvalue( result, 0, col++ ); int perDeviceSum = atoi( PQgetvalue( result, 0, col++ ) ); @@ -336,8 +336,8 @@ DBMgr::FindOpen( const char* cookie, int lang, int nPlayersT, int nPlayersH, CookieID cid = 0; if ( 1 == PQntuples( result ) ) { int col = 0; - cid = m_cidsMap[connNameBuf]; snprintf( connNameBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); + cid = GetCIDImpl(connNameBuf); *nPlayersHP = atoi( PQgetvalue( result, 0, col++ ) ); /* cid may be 0, but should use game anyway */ } @@ -641,20 +641,44 @@ DBMgr::HaveDevice( const char* connName, HostID hid, int seed ) return found; } +bool +DBMgr::AddCIDImpl( const char* const connName, CookieID cid ) +{ + logf( XW_LOGINFO, "%s(%s, %d)", __func__, connName, cid ); + assert( cid != 0 ); + assert( m_cidsMap.find(connName) == m_cidsMap.end() ); + m_cidsMap[connName] = cid; + assert( m_cidsMap.find(connName) != m_cidsMap.end() ); + return TRUE; +} + bool DBMgr::AddCID( const char* const connName, CookieID cid ) { MutexLock ml( &m_cidsMutex ); - m_cidsMap[ connName ] = cid; + return AddCIDImpl( connName, cid ); +} - return TRUE; +CookieID +DBMgr::GetCIDImpl( const char* const connName ) +{ + CookieID cid = 0; + map::const_iterator iter = m_cidsMap.find(connName); + if (iter != m_cidsMap.end()) { + cid = iter->second; + } + logf( XW_LOGINFO, "%s(%s) => %d", __func__, connName, cid ); + return cid; } void DBMgr::ClearCID( const char* connName ) { + logf( XW_LOGINFO, "%s(%s)", __func__, connName ); MutexLock ml( &m_cidsMutex ); - m_cidsMap.erase( connName ); + assert( 0 != GetCIDImpl(connName) ); + m_cidsMap.erase( m_cidsMap.find( connName )); + assert( 0 == GetCIDImpl(connName) ); } void @@ -773,13 +797,6 @@ DBMgr::WaitDBConn( void ) logf( XW_LOGERROR, "%s() done", __func__ ); } -void -DBMgr::ClearCIDs( void ) -{ - MutexLock ml( &m_cidsMutex ); - m_cidsMap.clear(); -} - void DBMgr::PublicRooms( int lang, int nPlayers, int* nNames, string& names ) { diff --git a/xwords4/relay/dbmgr.h b/xwords4/relay/dbmgr.h index 5550d90df..643096057 100644 --- a/xwords4/relay/dbmgr.h +++ b/xwords4/relay/dbmgr.h @@ -65,8 +65,6 @@ class DBMgr { void WaitDBConn( void ); - void ClearCIDs( void ); - void AddNew( const char* cookie, const char* connName, CookieID cid, int langCode, int nPlayersT, bool isPublic ); @@ -171,6 +169,9 @@ class DBMgr { int clientVersion, const char* const model, const char* const osVers, DevIDRelay relayID ); + bool AddCIDImpl( const char* const connName, CookieID cid ); + CookieID GetCIDImpl( const char* const connName ); + PGconn* getThreadConn( void ); void clearThreadConn(); diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index bb260dabe..d17da9c06 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -2264,7 +2264,7 @@ main( int argc, char** argv ) exit( 1 ); } - DBMgr::Get()->ClearCIDs(); /* get prev boot's state in db */ + // DBMgr::Get()->ClearCIDs(); /* get prev boot's state in db */ vector::const_iterator iter_game; for ( iter_game = ints_game.begin(); iter_game != ints_game.end(); From 3215affd68e769ccaafe285023ff74810edd37c4 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 28 Oct 2017 15:31:17 -0700 Subject: [PATCH 042/138] tweaks to test script --- xwords4/linux/scripts/start-pair.sh | 58 ++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/xwords4/linux/scripts/start-pair.sh b/xwords4/linux/scripts/start-pair.sh index 68175be81..767169b6c 100755 --- a/xwords4/linux/scripts/start-pair.sh +++ b/xwords4/linux/scripts/start-pair.sh @@ -2,6 +2,42 @@ set -e -u +IN_SEQ='' +HTTP='' + +usage() { + [ $# -gt 0 ] && echo "ERROR: $1" + echo "usage: $0 --in-sequence|--at-once [--use-http]" + cat </dev/null 2>>$LOG & - PIDS="$PIDS $!" + PID=$! + echo "launched $PID" + if [ $IN_SEQ -eq 1 ]; then + sleep 9 + kill $PID + sleep 1 + elif [ $IN_SEQ -eq 0 ]; then + PIDS="$PIDS $PID" + fi done done -echo "launched $PIDS" - -# --use-http \ - -sleep 10 +[ -n "${PIDS}" ] && sleep 10 for PID in $PIDS; do kill $PID done From 7d4fb1cc5d8a7f1f1783d33790f6efbf4b60ba68 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 28 Oct 2017 15:36:33 -0700 Subject: [PATCH 043/138] get rid of last [] from map access --- xwords4/relay/dbmgr.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xwords4/relay/dbmgr.cpp b/xwords4/relay/dbmgr.cpp index 87bb669c6..47f5da3c8 100644 --- a/xwords4/relay/dbmgr.cpp +++ b/xwords4/relay/dbmgr.cpp @@ -647,7 +647,7 @@ DBMgr::AddCIDImpl( const char* const connName, CookieID cid ) logf( XW_LOGINFO, "%s(%s, %d)", __func__, connName, cid ); assert( cid != 0 ); assert( m_cidsMap.find(connName) == m_cidsMap.end() ); - m_cidsMap[connName] = cid; + m_cidsMap.insert( pair(connName, cid) ); assert( m_cidsMap.find(connName) != m_cidsMap.end() ); return TRUE; } @@ -666,6 +666,7 @@ DBMgr::GetCIDImpl( const char* const connName ) map::const_iterator iter = m_cidsMap.find(connName); if (iter != m_cidsMap.end()) { cid = iter->second; + assert( cid != 0 ); } logf( XW_LOGINFO, "%s(%s) => %d", __func__, connName, cid ); return cid; @@ -678,7 +679,7 @@ DBMgr::ClearCID( const char* connName ) MutexLock ml( &m_cidsMutex ); assert( 0 != GetCIDImpl(connName) ); m_cidsMap.erase( m_cidsMap.find( connName )); - assert( 0 == GetCIDImpl(connName) ); + assert( m_cidsMap.find(connName) == m_cidsMap.end() ); } void From 3e9381d9460c5496a2032feab50a7e2a988936a4 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 28 Oct 2017 16:13:11 -0700 Subject: [PATCH 044/138] use a single timer and a queue for received data using g_add_idle() for each piece of data received on the (background) curl-query thread wasn't working. They were getting starved, and I think some were considered duplicates and never scheduled. So add a single timer proc called every 50 ms and a queue that it checks and into which the network thread can put stuff. --- xwords4/linux/linuxutl.c | 2 +- xwords4/linux/relaycon.c | 84 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/xwords4/linux/linuxutl.c b/xwords4/linux/linuxutl.c index dd9e87f9b..fbab06504 100644 --- a/xwords4/linux/linuxutl.c +++ b/xwords4/linux/linuxutl.c @@ -50,7 +50,7 @@ linux_debugf( const char* format, ... ) gettimeofday( &tv, &tz ); timp = localtime( &tv.tv_sec ); - size_t len = snprintf( buf, sizeof(buf), "<%d:%lu>%.2d:%.2d:%.2d:", getpid(), + size_t len = snprintf( buf, sizeof(buf), "<%d:%lx>%.2d:%.2d:%.2d:", getpid(), pthread_self(), timp->tm_hour, timp->tm_min, timp->tm_sec ); XP_ASSERT( len < sizeof(buf) ); diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 555db4120..63e0f911b 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -40,6 +40,9 @@ typedef struct _RelayConStorage { pthread_mutex_t relayMutex; GSList* relayTaskList; + pthread_mutex_t gotDataMutex; + GSList* gotDataTaskList; + int socket; RelayConnProcs procs; void* procsClosure; @@ -48,6 +51,7 @@ typedef struct _RelayConStorage { XWPDevProto proto; LaunchParams* params; XP_UCHAR host[64]; + int nextTaskID; } RelayConStorage; typedef struct _MsgHeader { @@ -63,6 +67,7 @@ static gboolean relaycon_receive( GIOChannel *source, GIOCondition condition, static void schedule_next_check( RelayConStorage* storage ); static void reset_schedule_check_interval( RelayConStorage* storage ); static void checkForMovesOnce( RelayConStorage* storage ); +static gboolean gotDataTimer(gpointer user_data); static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ); static size_t addVLIStr( XP_U8* buf, size_t len, const XP_UCHAR* str ); @@ -91,6 +96,7 @@ typedef enum { POST, QUERY, } TaskType; typedef struct _RelayTask { TaskType typ; + int id; RelayConStorage* storage; WriteState ws; union { @@ -108,7 +114,8 @@ static RelayTask* makeRelayTask(RelayConStorage* storage, TaskType typ); static void freeRelayTask(RelayTask* task); static void handlePost( RelayTask* task ); static void handleQuery( RelayTask* task ); - +static void addToGotData( RelayTask* task ); +static RelayTask* getFromGotData( RelayConStorage* storage ); static size_t write_callback(void *contents, size_t size, size_t nmemb, void* data) @@ -203,12 +210,15 @@ relaycon_init( LaunchParams* params, const RelayConnProcs* procs, if ( params->useHTTP ) { storage->mainThread = pthread_self(); - pthread_mutex_init ( &storage->relayMutex, NULL ); + pthread_mutex_init( &storage->relayMutex, NULL ); pthread_cond_init( &storage->relayCondVar, NULL ); pthread_t thread; (void)pthread_create( &thread, NULL, relayThread, storage ); pthread_detach( thread ); + pthread_mutex_init( &storage->gotDataMutex, NULL ); + g_timeout_add( 50, gotDataTimer, storage ); + XP_ASSERT( XP_STRLEN(host) < VSIZE(storage->host) ); XP_MEMCPY( storage->host, host, XP_STRLEN(host) + 1 ); } else { @@ -421,15 +431,19 @@ addTask( RelayConStorage* storage, RelayTask* task ) static RelayTask* makeRelayTask( RelayConStorage* storage, TaskType typ ) { + XP_ASSERT( onMainThread(storage) ); RelayTask* task = (RelayTask*)g_malloc0(sizeof(*task)); task->typ = typ; + task->id = ++storage->nextTaskID; task->storage = storage; + XP_LOGF( "%s(): made with id %d from storage %p", __func__, task->id, storage ); return task; } static void freeRelayTask( RelayTask* task ) { + XP_LOGF( "%s(): deleting id %d", __func__, task->id ); g_free( task->ws.ptr ); g_free( task ); } @@ -657,7 +671,7 @@ onGotPostData(gpointer user_data) static void handlePost( RelayTask* task ) { - XP_LOGF( "%s(len=%d)", __func__, task->u.post.len ); + XP_LOGF( "%s(task.post.len=%d)", __func__, task->u.post.len ); XP_ASSERT( !onMainThread(task->storage) ); char* data = g_base64_encode( task->u.post.msgbuf, task->u.post.len ); struct json_object* jobj = json_object_new_object(); @@ -668,7 +682,7 @@ handlePost( RelayTask* task ) runWitCurl( task, "post", "params", jobj ); // Put the data on the main thread for processing - (void)g_idle_add( onGotPostData, task ); + addToGotData( task ); } /* handlePost */ static ssize_t @@ -750,7 +764,7 @@ handleQuery( RelayTask* task ) runWitCurl( task, "query", "ids", jIds ); } /* Put processing back on the main thread */ - g_idle_add( onGotQueryData, task ); + addToGotData( task ); } /* handleQuery */ static void @@ -774,6 +788,66 @@ checkForMoves( gpointer user_data ) return FALSE; } +static gboolean +gotDataTimer(gpointer user_data) +{ + RelayConStorage* storage = (RelayConStorage*)user_data; + assert( onMainThread(storage) ); + + for ( ; ; ) { + RelayTask* task = getFromGotData( storage ); + + if ( !task ) { + break; + } else { + switch ( task->typ ) { + case POST: + onGotPostData( task ); + break; + case QUERY: + onGotQueryData( task ); + break; + default: + XP_ASSERT(0); + } + } + } + return TRUE; +} + +static void +addToGotData( RelayTask* task ) +{ + RelayConStorage* storage = task->storage; + pthread_mutex_lock( &storage->gotDataMutex ); + storage->gotDataTaskList = g_slist_append( storage->gotDataTaskList, task ); + XP_LOGF( "%s(): added id %d; len now %d", __func__, task->id, + g_slist_length(storage->gotDataTaskList) ); + pthread_mutex_unlock( &storage->gotDataMutex ); +} + +static RelayTask* +getFromGotData( RelayConStorage* storage ) +{ + RelayTask* task = NULL; + XP_ASSERT( onMainThread(storage) ); + pthread_mutex_lock( &storage->gotDataMutex ); + int len = g_slist_length( storage->gotDataTaskList ); + // XP_LOGF( "%s(): before: len: %d", __func__, len ); + if ( len > 0 ) { + GSList* head = storage->gotDataTaskList; + storage->gotDataTaskList + = g_slist_remove_link( storage->gotDataTaskList, + storage->gotDataTaskList ); + task = head->data; + g_slist_free( head ); + XP_LOGF( "%s(): got id %d!", __func__, task->id ); + } + // XP_LOGF( "%s(): len now %d", __func__, g_slist_length(storage->gotDataTaskList) ); + pthread_mutex_unlock( &storage->gotDataMutex ); + return task; +} + static void reset_schedule_check_interval( RelayConStorage* storage ) { From 7c22d1fdf8218fcb1987c948d04d72a9f7b20558 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 28 Oct 2017 16:29:04 -0700 Subject: [PATCH 045/138] fix failure of http apps to ack relay Change how acks are handled by adding ability to look up connname by combination of hid and token. It's a bit of a hack, but it's already there in the protocol and enough to find the game. --- xwords4/relay/crefmgr.cpp | 55 +++++++++++++++++++++++++++++++++++++++ xwords4/relay/crefmgr.h | 2 ++ xwords4/relay/dbmgr.cpp | 35 +++++++++++++++++++++++++ xwords4/relay/dbmgr.h | 4 +++ xwords4/relay/xwrelay.cpp | 17 +++++++----- 5 files changed, 107 insertions(+), 6 deletions(-) diff --git a/xwords4/relay/crefmgr.cpp b/xwords4/relay/crefmgr.cpp index 43d09b618..882f0eb27 100644 --- a/xwords4/relay/crefmgr.cpp +++ b/xwords4/relay/crefmgr.cpp @@ -375,6 +375,48 @@ CRefMgr::getMakeCookieRef( const char* const connName, HostID hid, bool* isDead return cinfo; } +CidInfo* +CRefMgr::getMakeCookieRef( const AddrInfo::ClientToken clientToken, HostID srcID ) +{ + CookieRef* cref = NULL; + CidInfo* cinfo = NULL; + char curCookie[MAX_INVITE_LEN+1]; + int curLangCode; + int nPlayersT = 0; + int nAlreadyHere = 0; + + for ( ; ; ) { /* for: see comment above */ + char connName[MAX_CONNNAME_LEN+1] = {0}; + CookieID cid = m_db->FindGame( clientToken, srcID, + connName, sizeof(connName), + curCookie, sizeof(curCookie), + &curLangCode, &nPlayersT, &nAlreadyHere ); + // &seed ); + if ( 0 != cid ) { /* already open */ + cinfo = m_cidlock->Claim( cid ); + if ( NULL == cinfo->GetRef() ) { + m_cidlock->Relinquish( cinfo, true ); + continue; + } + } else if ( nPlayersT == 0 ) { /* wasn't in the DB */ + /* do nothing; insufficient info to fake it */ + } else { + cinfo = m_cidlock->Claim(); + if ( !m_db->AddCID( connName, cinfo->GetCid() ) ) { + m_cidlock->Relinquish( cinfo, true ); + continue; + } + logf( XW_LOGINFO, "%s(): added cid???", __func__ ); + cref = AddNew( curCookie, connName, cinfo->GetCid(), curLangCode, + nPlayersT, nAlreadyHere ); + cinfo->SetRef( cref ); + } + break; + } + logf( XW_LOGINFO, "%s() => %p", __func__, cinfo ); + return cinfo; +} + void CRefMgr::RemoveSocketRefs( const AddrInfo* addr ) { @@ -722,6 +764,19 @@ SafeCref::SafeCref( const AddrInfo* addr ) } } +SafeCref::SafeCref( const AddrInfo::ClientToken clientToken, HostID srcID ) + : m_cinfo( NULL ) + , m_mgr( CRefMgr::Get() ) + , m_isValid( false ) +{ + CidInfo* cinfo = m_mgr->getMakeCookieRef( clientToken, srcID ); + if ( NULL != cinfo && NULL != cinfo->GetRef() ) { + m_locked = cinfo->GetRef()->Lock(); + m_cinfo = cinfo; + m_isValid = true; + } +} + SafeCref::~SafeCref() { if ( m_cinfo != NULL ) { diff --git a/xwords4/relay/crefmgr.h b/xwords4/relay/crefmgr.h index 809dea796..7694759dd 100644 --- a/xwords4/relay/crefmgr.h +++ b/xwords4/relay/crefmgr.h @@ -129,6 +129,7 @@ class CRefMgr { bool isPublic, bool* isDead ); CidInfo* getMakeCookieRef( const char* const connName, HostID hid, bool* isDead ); + CidInfo* getMakeCookieRef( const AddrInfo::ClientToken clientToken, HostID srcID ); CidInfo* getCookieRef( CookieID cid, bool failOk = false ); CidInfo* getCookieRef( const AddrInfo* addr ); @@ -182,6 +183,7 @@ class SafeCref { SafeCref( const char* const connName, HostID hid ); SafeCref( CookieID cid, bool failOk = false ); SafeCref( const AddrInfo* addr ); + SafeCref( const AddrInfo::ClientToken clientToken, HostID srcID ); /* SafeCref( CookieRef* cref ); */ ~SafeCref(); diff --git a/xwords4/relay/dbmgr.cpp b/xwords4/relay/dbmgr.cpp index 47f5da3c8..eb55898cd 100644 --- a/xwords4/relay/dbmgr.cpp +++ b/xwords4/relay/dbmgr.cpp @@ -185,6 +185,41 @@ DBMgr::FindGame( const char* connName, HostID hid, char* roomBuf, int roomBufLen return cid; } /* FindGame */ +CookieID +DBMgr::FindGame( const AddrInfo::ClientToken clientToken, HostID hid, + char* connNameBuf, int connNameBufLen, + char* roomBuf, int roomBufLen, + int* langP, int* nPlayersTP, int* nPlayersHP ) +{ + CookieID cid = 0; + const char* fmt = "SELECT room, lang, nTotal, nPerDevice[%d], connname FROM " + GAMES_TABLE " WHERE tokens[%d] = %d and NOT dead"; + // " LIMIT 1" + ; + StrWPF query; + query.catf( fmt, hid, hid, clientToken ); + logf( XW_LOGINFO, "query: %s", query.c_str() ); + + PGresult* result = PQexec( getThreadConn(), query.c_str() ); + assert( 1 >= PQntuples( result ) ); + if ( 1 == PQntuples( result ) ) { + int col = 0; + // room + snprintf( roomBuf, roomBufLen, "%s", PQgetvalue( result, 0, col++ ) ); + // lang + *langP = atoi( PQgetvalue( result, 0, col++ ) ); + *nPlayersTP = atoi( PQgetvalue( result, 0, col++ ) ); + *nPlayersHP = atoi( PQgetvalue( result, 0, col++ ) ); + snprintf( connNameBuf, connNameBufLen, "%s", PQgetvalue( result, 0, col++ ) ); + cid = GetCIDImpl(connNameBuf); + } + PQclear( result ); + + logf( XW_LOGINFO, "%s(ct=%d,hid=%d) => %d (connname=%s)", __func__, clientToken, + hid, cid, connNameBuf ); + return cid; +} + bool DBMgr::FindPlayer( DevIDRelay relayID, AddrInfo::ClientToken token, string& connName, HostID* hidp, unsigned short* seed ) diff --git a/xwords4/relay/dbmgr.h b/xwords4/relay/dbmgr.h index 643096057..1185b7d7c 100644 --- a/xwords4/relay/dbmgr.h +++ b/xwords4/relay/dbmgr.h @@ -76,6 +76,10 @@ class DBMgr { CookieID FindGame( const char* connName, HostID hid, char* cookieBuf, int bufLen, int* langP, int* nPlayersTP, int* nPlayersHP, bool* isDead ); + CookieID FindGame( const AddrInfo::ClientToken clientToken, HostID hid, + char* connNameBuf, int connNameBufLen, + char* cookieBuf, int cookieBufLen, + int* langP, int* nPlayersTP, int* nPlayersHP ); bool FindGameFor( const char* connName, char* cookieBuf, int bufLen, unsigned short seed, HostID hid, diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index d17da9c06..2bfcfc017 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -996,13 +996,13 @@ processReconnect( const uint8_t* bufp, int bufLen, const AddrInfo* addr ) } /* processReconnect */ static bool -processAck( const uint8_t* bufp, int bufLen, const AddrInfo* addr ) +processAck( const uint8_t* bufp, int bufLen, AddrInfo::ClientToken clientToken ) { bool success = false; const uint8_t* end = bufp + bufLen; HostID srcID; if ( getNetByte( &bufp, end, &srcID ) ) { - SafeCref scr( addr ); + SafeCref scr( clientToken, srcID ); success = scr.HandleAck( srcID ); } return success; @@ -1092,7 +1092,8 @@ forwardMessage( const uint8_t* buf, int buflen, const AddrInfo* addr ) } /* forwardMessage */ static bool -processMessage( const uint8_t* buf, int bufLen, const AddrInfo* addr ) +processMessage( const uint8_t* buf, int bufLen, const AddrInfo* addr, + AddrInfo::ClientToken clientToken ) { bool success = false; /* default is failure */ XWRELAY_Cmd cmd = *buf; @@ -1107,7 +1108,11 @@ processMessage( const uint8_t* buf, int bufLen, const AddrInfo* addr ) success = processReconnect( buf+1, bufLen-1, addr ); break; case XWRELAY_ACK: - success = processAck( buf+1, bufLen-1, addr ); + if ( clientToken != 0 ) { + success = processAck( buf+1, bufLen-1, clientToken ); + } else { + logf( XW_LOGERROR, "%s(): null client token", __func__ ); + } break; case XWRELAY_GAME_DISCONNECT: success = processDisconnect( buf+1, bufLen-1, addr ); @@ -1468,7 +1473,7 @@ handleProxyMsgs( int sock, const AddrInfo* addr, const uint8_t* bufp, static void game_thread_proc( UdpThreadClosure* utc ) { - if ( !processMessage( utc->buf(), utc->len(), utc->addr() ) ) { + if ( !processMessage( utc->buf(), utc->len(), utc->addr(), 0 ) ) { XWThreadPool::GetTPool()->CloseSocket( utc->addr() ); } } @@ -1756,7 +1761,7 @@ handle_udp_packet( UdpThreadClosure* utc ) clientToken = ntohl( clientToken ); if ( AddrInfo::NULL_TOKEN != clientToken ) { AddrInfo addr( g_udpsock, clientToken, utc->saddr() ); - (void)processMessage( ptr, end - ptr, &addr ); + (void)processMessage( ptr, end - ptr, &addr, clientToken ); } else { logf( XW_LOGERROR, "%s: dropping packet with token of 0", __func__ ); From 7b50c90aacf41c4a1349812192fbcc2e93eb0e04 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 28 Oct 2017 20:12:05 -0700 Subject: [PATCH 046/138] pass timeoutSeconds ACK doesn't need to wait 2 seconds for a reply, and when it does so the next send waits too. Eventually we'll want to combine messages already in the queue into a single send. For now, this makes things better. --- xwords4/android/scripts/relay.py | 8 ++-- xwords4/linux/relaycon.c | 79 +++++++++++++++++++------------- 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index c8e6d3ef9..3f7aebf4b 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -11,14 +11,14 @@ try: except ImportError: apacheAvailable = False -def post(req, params, timeoutSecs = 1): +def post(req, params, timeoutSecs = 1.0): err = 'none' dataLen = 0 jobj = json.loads(params) data = base64.b64decode(jobj['data']) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(timeoutSecs) # seconds + sock.settimeout(float(timeoutSecs)) # seconds addr = ("127.0.0.1", 10997) sock.sendto(data, addr) @@ -35,7 +35,7 @@ def post(req, params, timeoutSecs = 1): jobj = {'err' : err, 'data' : responses} return json.dumps(jobj) -def query(req, ids): +def query(req, ids, timeoutSecs = 5.0): print('ids', ids) ids = json.loads(ids) @@ -43,7 +43,7 @@ def query(req, ids): for id in ids: idsLen += len(id) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(5) # seconds + sock.settimeout(float(timeoutSecs)) sock.connect(('127.0.0.1', 10998)) lenShort = 2 + idsLen + len(ids) + 2 diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 63e0f911b..37f53cba9 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -69,7 +69,7 @@ static void reset_schedule_check_interval( RelayConStorage* storage ); static void checkForMovesOnce( RelayConStorage* storage ); static gboolean gotDataTimer(gpointer user_data); -static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ); +static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len, XP_U16 timeout ); static size_t addVLIStr( XP_U8* buf, size_t len, const XP_UCHAR* str ); static void getNetString( const XP_U8** ptr, XP_U16 len, XP_UCHAR* buf ); static XP_U16 getNetShort( const XP_U8** ptr ); @@ -103,6 +103,7 @@ typedef struct _RelayTask { struct { XP_U8* msgbuf; XP_U16 len; + float timeoutSecs; } post; struct { GHashTable* map; @@ -110,7 +111,7 @@ typedef struct _RelayTask { } u; } RelayTask; -static RelayTask* makeRelayTask(RelayConStorage* storage, TaskType typ); +static RelayTask* makeRelayTask( RelayConStorage* storage, TaskType typ ); static void freeRelayTask(RelayTask* task); static void handlePost( RelayTask* task ); static void handleQuery( RelayTask* task ); @@ -139,26 +140,44 @@ write_callback(void *contents, size_t size, size_t nmemb, void* data) } static void -addJsonParams( CURL* curl, const char* name, json_object* param ) +addJsonParams( CURL* curl, va_list ap ) { - const char* asStr = json_object_to_json_string( param ); - XP_LOGF( "%s: adding param (with name %s): %s", __func__, name, asStr ); + gchar* buf = NULL; + for ( ; ; ) { + const char* name = va_arg(ap, const char*); + if ( !name ) { + break; + } + json_object* param = va_arg(ap, json_object*); + XP_ASSERT( !!param ); - char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); - gchar* buf = g_strdup_printf( "%s=%s", name, curl_params ); - curl_free( curl_params ); + const char* asStr = json_object_to_json_string( param ); + XP_LOGF( "%s: adding param (with name %s): %s", __func__, name, asStr ); + char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); + gchar* tmp = g_strdup_printf( "%s=%s", name, curl_params ); + curl_free( curl_params ); + + if ( !buf ) { + buf = tmp; + } else { + gchar* cur = buf; + buf = g_strdup_printf( "%s&%s", cur, tmp ); + g_free( tmp ); + g_free( cur ); + } + json_object_put( param ); + } + XP_LOGF( "%s(): setting params: %s", __func__, buf ); curl_easy_setopt( curl, CURLOPT_POSTFIELDS, buf ); curl_easy_setopt( curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf) ); // Can't free the buf!! Well, maybe after the send... // g_free( buf ); - json_object_put( param ); } static XP_Bool -runWitCurl( RelayTask* task, const gchar* proc, const gchar* key, - json_object* jVal ) +runWitCurl( RelayTask* task, const gchar* proc, ...) { CURLcode res = curl_global_init(CURL_GLOBAL_DEFAULT); XP_ASSERT(res == CURLE_OK); @@ -170,7 +189,10 @@ runWitCurl( RelayTask* task, const gchar* proc, const gchar* key, curl_easy_setopt( curl, CURLOPT_URL, url ); curl_easy_setopt( curl, CURLOPT_POST, 1L ); - addJsonParams( curl, key, jVal ); + va_list ap; + va_start( ap, proc ); + addJsonParams( curl, ap ); + va_end( ap ); curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); curl_easy_setopt( curl, CURLOPT_WRITEDATA, &task->ws ); @@ -266,7 +288,7 @@ relaycon_reg( LaunchParams* params, const XP_UCHAR* rDevID, indx += addVLIStr( &tmpbuf[indx], sizeof(tmpbuf) - indx, "linux box" ); indx += addVLIStr( &tmpbuf[indx], sizeof(tmpbuf) - indx, "linux version" ); - sendIt( storage, tmpbuf, indx ); + sendIt( storage, tmpbuf, indx, 2.0 ); } void @@ -303,7 +325,7 @@ relaycon_invite( LaunchParams* params, XP_U32 destDevID, indx += writeBytes( &tmpbuf[indx], sizeof(tmpbuf) - indx, ptr, len ); stream_destroy( stream ); - sendIt( storage, tmpbuf, indx ); + sendIt( storage, tmpbuf, indx, 2.0 ); LOG_RETURN_VOID(); } @@ -320,7 +342,7 @@ relaycon_send( LaunchParams* params, const XP_U8* buf, XP_U16 buflen, indx += writeHeader( storage, tmpbuf, XWPDEV_MSG ); indx += writeLong( &tmpbuf[indx], sizeof(tmpbuf) - indx, gameToken ); indx += writeBytes( &tmpbuf[indx], sizeof(tmpbuf) - indx, buf, buflen ); - nSent = sendIt( storage, tmpbuf, indx ); + nSent = sendIt( storage, tmpbuf, indx, 2.0 ); if ( nSent > buflen ) { nSent = buflen; } @@ -348,7 +370,7 @@ relaycon_sendnoconn( LaunchParams* params, const XP_U8* buf, XP_U16 buflen, (const XP_U8*)relayID, idLen ); tmpbuf[indx++] = '\n'; indx += writeBytes( &tmpbuf[indx], sizeof(tmpbuf) - indx, buf, buflen ); - nSent = sendIt( storage, tmpbuf, indx ); + nSent = sendIt( storage, tmpbuf, indx, 2.0 ); if ( nSent > buflen ) { nSent = buflen; } @@ -367,7 +389,7 @@ relaycon_requestMsgs( LaunchParams* params, const XP_UCHAR* devID ) indx += writeHeader( storage, tmpbuf, XWPDEV_RQSTMSGS ); indx += addVLIStr( &tmpbuf[indx], sizeof(tmpbuf) - indx, devID ); - sendIt( storage, tmpbuf, indx ); + sendIt( storage, tmpbuf, indx, 2.0 ); } void @@ -382,7 +404,7 @@ relaycon_deleted( LaunchParams* params, const XP_UCHAR* devID, indx += writeDevID( &tmpbuf[indx], sizeof(tmpbuf) - indx, devID ); indx += writeLong( &tmpbuf[indx], sizeof(tmpbuf) - indx, gameToken ); - sendIt( storage, tmpbuf, indx ); + sendIt( storage, tmpbuf, indx, 0.0 ); } static XP_Bool @@ -455,7 +477,7 @@ sendAckIf( RelayConStorage* storage, const MsgHeader* header ) XP_U8 tmpbuf[16]; int indx = writeHeader( storage, tmpbuf, XWPDEV_ACK ); indx += writeVLI( &tmpbuf[indx], header->packetID ); - sendIt( storage, tmpbuf, indx ); + sendIt( storage, tmpbuf, indx, 0.0 ); } } @@ -630,13 +652,6 @@ hostNameToIP( const XP_UCHAR* name ) return ip; } -typedef struct _PostArgs { - RelayConStorage* storage; - WriteState ws; - XP_U8* msgbuf; - XP_U16 len; -} PostArgs; - static gboolean onGotPostData(gpointer user_data) { @@ -679,18 +694,20 @@ handlePost( RelayTask* task ) g_free( data ); json_object_object_add( jobj, "data", jstr ); - runWitCurl( task, "post", "params", jobj ); + json_object* jTimeout = json_object_new_double( task->u.post.timeoutSecs ); + runWitCurl( task, "post", "params", jobj, "timeoutSecs", jTimeout, NULL ); // Put the data on the main thread for processing addToGotData( task ); } /* handlePost */ static ssize_t -post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) +post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len, float timeout ) { XP_LOGF( "%s(len=%d)", __func__, len ); RelayTask* task = makeRelayTask( storage, POST ); task->u.post.msgbuf = g_malloc(len); + task->u.post.timeoutSecs = timeout; XP_MEMCPY( task->u.post.msgbuf, msgbuf, len ); task->u.post.len = len; addTask( storage, task ); @@ -761,7 +778,7 @@ handleQuery( RelayTask* task ) } g_list_free( ids ); - runWitCurl( task, "query", "ids", jIds ); + runWitCurl( task, "query", "ids", jIds, NULL ); } /* Put processing back on the main thread */ addToGotData( task ); @@ -879,11 +896,11 @@ schedule_next_check( RelayConStorage* storage ) } static ssize_t -sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len ) +sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len, XP_U16 timeout ) { ssize_t nSent; if ( storage->params->useHTTP ) { - nSent = post( storage, msgbuf, len ); + nSent = post( storage, msgbuf, len, timeout ); } else { nSent = sendto( storage->socket, msgbuf, len, 0, /* flags */ (struct sockaddr*)&storage->saddr, From 1373d0b1db1ce09260f49d5bbbb8e76e49c3291a Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 29 Oct 2017 09:26:07 -0700 Subject: [PATCH 047/138] make min-run configurable With the new http stuff, at least for now, it takes longer to get things communicated and so killing games after 2 seconds of runtime meant no moves ever got made. Making it configurable, and passing 10 (seconds) means nearly all games in a large test run complete reasonably quickly. --- xwords4/linux/scripts/discon_ok2.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index 9929da583..cb3a90787 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -512,7 +512,8 @@ run_cmds() { local KEYS=( ${!ARGS[*]} ) KEY=${KEYS[$INDX]} ROOM=${ROOMS[$KEY]} - if [ 0 -eq ${PIDS[$KEY]} ]; then + PID=${PIDS[$KEY]} + if [ 0 -eq ${PID} ]; then if [ -n "$ONE_PER_ROOM" -a 0 -ne ${ROOM_PIDS[$ROOM]} ]; then continue fi @@ -526,10 +527,12 @@ run_cmds() { ROOM_PIDS[$ROOM]=$PID MINEND[$KEY]=$(($NOW + $MINRUN)) else - PID=${PIDS[$KEY]} if [ -d /proc/$PID ]; then SLEEP=$((${MINEND[$KEY]} - $NOW)) - [ $SLEEP -gt 0 ] && sleep $SLEEP + if [ $SLEEP -gt 0 ]; then + sleep 1 + continue + fi kill $PID || /bin/true wait $PID fi @@ -606,6 +609,7 @@ function usage() { echo " [--host ] \\" >&2 echo " [--max-devs ] \\" >&2 echo " [--min-devs ] \\" >&2 + echo " [--min-run ] # run each at least this long \\" >&2 echo " [--new-app &2 echo " [--new-app-args [arg*]] # passed only to new app \\" >&2 echo " [--num-games ] \\" >&2 @@ -683,6 +687,11 @@ while [ "$#" -gt 0 ]; do MAXDEVS=$(getArg $*) shift ;; + --min-run) + MINRUN=$(getArg $*) + [ $MINRUN -ge 2 -a $MINRUN -le 60 ] || usage "$1: n must be 2 <= n <= 60" + shift + ;; --one-per) ONEPER=TRUE ;; From 3d3a986dbd05bf58dcb0ea71974c542c3e70f203 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 29 Oct 2017 09:46:51 -0700 Subject: [PATCH 048/138] log count of tasks abandoned in queue on shutdown This confirms that I need to process outgoing messages more quickly, likely by combining them. --- xwords4/linux/relaycon.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 37f53cba9..79243b8f9 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -619,6 +619,19 @@ relaycon_receive( GIOChannel* source, GIOCondition XP_UNUSED_DBG(condition), gpo void relaycon_cleanup( LaunchParams* params ) { + RelayConStorage* storage = (RelayConStorage*)params->relayConStorage; + if ( storage->params->useHTTP ) { + pthread_mutex_lock( &storage->relayMutex ); + int nRelayTasks = g_slist_length( storage->relayTaskList ); + pthread_mutex_unlock( &storage->relayMutex ); + + pthread_mutex_lock( &storage->gotDataMutex ); + int nDataTasks = g_slist_length( storage->gotDataTaskList ); + pthread_mutex_unlock( &storage->gotDataMutex ); + + XP_LOGF( "%s(): sends pending: %d; data tasks pending: %d", __func__, + nRelayTasks, nDataTasks ); + } XP_FREEP( params->mpool, ¶ms->relayConStorage ); } From 8ee17493ac3a8c9d5b25ac38b40e8867e4ff1f2d Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 30 Oct 2017 07:12:34 -0700 Subject: [PATCH 049/138] log length of task lifetime Need to figure out where the delays are. --- xwords4/linux/relaycon.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 79243b8f9..7b172bcf8 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -99,6 +99,7 @@ typedef struct _RelayTask { int id; RelayConStorage* storage; WriteState ws; + XP_U32 ctime; union { struct { XP_U8* msgbuf; @@ -423,10 +424,18 @@ relayThread( void* arg ) pthread_cond_wait( &storage->relayCondVar, &storage->relayMutex ); } - RelayTask* task = storage->relayTaskList->data; - storage->relayTaskList = storage->relayTaskList->next; + int len = g_slist_length( storage->relayTaskList ); + GSList* head = storage->relayTaskList; + storage->relayTaskList = g_slist_remove_link( storage->relayTaskList, + storage->relayTaskList ); + RelayTask* task = head->data; + g_slist_free( head ); + pthread_mutex_unlock( &storage->relayMutex ); + XP_LOGF( "%s(): processing one of %d; created %d secs ago", + __func__, len, ((XP_U32)time(NULL)) - task->ctime ); + switch ( task->typ ) { case POST: handlePost( task ); @@ -457,6 +466,7 @@ makeRelayTask( RelayConStorage* storage, TaskType typ ) RelayTask* task = (RelayTask*)g_malloc0(sizeof(*task)); task->typ = typ; task->id = ++storage->nextTaskID; + task->ctime = (XP_U32)time(NULL); task->storage = storage; XP_LOGF( "%s(): made with id %d from storage %p", __func__, task->id, storage ); return task; @@ -465,7 +475,8 @@ makeRelayTask( RelayConStorage* storage, TaskType typ ) static void freeRelayTask( RelayTask* task ) { - XP_LOGF( "%s(): deleting id %d", __func__, task->id ); + XP_LOGF( "%s(): deleting id %d (%d secs old)", __func__, task->id, + ((XP_U32)time(NULL)) - task->ctime ); g_free( task->ws.ptr ); g_free( task ); } @@ -666,22 +677,23 @@ hostNameToIP( const XP_UCHAR* name ) } static gboolean -onGotPostData(gpointer user_data) +onGotPostData( gpointer user_data ) { RelayTask* task = (RelayTask*)user_data; + RelayConStorage* storage = task->storage; /* Now pull any data from the reply */ // got "{"status": "ok", "dataLen": 14, "data": "AYQDiDAyMUEzQ0MyADw=", "err": "none"}" if ( !!task->ws.ptr ) { json_object* reply = json_tokener_parse( task->ws.ptr ); json_object* replyData; if ( json_object_object_get_ex( reply, "data", &replyData ) && !!replyData ) { - int len = json_object_array_length(replyData); + const int len = json_object_array_length(replyData); for ( int ii = 0; ii < len; ++ii ) { json_object* datum = json_object_array_get_idx( replyData, ii ); const char* str = json_object_get_string( datum ); gsize out_len; guchar* buf = g_base64_decode( (const gchar*)str, &out_len ); - process( task->storage, buf, out_len ); + process( storage, buf, out_len ); g_free( buf ); } (void)json_object_put( replyData ); From d2897d6dc8fc06d9d7a8258a6487dd0f077dff99 Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 30 Oct 2017 19:06:38 -0700 Subject: [PATCH 050/138] fix to match format of empty return case --- xwords4/android/scripts/relay.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 3f7aebf4b..3de8fedb6 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -54,14 +54,16 @@ def query(req, ids, timeoutSecs = 5.0): for id in ids: sock.send(id + '\n') - unpacker = struct.Struct('!2H') # 2s f') - resLen, nameCount = unpacker.unpack(sock.recv(unpacker.size)) # problem when ids empty + shortUnpacker = struct.Struct('!H') + resLen, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) + nameCount, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) + resLen -= shortUnpacker.size print('resLen:', resLen, 'nameCount:', nameCount) msgsLists = {} - if nameCount == len(ids): + if nameCount == len(ids) and resLen > 0: + print('nameCount', nameCount) for ii in range(nameCount): perGame = [] - shortUnpacker = struct.Struct('!H') countsThisGame, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) # problem print('countsThisGame:', countsThisGame) for jj in range(countsThisGame): From 23f0d54f638a64a9b3f0de914bb1e8c2df86bfc4 Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 30 Oct 2017 19:07:13 -0700 Subject: [PATCH 051/138] tweaks: add comment; show only pending messages --- xwords4/relay/scripts/showinplay.sh | 3 ++- xwords4/relay/xwrelay.cpp | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/xwords4/relay/scripts/showinplay.sh b/xwords4/relay/scripts/showinplay.sh index 24e906ada..a1cc65a6d 100755 --- a/xwords4/relay/scripts/showinplay.sh +++ b/xwords4/relay/scripts/showinplay.sh @@ -59,8 +59,9 @@ echo "SELECT dead as d,connname,room,lang as lg,clntVers as cv ,ntotal as t,nper | psql xwgames # Messages +echo "Unack'd msgs count:" $(psql -t xwgames -c "select count(*) FROM msgs where stime = 'epoch' AND connname IN (SELECT connname from games $QUERY);") echo "SELECT id,connName,hid as h,token,ctime,stime,devid,msg64 "\ - "FROM msgs WHERE connname IN (SELECT connname from games $QUERY) "\ + "FROM msgs WHERE stime = 'epoch' AND connname IN (SELECT connname from games $QUERY) "\ "ORDER BY ctime DESC, connname LIMIT $LIMIT;" \ | psql xwgames diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index 2bfcfc017..eaa407695 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -1347,6 +1347,9 @@ handleMsgsMsg( const AddrInfo* addr, bool sendFull, logf( XW_LOGVERBOSE0, "%s: wrote %d bytes", __func__, nwritten ); if ( sendFull && nwritten >= 0 && (size_t)nwritten == out.size() ) { dbmgr->RecordSent( &msgIDs[0], msgIDs.size() ); + // This is wrong: should be removed when ACK returns and not + // before. But for some reason if I make that change apps wind up + // stalling. dbmgr->RemoveStoredMessages( msgIDs ); } } From 8aeba861cf7a7fc1f4f0f07506f13e7a1294a5dd Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 31 Oct 2017 06:38:32 -0700 Subject: [PATCH 052/138] fix script to move db files again --- xwords4/linux/scripts/discon_ok2.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index cb3a90787..a124a2cd7 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -195,9 +195,6 @@ build_cmds() { for NLOCALS in ${LOCALS[@]}; do DEV=$((DEV + 1)) FILE="${LOGDIR}/GAME_${GAME}_${DEV}.sql3" - if [ $((RANDOM % 100)) -lt $UDP_PCT_START ]; then - FILE="$FILE --use-udp" - fi LOG=${LOGDIR}/${GAME}_${DEV}_LOG.txt > $LOG # clear the log @@ -220,6 +217,9 @@ build_cmds() { PARAMS="$PARAMS --game-dict $DICT --relay-port $PORT --host $HOST " PARAMS="$PARAMS --slow-robot 1:3 --skip-confirm" PARAMS="$PARAMS --db $FILE" + if [ $((RANDOM % 100)) -lt $UDP_PCT_START ]; then + PARAMS="$PARAMS --use-udp" + fi PARAMS="$PARAMS --drop-nth-packet $DROP_N $PLAT_PARMS" if [ $((${RANDOM}%100)) -lt $HTTP_PCT ]; then PARAMS="$PARAMS --use-http" From a1433e5f3ddafa9d56716b59114841342e06d20b Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 31 Oct 2017 20:05:07 -0700 Subject: [PATCH 053/138] add kill to relay.py; call from test script using curl It'll eventually get called from Android and maybe linux, but for now it's cool to have games disappear from the showinplay display when they're done. --- xwords4/android/scripts/relay.py | 60 ++++++++++++++++++++++++----- xwords4/linux/scripts/discon_ok2.sh | 16 ++++++++ 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 3de8fedb6..371f884d6 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -3,14 +3,40 @@ import base64, json, mod_python, socket, struct, sys PROTOCOL_VERSION = 0 +PRX_DEVICE_GONE = 3 PRX_GET_MSGS = 4 -try: - from mod_python import apache - apacheAvailable = True -except ImportError: - apacheAvailable = False +# try: +# from mod_python import apache +# apacheAvailable = True +# except ImportError: +# apacheAvailable = False +def kill(req, params): + print(params) + params = json.loads(params) + count = len(params) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('127.0.0.1', 10998)) + + header = struct.Struct('!BBh') + strLens = 0 + for ii in range(count): + strLens += len(params[ii]['relayID']) + 1 + size = header.size + (2*count) + strLens + sock.send(struct.Struct('!h').pack(size)) + sock.send(header.pack(PROTOCOL_VERSION, PRX_DEVICE_GONE, count)) + + for ii in range(count): + elem = params[ii] + asBytes = bytes(elem['relayID']) + sock.send(struct.Struct('!H%dsc' % (len(asBytes))).pack(elem['seed'], asBytes, '\n')) + sock.close() + + result = {'err': 0} + return json.dumps(result) + def post(req, params, timeoutSecs = 1.0): err = 'none' dataLen = 0 @@ -81,10 +107,26 @@ def query(req, ids, timeoutSecs = 5.0): return json.dumps(msgsLists) def main(): - print(query(None, json.dumps(sys.argv[1:]))) - # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } - # params = json.dumps(params) - # print(post(None, params)) + result = None + if len(sys.argv) > 1: + cmd = sys.argv[1] + args = sys.argv[2:] + if cmd == 'query': + result = query(None, json.dumps(args)) + elif cmd == 'post': + # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } + # params = json.dumps(params) + # print(post(None, params)) + None + elif cmd == 'kill': + result = kill( None, json.dumps([{'relayID': args[0], 'seed':int(args[1])}]) ) + + if result: + print '->', result + else: + print 'USAGE: query [connname/hid]*' + # print ' post ' + print ' kill ' ############################################################################## if __name__ == '__main__': diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index a124a2cd7..24824b895 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -308,6 +308,21 @@ launch() { # exec $CMD >/dev/null 2>>$LOG # } +send_dead() { + ID=$1 + DB=${FILES[$ID]} + while :; do + [ -f $DB ] || break # it's gone + RES=$(echo 'select relayid, seed from games limit 1;' | sqlite3 -separator ' ' $DB || /bin/true) + [ -n "$RES" ] && break + sleep 0.2 + done + RELAYID=$(echo $RES | awk '{print $1}') + SEED=$(echo $RES | awk '{print $2}') + JSON="[{\"relayID\":\"$RELAYID\", \"seed\":$SEED}]" + curl -G --data-urlencode params="$JSON" http://$HOST/xw4/relay.py/kill >/dev/null 2>&1 +} + close_device() { ID=$1 MVTO=$2 @@ -423,6 +438,7 @@ check_game() { for ID in $OTHERS $KEY; do echo -n "${ID}:${LOGS[$ID]}, " kill_from_log ${LOGS[$ID]} || /bin/true + send_dead $ID close_device $ID $DONEDIR "game over" done echo "" From 61937ac2d000a29aa5766ae0f831609b4310131d Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 31 Oct 2017 20:07:12 -0700 Subject: [PATCH 054/138] use float consistently timeout of 0 was not making the relay.py script happy --- xwords4/linux/relaycon.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 7b172bcf8..112b0e54c 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -69,7 +69,7 @@ static void reset_schedule_check_interval( RelayConStorage* storage ); static void checkForMovesOnce( RelayConStorage* storage ); static gboolean gotDataTimer(gpointer user_data); -static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len, XP_U16 timeout ); +static ssize_t sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len, float timeoutSecs ); static size_t addVLIStr( XP_U8* buf, size_t len, const XP_UCHAR* str ); static void getNetString( const XP_U8** ptr, XP_U16 len, XP_UCHAR* buf ); static XP_U16 getNetShort( const XP_U8** ptr ); @@ -405,7 +405,7 @@ relaycon_deleted( LaunchParams* params, const XP_UCHAR* devID, indx += writeDevID( &tmpbuf[indx], sizeof(tmpbuf) - indx, devID ); indx += writeLong( &tmpbuf[indx], sizeof(tmpbuf) - indx, gameToken ); - sendIt( storage, tmpbuf, indx, 0.0 ); + sendIt( storage, tmpbuf, indx, 0.1 ); } static XP_Bool @@ -488,7 +488,7 @@ sendAckIf( RelayConStorage* storage, const MsgHeader* header ) XP_U8 tmpbuf[16]; int indx = writeHeader( storage, tmpbuf, XWPDEV_ACK ); indx += writeVLI( &tmpbuf[indx], header->packetID ); - sendIt( storage, tmpbuf, indx, 0.0 ); + sendIt( storage, tmpbuf, indx, 0.1 ); } } @@ -921,11 +921,11 @@ schedule_next_check( RelayConStorage* storage ) } static ssize_t -sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len, XP_U16 timeout ) +sendIt( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len, float timeoutSecs ) { ssize_t nSent; if ( storage->params->useHTTP ) { - nSent = post( storage, msgbuf, len, timeout ); + nSent = post( storage, msgbuf, len, timeoutSecs ); } else { nSent = sendto( storage->socket, msgbuf, len, 0, /* flags */ (struct sockaddr*)&storage->saddr, From 220918e5cd908e406fcac55e4e416884d2a6d806 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 1 Nov 2017 05:16:32 -0700 Subject: [PATCH 055/138] fix to compile with gdk3.2 (latest Debian) --- xwords4/linux/Makefile | 4 ++-- xwords4/linux/gtkboard.h | 4 ++++ xwords4/linux/gtkdraw.c | 18 +++++++++++++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/xwords4/linux/Makefile b/xwords4/linux/Makefile index f09c514ee..fc85f7f7c 100644 --- a/xwords4/linux/Makefile +++ b/xwords4/linux/Makefile @@ -226,7 +226,7 @@ OBJ = \ $(BUILD_PLAT_DIR)/relaycon.o \ $(CURSES_OBJS) $(GTK_OBJS) $(MAIN_OBJS) -LIBS = -lm -luuid -lcurl -ljson-c $(GPROFFLAG) +LIBS = -lm -lpthread -luuid -lcurl -ljson-c $(GPROFFLAG) ifdef USE_SQLITE LIBS += -lsqlite3 DEFINES += -DUSE_SQLITE @@ -242,7 +242,7 @@ endif ifneq (,$(findstring DPLATFORM_GTK,$(DEFINES))) LIBS += `pkg-config --libs gtk+-3.0` CFLAGS += `pkg-config --cflags gtk+-3.0` -# CFLAGS += -DGDK_DISABLE_DEPRECATED + CFLAGS += -DGDK_DISABLE_DEPRECATED POINTER_SUPPORT = -DPOINTER_SUPPORT endif diff --git a/xwords4/linux/gtkboard.h b/xwords4/linux/gtkboard.h index 491be224f..d2abde645 100644 --- a/xwords4/linux/gtkboard.h +++ b/xwords4/linux/gtkboard.h @@ -46,6 +46,10 @@ typedef struct GtkDrawCtx { /* GdkDrawable* pixmap; */ GtkWidget* drawing_area; cairo_surface_t* surface; +#ifdef GDK_AVAILABLE_IN_3_22 + GdkDrawingContext* dc; +#endif + struct GtkGameGlobals* globals; #ifdef USE_CAIRO diff --git a/xwords4/linux/gtkdraw.c b/xwords4/linux/gtkdraw.c index aeab1f824..fa04326ad 100644 --- a/xwords4/linux/gtkdraw.c +++ b/xwords4/linux/gtkdraw.c @@ -1,6 +1,6 @@ -/* -*- compile-command: "make MEMDEBUG=TRUE -j3"; -*- */ +/* -*- compile-command: "make MEMDEBUG=TRUE -j5"; -*- */ /* - * Copyright 1997-2011 by Eric House (xwords@eehouse.org). All rights + * Copyright 1997 - 2017 by Eric House (xwords@eehouse.org). All rights * reserved. * * This program is free software; you can redistribute it and/or @@ -86,7 +86,14 @@ initCairo( GtkDrawCtx* dctx ) if ( !!dctx->surface ) { cairo = cairo_create( dctx->surface ); } else if ( !!dctx->drawing_area ) { +#ifdef GDK_AVAILABLE_IN_3_22 + GdkWindow* window = gtk_widget_get_window( dctx->drawing_area ); + const cairo_region_t* region = gdk_window_get_visible_region( window ); + dctx->dc = gdk_window_begin_draw_frame( window, region ); + cairo = gdk_drawing_context_get_cairo_context( dctx->dc ); +#else cairo = gdk_cairo_create( gtk_widget_get_window(dctx->drawing_area) ); +#endif } else { XP_ASSERT( 0 ); } @@ -108,7 +115,12 @@ destroyCairo( GtkDrawCtx* dctx ) { /* XP_LOGF( "%s(dctx=%p)", __func__, dctx ); */ XP_ASSERT( !!dctx->_cairo ); - cairo_destroy(dctx->_cairo); +#ifdef GDK_AVAILABLE_IN_3_22 + GdkWindow* window = gtk_widget_get_window( dctx->drawing_area ); + gdk_window_end_draw_frame( window, dctx->dc ); +#else + cairo_destroy( dctx->_cairo ); +#endif dctx->_cairo = NULL; } From 66d6240ececb4531118281ef1d52875418fa1e9a Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 1 Nov 2017 06:12:04 -0700 Subject: [PATCH 056/138] better git revision fallback --- xwords4/relay/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwords4/relay/Makefile b/xwords4/relay/Makefile index af2bbae6f..6cbaadb5b 100644 --- a/xwords4/relay/Makefile +++ b/xwords4/relay/Makefile @@ -42,7 +42,7 @@ SRC = \ # STATIC ?= -static GITINFO = gitversion.txt -HASH=$(shell git describe) +HASH=$(shell git rev-parse --verify HEAD) OBJ = $(patsubst %.cpp,obj/%.o,$(SRC)) #LDFLAGS += -pthread -g -lmcheck $(STATIC) From 55f5b500e37a2c9533c9f34b267c375db16cf69e Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 2 Nov 2017 06:19:35 -0700 Subject: [PATCH 057/138] cleanup: method names and logging --- xwords4/common/comms.c | 2 +- xwords4/linux/gtkmain.c | 2 +- xwords4/linux/relaycon.c | 34 ++++++++++++++++++++++++++--- xwords4/linux/relaycon.h | 2 +- xwords4/linux/scripts/discon_ok2.sh | 6 ++--- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/xwords4/common/comms.c b/xwords4/common/comms.c index c641fc98e..b980441a6 100644 --- a/xwords4/common/comms.c +++ b/xwords4/common/comms.c @@ -1773,7 +1773,7 @@ relayPreProcess( CommsCtxt* comms, XWStreamCtxt* stream, XWHostID* senderID ) } if ( consumed ) { - XP_LOGF( "%s: rejecting data message", __func__ ); + XP_LOGF( "%s: rejecting data message (consumed)", __func__ ); } else { *senderID = srcID; } diff --git a/xwords4/linux/gtkmain.c b/xwords4/linux/gtkmain.c index 51da3f2ce..9058ad5fe 100644 --- a/xwords4/linux/gtkmain.c +++ b/xwords4/linux/gtkmain.c @@ -513,7 +513,7 @@ static void handle_movescheck( GtkWidget* XP_UNUSED(widget), GtkAppGlobals* apg ) { LaunchParams* params = apg->params; - checkForMsgsNow( params ); + relaycon_checkMsgs( params ); } static void diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 112b0e54c..e5b377a70 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -215,8 +215,9 @@ runWitCurl( RelayTask* task, const gchar* proc, ...) } void -checkForMsgsNow( LaunchParams* params ) +relaycon_checkMsgs( LaunchParams* params ) { + LOG_FUNC(); RelayConStorage* storage = getStorage( params ); XP_ASSERT( onMainThread(storage) ); checkForMovesOnce( storage ); @@ -414,6 +415,30 @@ onMainThread( RelayConStorage* storage ) return storage->mainThread = pthread_self(); } +static const gchar* +taskName( const RelayTask* task ) +{ + switch (task->typ) { + case POST: return "POST"; + case QUERY: return "QUERY"; + default: XP_ASSERT(0); + return NULL; + } +} + +static gchar* +listTasks( GSList* tasks ) +{ + gchar* names[1 + g_slist_length(tasks)]; + names[g_slist_length(tasks)] = NULL; + for ( int ii = 0; !!tasks; ++ii ) { + names[ii] = (gchar*)taskName( (RelayTask*)tasks->data ); + tasks = tasks->next; + } + + return g_strjoinv( ",", names ); +} + static void* relayThread( void* arg ) { @@ -431,10 +456,13 @@ relayThread( void* arg ) RelayTask* task = head->data; g_slist_free( head ); + gchar* strs = listTasks(storage->relayTaskList); + pthread_mutex_unlock( &storage->relayMutex ); - XP_LOGF( "%s(): processing one of %d; created %d secs ago", - __func__, len, ((XP_U32)time(NULL)) - task->ctime ); + XP_LOGF( "%s(): processing one of %d (%s); created %d secs ago", + __func__, len, strs, ((XP_U32)time(NULL)) - task->ctime ); + g_free( strs ); switch ( task->typ ) { case POST: diff --git a/xwords4/linux/relaycon.h b/xwords4/linux/relaycon.h index fc57e1a79..494c86bad 100644 --- a/xwords4/linux/relaycon.h +++ b/xwords4/linux/relaycon.h @@ -59,5 +59,5 @@ void relaycon_cleanup( LaunchParams* params ); XP_U32 makeClientToken( sqlite3_int64 rowid, XP_U16 seed ); void rowidFromToken( XP_U32 clientToken, sqlite3_int64* rowid, XP_U16* seed ); -void checkForMsgsNow( LaunchParams* params ); +void relaycon_checkMsgs( LaunchParams* params ); #endif diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index 24824b895..5990beccb 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -478,10 +478,8 @@ update_ldevid() { if [ $RNUM -lt 30 ]; then # upgrade or first run CMD="--ldevid LINUX_TEST_$(printf %.5d ${KEY})_" fi - else - if [ $RNUM -lt 10 ]; then - CMD="${CMD}x" # give it a new local ID - fi + elif [ $RNUM -lt 10 ]; then + CMD="${CMD}x" # give it a new local ID fi ARGS_DEVID[$KEY]="$CMD" fi From 3dfd419ec704b01853fb1ff6269ae368469fab50 Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 2 Nov 2017 07:16:11 -0700 Subject: [PATCH 058/138] relay returning NOTHING is a thing; deal with it --- xwords4/android/scripts/relay.py | 57 +++++++++++++++++--------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 371f884d6..78769f316 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -68,41 +68,44 @@ def query(req, ids, timeoutSecs = 5.0): idsLen = 0 for id in ids: idsLen += len(id) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(float(timeoutSecs)) - sock.connect(('127.0.0.1', 10998)) + tcpSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcpSock.settimeout(float(timeoutSecs)) + tcpSock.connect(('127.0.0.1', 10998)) lenShort = 2 + idsLen + len(ids) + 2 print(lenShort, PROTOCOL_VERSION, PRX_GET_MSGS, len(ids)) header = struct.Struct('!hBBh') assert header.size == 6 - sock.send(header.pack(lenShort, PROTOCOL_VERSION, PRX_GET_MSGS, len(ids))) + tcpSock.send(header.pack(lenShort, PROTOCOL_VERSION, PRX_GET_MSGS, len(ids))) - for id in ids: sock.send(id + '\n') + for id in ids: tcpSock.send(id + '\n') - shortUnpacker = struct.Struct('!H') - resLen, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) - nameCount, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) - resLen -= shortUnpacker.size - print('resLen:', resLen, 'nameCount:', nameCount) msgsLists = {} - if nameCount == len(ids) and resLen > 0: - print('nameCount', nameCount) - for ii in range(nameCount): - perGame = [] - countsThisGame, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) # problem - print('countsThisGame:', countsThisGame) - for jj in range(countsThisGame): - msgLen, = shortUnpacker.unpack(sock.recv(shortUnpacker.size)) - print('msgLen:', msgLen) - msgs = [] - if msgLen > 0: - msg = sock.recv(msgLen) - print('msg len:', len(msg)) - msg = base64.b64encode(msg) - msgs.append(msg) - perGame.append(msgs) - msgsLists[ids[ii]] = perGame + try: + shortUnpacker = struct.Struct('!H') + resLen, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) # not getting all bytes + nameCount, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) + resLen -= shortUnpacker.size + print('resLen:', resLen, 'nameCount:', nameCount) + if nameCount == len(ids) and resLen > 0: + print('nameCount', nameCount) + for ii in range(nameCount): + perGame = [] + countsThisGame, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) # problem + print('countsThisGame:', countsThisGame) + for jj in range(countsThisGame): + msgLen, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) + print('msgLen:', msgLen) + msgs = [] + if msgLen > 0: + msg = tcpSock.recv(msgLen) + print('msg len:', len(msg)) + msg = base64.b64encode(msg) + msgs.append(msg) + perGame.append(msgs) + msgsLists[ids[ii]] = perGame + except: + None return json.dumps(msgsLists) From 24171d83170997fbd501d7e7d44c7d9420a1b38f Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 2 Nov 2017 07:16:51 -0700 Subject: [PATCH 059/138] log more about tasks --- xwords4/linux/relaycon.c | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index e5b377a70..b5184180a 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -429,14 +429,22 @@ taskName( const RelayTask* task ) static gchar* listTasks( GSList* tasks ) { + XP_U32 now = (XP_U32)time(NULL); gchar* names[1 + g_slist_length(tasks)]; - names[g_slist_length(tasks)] = NULL; + int len = g_slist_length(tasks); + names[len] = NULL; for ( int ii = 0; !!tasks; ++ii ) { - names[ii] = (gchar*)taskName( (RelayTask*)tasks->data ); + RelayTask* task = (RelayTask*)tasks->data; + names[ii] = g_strdup_printf( "{%s:id:%d;age:%ds}", taskName(task), + task->id, now - task->ctime ); tasks = tasks->next; } - return g_strjoinv( ",", names ); + gchar* result = g_strjoinv( ",", names ); + for ( int ii = 0; ii < len; ++ii ) { + g_free( names[ii] ); + } + return result; } static void* @@ -450,18 +458,17 @@ relayThread( void* arg ) } int len = g_slist_length( storage->relayTaskList ); + gchar* strs = listTasks( storage->relayTaskList ); GSList* head = storage->relayTaskList; storage->relayTaskList = g_slist_remove_link( storage->relayTaskList, storage->relayTaskList ); RelayTask* task = head->data; g_slist_free( head ); - gchar* strs = listTasks(storage->relayTaskList); pthread_mutex_unlock( &storage->relayMutex ); - XP_LOGF( "%s(): processing one of %d (%s); created %d secs ago", - __func__, len, strs, ((XP_U32)time(NULL)) - task->ctime ); + XP_LOGF( "%s(): processing first of %d (%s)", __func__, len, strs ); g_free( strs ); switch ( task->typ ) { @@ -483,8 +490,11 @@ addTask( RelayConStorage* storage, RelayTask* task ) { pthread_mutex_lock( &storage->relayMutex ); storage->relayTaskList = g_slist_append( storage->relayTaskList, task ); + gchar* strs = listTasks( storage->relayTaskList ); pthread_cond_signal( &storage->relayCondVar ); pthread_mutex_unlock( &storage->relayMutex ); + XP_LOGF( "%s(): task list now: %s", __func__, strs ); + g_free( strs ); } static RelayTask* @@ -496,15 +506,16 @@ makeRelayTask( RelayConStorage* storage, TaskType typ ) task->id = ++storage->nextTaskID; task->ctime = (XP_U32)time(NULL); task->storage = storage; - XP_LOGF( "%s(): made with id %d from storage %p", __func__, task->id, storage ); return task; } static void freeRelayTask( RelayTask* task ) { - XP_LOGF( "%s(): deleting id %d (%d secs old)", __func__, task->id, - ((XP_U32)time(NULL)) - task->ctime ); + GSList faker = { .next = NULL, .data = task }; + gchar* str = listTasks(&faker); + XP_LOGF( "%s(): deleting %s", __func__, str ); + g_free( str ); g_free( task->ws.ptr ); g_free( task ); } @@ -662,14 +673,19 @@ relaycon_cleanup( LaunchParams* params ) if ( storage->params->useHTTP ) { pthread_mutex_lock( &storage->relayMutex ); int nRelayTasks = g_slist_length( storage->relayTaskList ); + gchar* taskStrs = listTasks( storage->relayTaskList ); pthread_mutex_unlock( &storage->relayMutex ); pthread_mutex_lock( &storage->gotDataMutex ); int nDataTasks = g_slist_length( storage->gotDataTaskList ); + gchar* gotStrs = listTasks( storage->gotDataTaskList ); pthread_mutex_unlock( &storage->gotDataMutex ); - XP_LOGF( "%s(): sends pending: %d; data tasks pending: %d", __func__, - nRelayTasks, nDataTasks ); + XP_LOGF( "%s(): sends pending: %d (%s); data tasks pending: %d (%s)", __func__, + nRelayTasks, gotStrs, nDataTasks, taskStrs ); + + g_free( gotStrs ); + g_free( taskStrs ); } XP_FREEP( params->mpool, ¶ms->relayConStorage ); } From 9f5f5da29c86374445a85674bf16a1bf27bd38c6 Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 3 Nov 2017 18:07:12 -0700 Subject: [PATCH 060/138] add more required debs --- xwords4/relay/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwords4/relay/Makefile b/xwords4/relay/Makefile index 6cbaadb5b..06fc29fa8 100644 --- a/xwords4/relay/Makefile +++ b/xwords4/relay/Makefile @@ -70,7 +70,7 @@ endif memdebug all: xwrelay rq -REQUIRED_DEBS = libpq-dev \ +REQUIRED_DEBS = libpq-dev g++ libglib2.0-dev postgresql \ .PHONY: debcheck debs_install From c4312a2158f341c20ce2ca376282450b9ea0438c Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 3 Nov 2017 08:01:19 -0700 Subject: [PATCH 061/138] new files that may replace GSList in relaycon If I want to move relaycon into common so Android can use it (assuming the jni code starts including json-c and libcurl so it can handle networking) I'll need a replacement for GSList. This is a start. --- xwords4/common/xwlist.c | 127 ++++++++++++++++++++++++++++++++++++++++ xwords4/common/xwlist.h | 44 ++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 xwords4/common/xwlist.c create mode 100644 xwords4/common/xwlist.h diff --git a/xwords4/common/xwlist.c b/xwords4/common/xwlist.c new file mode 100644 index 000000000..31a85b127 --- /dev/null +++ b/xwords4/common/xwlist.c @@ -0,0 +1,127 @@ +/* -*-mode: C; fill-column: 78; c-basic-offset: 4; -*- */ +/* + * Copyright 2009 by Eric House (xwords@eehouse.org). All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#include "xwlist.h" + +#define MAX_HERE 16 + +typedef struct XWList { + XP_U16 len; + XP_U16 size; + elem* list; + MPSLOT +} XWList; + +XWList* +mk_list(MPFORMAL XP_U16 XP_UNUSED(sizeHint)) +{ + XWList* list = XP_CALLOC( mpool, sizeof(*list)); + MPASSIGN( list->mpool, mpool); + return list; +} + +void +list_append( XWList* self, elem one ) +{ + if ( self->size == 0 ) { /* initial case */ + self->size = 2; + self->list = XP_MALLOC( self->mpool, self->size * sizeof(self->list[0]) ); + } + if ( self->len == self->size ) { /* need to grow? */ + self->size *= 2; + self->list = XP_REALLOC( self->mpool, self->list, self->size * sizeof(self->list[0]) ); + } + + self->list[self->len++] = one; + XP_LOGF( "%s(): put %p at position %d (size: %d)", __func__, one, self->len-1, self->size ); +} + +XP_U16 +list_get_len( const XWList* list ) +{ + return list->len; +} + +void +list_remove_front( XWList* self, elem* out, XP_U16* countp ) +{ + const XP_U16 nMoved = XP_MIN( *countp, self->len ); + XP_MEMCPY( out, self->list, nMoved * sizeof(out[0]) ); + *countp = nMoved; + + // Now copy the survivors down + self->len -= nMoved; + XP_MEMMOVE( &self->list[0], &self->list[nMoved], self->len * sizeof(self->list[0])); +} + +void +list_remove_back(XWList* XP_UNUSED(self), elem* XP_UNUSED(here), XP_U16* XP_UNUSED(count)) +{ +} + +void +list_free( XWList* self, destructor proc, void* closure ) +{ + if ( !!proc ) { + for ( XP_U16 ii = 0; ii < self->len; ++ii ) { + (*proc)(self->list[ii], closure); + } + } + + if ( !!self->list ) { + XP_FREE( self->mpool, self->list ); + } + XP_FREE( self->mpool, self ); +} + +#ifdef DEBUG + +static void +dest(elem elem, void* XP_UNUSED(closure)) +{ + XP_LOGF( "%s(%p)", __func__, elem); +} + +void +list_test_lists(MPFORMAL_NOCOMMA) +{ + XWList* list = mk_list( mpool, 16 ); + for ( char* ii = 0; ii < (char*)100; ++ii ) { + (void)list_append( list, ii ); + } + + XP_ASSERT( list_get_len(list) == 100 ); + + char* prev = 0; + while ( 0 < list_get_len( list ) ) { + elem here; + XP_U16 count = 1; + list_remove_front( list, &here, &count ); + XP_LOGF( "%s(): got here: %p", __func__, here ); + XP_ASSERT( count == 1 ); + XP_ASSERT( prev++ == here ); + } + + for ( char* ii = 0; ii < (char*)10; ++ii ) { + (void)list_append( list, ii ); + } + + list_free( list, dest, NULL ); +} +#endif diff --git a/xwords4/common/xwlist.h b/xwords4/common/xwlist.h new file mode 100644 index 000000000..d404590e6 --- /dev/null +++ b/xwords4/common/xwlist.h @@ -0,0 +1,44 @@ +/* -*-mode: C; fill-column: 78; c-basic-offset: 4; -*- */ +/* + * Copyright 2017 by Eric House (xwords@eehouse.org). All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef _XWLIST_H_ +#define _XWLIST_H_ + +#include "comtypes.h" +#include "mempool.h" + +#include "xptypes.h" + +typedef void* elem; +typedef struct XWList XWList; +typedef void (*destructor)(elem one, void* closure); + +XWList* mk_list(MPFORMAL XP_U16 sizeHint); +void list_free(XWList* list, destructor proc, void* closure); + +void list_append(XWList* list, elem one); +XP_U16 list_get_len(const XWList* list); +void list_remove_front(XWList* list, elem* here, XP_U16* count); +void list_remove_back(XWList* list, elem* here, XP_U16* count); + +#ifdef DEBUG +void list_test_lists(MPFORMAL_NOCOMMA); +#endif + +#endif From 3a2953427a28f70f9052243e8a3fa4f69dcd0f7a Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 3 Nov 2017 08:03:07 -0700 Subject: [PATCH 062/138] combine adjacent QUERY tasks Trying to reduce the size of the queue. Eventually the lists of relayIDs will need to be merged. --- xwords4/linux/relaycon.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index b5184180a..707c20756 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -485,11 +485,26 @@ relayThread( void* arg ) return NULL; } +static XP_Bool +didCombine( const RelayTask* one, const RelayTask* two ) +{ + /* For now.... */ + XP_Bool result = one->typ == QUERY && two->typ == QUERY; + return result; +} + static void addTask( RelayConStorage* storage, RelayTask* task ) { pthread_mutex_lock( &storage->relayMutex ); - storage->relayTaskList = g_slist_append( storage->relayTaskList, task ); + + /* Let's see if the current last task is the same. */ + GSList* last = g_slist_last( storage->relayTaskList ); + if ( !!last && didCombine( last->data, task ) ) { + freeRelayTask( task ); + } else { + storage->relayTaskList = g_slist_append( storage->relayTaskList, task ); + } gchar* strs = listTasks( storage->relayTaskList ); pthread_cond_signal( &storage->relayCondVar ); pthread_mutex_unlock( &storage->relayMutex ); @@ -938,13 +953,14 @@ static void reset_schedule_check_interval( RelayConStorage* storage ) { XP_ASSERT( onMainThread(storage) ); - storage->nextMoveCheckMS = 0; + storage->nextMoveCheckMS = 500; } static void schedule_next_check( RelayConStorage* storage ) { XP_ASSERT( onMainThread(storage) ); + XP_ASSERT( !storage->params->noHTTPAuto ); if ( !storage->params->noHTTPAuto ) { if ( storage->moveCheckerID != 0 ) { g_source_remove( storage->moveCheckerID ); @@ -955,7 +971,7 @@ schedule_next_check( RelayConStorage* storage ) if ( storage->nextMoveCheckMS > MAX_MOVE_CHECK_MS ) { storage->nextMoveCheckMS = MAX_MOVE_CHECK_MS; } else if ( storage->nextMoveCheckMS == 0 ) { - storage->nextMoveCheckMS = 250; + storage->nextMoveCheckMS = 1000; } storage->moveCheckerID = g_timeout_add( storage->nextMoveCheckMS, From 1648c9b3e3cb3ded372d9d1f4487269658dae5cb Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 3 Nov 2017 08:04:38 -0700 Subject: [PATCH 063/138] tweak to start GTK games and run a long time I'm trying to fix game-start-time right now.... --- xwords4/linux/scripts/start-pair.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/xwords4/linux/scripts/start-pair.sh b/xwords4/linux/scripts/start-pair.sh index 767169b6c..c579123a1 100755 --- a/xwords4/linux/scripts/start-pair.sh +++ b/xwords4/linux/scripts/start-pair.sh @@ -3,11 +3,13 @@ set -e -u IN_SEQ='' -HTTP='' +HTTP='--use-http' +CURSES='--curses' +SLEEP_SEC=10000 usage() { [ $# -gt 0 ] && echo "ERROR: $1" - echo "usage: $0 --in-sequence|--at-once [--use-http]" + echo "usage: $0 --in-sequence|--at-once [--no-use-http] [--gtk]" cat </dev/null 2>>$LOG & @@ -71,7 +76,7 @@ for GAME in $(seq 1); do done done -[ -n "${PIDS}" ] && sleep 10 +[ -n "${PIDS}" ] && sleep $SLEEP_SEC for PID in $PIDS; do kill $PID done From 7efcdf0cb89589f2cda48832473382a570acb748 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 4 Nov 2017 09:25:09 -0700 Subject: [PATCH 064/138] script for tracking message delivery The http-based comms flow is stalling. This will help figure out why. --- xwords4/linux/scripts/list-message-flow.py | 100 +++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100755 xwords4/linux/scripts/list-message-flow.py diff --git a/xwords4/linux/scripts/list-message-flow.py b/xwords4/linux/scripts/list-message-flow.py new file mode 100755 index 000000000..92dbd08a8 --- /dev/null +++ b/xwords4/linux/scripts/list-message-flow.py @@ -0,0 +1,100 @@ +#!/usr/bin/python3 + +import getopt, re, sys +import json, psycopg2 + +""" + +I want to understand why some messages linger on the database so +long. So given one or more logfiles that track a linux client's +interaction, look at what it sends and receives and compare that with +what's in the relay's msgs table. + +""" + +DEVID_PAT = re.compile('.*linux_getDevIDRelay => (\d+)$') +QUERY_GOT_PAT = re.compile('.*>(\d+:\d+:\d+):runWitCurl\(\): got for query: \"({.*})\"$') +# <26828:7f03b7fff700>07:47:20:runWitCurl(): got for post: "{"data": ["AR03ggcAH2gwBwESbnVja3k6NTlmYTFjZmM6MTEw", "AR43ggcAH2gwDQBvAgEAAAAAvdAAAAAAAAAAAJIGUGxheWVyGg==", "AYALgw=="], "err": "timeout"}" +POST_GOT_PAT = re.compile('.*>(\d+:\d+:\d+):runWitCurl\(\): got for post: \"({.*})\"$') +def usage(msg = None): + if msg: sys.stderr.write('ERROR:' + msg + '\n') + sys.stderr.write('usage: ' + sys.argv[0] + ': (-l logfile)+ \n') + sys.exit(1) + +def parseLog(log, data): + devIDs = [] + msgMap = {} + for line in open(log): + line = line.strip() + aMatch = DEVID_PAT.match(line) + if aMatch: + devID = int(aMatch.group(1)) + if devID and (len(devIDs) == 0 or devIDs[-1] != devID): + devIDs.append(devID) + + aMatch = QUERY_GOT_PAT.match(line) + if aMatch: + rtime = aMatch.group(1) + jobj = json.loads(aMatch.group(2)) + for relayID in jobj: + msgs = jobj[relayID] + for msgarr in msgs: + for msg in msgarr: + if not msg in msgMap: msgMap[msg] = [] + msgMap[msg].append({'rtime' : rtime,}) + if len(msgMap[msg]) > 1: print('big case') + + aMatch = POST_GOT_PAT.match(line) + if aMatch: + jobj = json.loads(aMatch.group(2)) + for datum in jobj['data']: + data.add(datum) + + return devIDs, msgMap + +def fetchMsgs(devIDs, msgMaps, data): + foundCount = 0 + notFoundCount = 0 + + con = psycopg2.connect(database='xwgames') + cur = con.cursor() + query = "SELECT ctime, stime, stime-ctime as age, msg64 FROM msgs WHERE devid in (%s) order by ctime" \ + % (','.join([str(id) for id in devIDs])) + # print(query) + cur.execute(query) + for row in cur: + msg64 = row[3] + for msgMap in msgMaps: + if msg64 in msgMap: + print('added:', row[0], 'sent:', row[1], 'received:', msgMap[msg64][0]['rtime'], 'age:', row[2]) + if msg64 in data: + foundCount += 1 + else: + notFoundCount += 1 + print('found:', foundCount, 'not found:', notFoundCount); + + +def main(): + logs = [] + opts, args = getopt.getopt(sys.argv[1:], "l:") + for option, value in opts: + if option == '-l': logs.append(value) + else: usage("unknown option" + option) + + if len(logs) == 0: usage('at least one -l requried') + + msgMaps = [] + devIDs = set() + data = set() + for log in logs: + ids, msgMap = parseLog(log, data) + msgMaps.append(msgMap) + for id in ids: devIDs.add(id) + + print(msgMaps) + print(devIDs) + fetchMsgs(devIDs, msgMaps, data) + +############################################################################## +if __name__ == '__main__': + main() From 02f05dc8679d3d2aff668540ca4a191fd88218ea Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 4 Nov 2017 09:26:26 -0700 Subject: [PATCH 065/138] fix to compile with UDP packet logged to match up with delivery on client --- xwords4/relay/xwrelay.cpp | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index eaa407695..269fa2eab 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -550,18 +550,18 @@ assemble_packet( vector& packet, uint32_t* packetIDP, XWRelayReg cmd, } #ifdef LOG_UDP_PACKETS - gsize size = 0; - gint state = 0; - gint save = 0; - gchar out[1024]; - for ( unsigned int ii = 0; ii < iocount; ++ii ) { - size += g_base64_encode_step( (const guchar*)vec[ii].iov_base, - vec[ii].iov_len, - FALSE, &out[size], &state, &save ); - } - size += g_base64_encode_close( FALSE, &out[size], &state, &save ); - assert( size < sizeof(out) ); - out[size] = '\0'; + // gsize size = 0; + // gint state = 0; + // gint save = 0; + // gchar out[1024]; + // for ( unsigned int ii = 0; ii < iocount; ++ii ) { + // size += g_base64_encode_step( (const guchar*)vec[ii].iov_base, + // vec[ii].iov_len, + // FALSE, &out[size], &state, &save ); + // } + // size += g_base64_encode_close( FALSE, &out[size], &state, &save ); + // assert( size < sizeof(out) ); + // out[size] = '\0'; #endif } @@ -640,8 +640,10 @@ send_via_udp_impl( int sock, const struct sockaddr* dest_addr, #ifdef LOG_UDP_PACKETS gchar* b64 = g_base64_encode( (uint8_t*)dest_addr, sizeof(*dest_addr) ); + gchar* out = g_base64_encode( packet.data(), packet.size() ); logf( XW_LOGINFO, "%s()=>%d; addr='%s'; msg='%s'", __func__, nSent, b64, out ); + g_free( out ); g_free( b64 ); #else logf( XW_LOGINFO, "%s()=>%d", __func__, nSent ); From e6acd0fbccc63166dca90bc3f4e967afc292c125 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 5 Nov 2017 19:24:41 -0800 Subject: [PATCH 066/138] beginnings of new relay: shows persistence Mostly comments, it's where I'll start to think about the API for a web-app version of the relay. The server itself ought to exist already. --- xwords4/newrelay/nr.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 xwords4/newrelay/nr.py diff --git a/xwords4/newrelay/nr.py b/xwords4/newrelay/nr.py new file mode 100755 index 000000000..4700dd534 --- /dev/null +++ b/xwords4/newrelay/nr.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +import json, shelve + +"""This will be a prototype of a simple store-and-forward message +passing server. Target clients are peer-to-peer apps like turn-based +games and maybe chat apps. It's expected that they depend on the +server for nothing but message passing and any group-formation that it +depends on (e.g three devices agreeing to participate in a single game +of Fish, a process that gets them a token that can be used to address +messages that are part of that game.) Clients can use this server as +one of several means of communicating, depending on it to deliver +messages e.g. when the devices are out of range of bluetooth. + +""" + +# register: a device is meant to call this once to get from the server +# an identifier that will identify it from then on. Other APIs will +# require this identifier. +# +# @param clientID: a String the client can optionally provide to link +# this registration to an earlier one. For example, if a client app +# wants state to survive a hard reset of the device but there are IDs +# like serial numbers or a user's email address that will survive that +# process, such an id could be used. +def register(req, clientID = None): + shelf = openShelf() + obj = {'deviceID' : shelf['nextID']} + shelf['nextID'] += 1 + shelf.close() + return json.dumps(obj) + +# Associate attributes with a device that can be used for indirectly +# related purposes. The one I have in mind is GCM (Google Cloud +# Messaging), where the device provides a server an ID that the server +# can use to ask google's servers to forward a push message to the +# device. +def setAttr(req, deviceID, attrKey, attrValue): + pass + +# joinRoom: called when a device wants to start a new game to which +# other devices will also connect. Returns a gameID that internally +# refers to the game joined and the device's position in it. +# @param +def joinRoom(deviceID, room, lang, nTotal, nHere = 1, position = 0): + pass + +def forward(req, deviceID, msg, roomID, positions): + pass + +def openShelf(): + shelf = shelve.open("/tmp/nr.shelf") + if not 'nextID' in shelf: shelf['nextID'] = 0; + return shelf + +def main(): + pass + +############################################################################## +if __name__ == '__main__': + main() From c661c8a74ec05bd7dd05cffbb09a1a52a23a6205 Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 7 Nov 2017 07:33:04 -0800 Subject: [PATCH 067/138] basic join() (inserting/updating game records) works --- xwords4/android/scripts/relay.py | 49 ++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 78769f316..10443a2d2 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -1,6 +1,7 @@ #!/usr/bin/python import base64, json, mod_python, socket, struct, sys +import psycopg2, random PROTOCOL_VERSION = 0 PRX_DEVICE_GONE = 3 @@ -12,6 +13,40 @@ PRX_GET_MSGS = 4 # except ImportError: # apacheAvailable = False +def join(req, devID, room, lang = 1, nInGame = 2, nHere = 1, inviteID = None): + connname = None + con = psycopg2.connect(database='xwgames') + + query = """UPDATE games SET njoined = njoined + %d + WHERE lang = %d AND nTotal = %d AND room = '%s' AND njoined + %d <= ntotal + RETURNING connname""" + query = query % (nHere, lang, nInGame, room, nHere) + print(query) + cur = con.cursor() + cur.execute(query) + for row in cur: + connname = row[0] + print 'got:', connname + + if not connname: + connname = str(random.randint(0, 10000000000)) + query = """INSERT INTO games (connname, room, lang, ntotal, njoined) + values ('%s', '%s', %d, %d, %d) RETURNING connname; """ + query %= (connname, room, lang, nInGame, nHere) + print(query) + cur.execute(query) + for row in cur: + print row + else: + print 'got nothing' + print 'did an insert' + con.commit() + con.close() + + result = {'connname': connname} + + return json.dumps(result) + def kill(req, params): print(params) params = json.loads(params) @@ -36,22 +71,22 @@ def kill(req, params): result = {'err': 0} return json.dumps(result) - + +# winds up in handle_udp_packet() in xwrelay.cpp def post(req, params, timeoutSecs = 1.0): err = 'none' - dataLen = 0 jobj = json.loads(params) data = base64.b64decode(jobj['data']) - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(float(timeoutSecs)) # seconds + udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + udpSock.settimeout(float(timeoutSecs)) # seconds addr = ("127.0.0.1", 10997) - sock.sendto(data, addr) + udpSock.sendto(data, addr) responses = [] while True: try: - data, server = sock.recvfrom(1024) + data, server = udpSock.recvfrom(1024) responses.append(base64.b64encode(data)) except socket.timeout: #If data is not received back from server, print it has timed out @@ -121,6 +156,8 @@ def main(): # params = json.dumps(params) # print(post(None, params)) None + elif cmd == 'join': + result = join(None, 1, args[0]) elif cmd == 'kill': result = kill( None, json.dumps([{'relayID': args[0], 'seed':int(args[1])}]) ) From 6c8b9d52770b502cd0622e6508809e2eb1314f46 Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 7 Nov 2017 07:34:06 -0800 Subject: [PATCH 068/138] fix query syntax: % operator bad --- xwords4/android/scripts/relay.py | 39 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 10443a2d2..fb2df03c2 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -17,13 +17,11 @@ def join(req, devID, room, lang = 1, nInGame = 2, nHere = 1, inviteID = None): connname = None con = psycopg2.connect(database='xwgames') - query = """UPDATE games SET njoined = njoined + %d - WHERE lang = %d AND nTotal = %d AND room = '%s' AND njoined + %d <= ntotal + query = """UPDATE games SET njoined = njoined + %s + WHERE lang = %s AND nTotal = %s AND room = %s AND njoined + %s <= ntotal RETURNING connname""" - query = query % (nHere, lang, nInGame, room, nHere) - print(query) cur = con.cursor() - cur.execute(query) + cur.execute(query, (nHere, lang, nInGame, room, nHere)) for row in cur: connname = row[0] print 'got:', connname @@ -31,10 +29,8 @@ def join(req, devID, room, lang = 1, nInGame = 2, nHere = 1, inviteID = None): if not connname: connname = str(random.randint(0, 10000000000)) query = """INSERT INTO games (connname, room, lang, ntotal, njoined) - values ('%s', '%s', %d, %d, %d) RETURNING connname; """ - query %= (connname, room, lang, nInGame, nHere) - print(query) - cur.execute(query) + values (%s, %s, %s, %s, %s) RETURNING connname; """ + cur.execute(query, (connname, room, lang, nInGame, nHere)) for row in cur: print row else: @@ -149,17 +145,19 @@ def main(): if len(sys.argv) > 1: cmd = sys.argv[1] args = sys.argv[2:] - if cmd == 'query': - result = query(None, json.dumps(args)) - elif cmd == 'post': - # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } - # params = json.dumps(params) - # print(post(None, params)) - None - elif cmd == 'join': - result = join(None, 1, args[0]) - elif cmd == 'kill': - result = kill( None, json.dumps([{'relayID': args[0], 'seed':int(args[1])}]) ) + try : + if cmd == 'query': + result = query(None, json.dumps(args)) + elif cmd == 'post': + # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } + # params = json.dumps(params) + # print(post(None, params)) + pass + elif cmd == 'join': + result = join(None, 1, args[0], int(args[1]), int(args[2])) + elif cmd == 'kill': + result = kill( None, json.dumps([{'relayID': args[0], 'seed':int(args[1])}]) ) + except : pass if result: print '->', result @@ -167,6 +165,7 @@ def main(): print 'USAGE: query [connname/hid]*' # print ' post ' print ' kill ' + print ' join ' ############################################################################## if __name__ == '__main__': From 205adf0d0fc5df23f53330b66629f46fd25e9da1 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 8 Nov 2017 08:04:57 -0800 Subject: [PATCH 069/138] add prototype of join() join's how devices will create or join existing games. It more compilicated than I'd like but seems to work except that once a slot's assigned it's unavailable to anybody else even if the other fails ever to respond (i.e. needs the ACK function of the c++ relay.) --- xwords4/android/scripts/relay.py | 117 +++++++++++++++++++++++-------- 1 file changed, 86 insertions(+), 31 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index fb2df03c2..486f67f37 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -13,33 +13,89 @@ PRX_GET_MSGS = 4 # except ImportError: # apacheAvailable = False -def join(req, devID, room, lang = 1, nInGame = 2, nHere = 1, inviteID = None): +# Joining a game. Basic idea is you have stuff to match on (room, +# number in game, language) and when somebody wants to join you add to +# an existing matching game if there's space otherwise create a new +# one. Problems are the unreliablity of transport: if you give a space +# and the device doesn't get the message you can't hold it forever. So +# device provides a seed that holds the space. If it asks again for a +# space with the same seed it gets the same space. If it never asks +# again (app deleted, say), the space needs eventually to be given to +# somebody else. I think that's done by adding a timestamp array and +# treating the space as available if TIME has expired. Need to think +# about this: what if app fails to ACK for TIME, then returns with +# seed to find it given away. Let's do a 30 minute reservation for +# now? [Note: much of this is PENDING] + +def join(req, devID, room, seed, hid = 0, lang = 1, nInGame = 2, nHere = 1, inviteID = None): connname = None con = psycopg2.connect(database='xwgames') - - query = """UPDATE games SET njoined = njoined + %s - WHERE lang = %s AND nTotal = %s AND room = %s AND njoined + %s <= ntotal - RETURNING connname""" cur = con.cursor() - cur.execute(query, (nHere, lang, nInGame, room, nHere)) - for row in cur: - connname = row[0] - print 'got:', connname + assert hid <= 4 + assert seed != 0 + + # First see if there's a game with a space for me. Must match on + # room, lang and size. Must have room OR must have already given a + # spot for a seed equal to mine, in which case I get it + # back. Assumption is I didn't ack in time. + + query = "SELECT connname, seeds, nperdevice FROM games " + query += "WHERE lang = %s AND nTotal = %s AND room = %s " + query += "AND (njoined + %s <= ntotal OR %s = ANY(seeds)) " + query += "LIMIT 1" + cur.execute( query, (lang, nInGame, room, nHere, seed)) + for row in cur: + (connname, seeds, nperdevice) = row + print('found', connname, seeds, nperdevice) + break # should be only one! + + # If we've found a match, we either need to UPDATE or, if the + # seeds match, remind the caller of where he belongs. If a hid's + # been specified, we honor it by updating if the slot's available; + # otherwise a new game has to be created. + if connname: + if seed in seeds and nHere == nperdevice[seeds.index(seed)]: + hid = seeds.index(seed) + 1 + print('resusing seed case; outta here!') + else: + if hid == 0: + # Any gaps? Assign it + if None in seeds: + hid = seeds.index(None) + 1 + else: + hid = len(seeds) + 1 + print('set hid to', hid, 'based on ', seeds) + else: + print('hid already', hid) + query = "UPDATE games SET njoined = njoined + %s, " + query += "seeds[%d] = %%s, " % hid + query += "jtimes[%d] = 'now', " % hid + query += "nperdevice[%d] = %%s " % hid + query += "WHERE connname = %s " + print(query) + params = (nHere, seed, nHere, connname) + cur.execute(query, params) + + # If nothing was found, add a new game and add me. Honor my hid + # preference if specified if not connname: connname = str(random.randint(0, 10000000000)) - query = """INSERT INTO games (connname, room, lang, ntotal, njoined) - values (%s, %s, %s, %s, %s) RETURNING connname; """ - cur.execute(query, (connname, room, lang, nInGame, nHere)) + useHid = hid == 0 and 1 or hid + print('not found case; inserting using hid:', useHid) + query = "INSERT INTO games (connname, room, lang, ntotal, njoined, seeds[%d], jtimes[%d], nperdevice[%d]) " % (useHid, useHid, useHid) + query += "VALUES (%s, %s, %s, %s, %s, %s, 'now', %s) " + query += "RETURNING connname, array_length(seeds,1); " + cur.execute(query, (connname, room, lang, nInGame, nHere, seed, nHere)) for row in cur: - print row - else: - print 'got nothing' - print 'did an insert' + connname, gothid = row + break + if hid == 0: hid = gothid + con.commit() con.close() - result = {'connname': connname} + result = {'connname': connname, 'hid' : hid} return json.dumps(result) @@ -145,19 +201,18 @@ def main(): if len(sys.argv) > 1: cmd = sys.argv[1] args = sys.argv[2:] - try : - if cmd == 'query': - result = query(None, json.dumps(args)) - elif cmd == 'post': - # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } - # params = json.dumps(params) - # print(post(None, params)) - pass - elif cmd == 'join': - result = join(None, 1, args[0], int(args[1]), int(args[2])) - elif cmd == 'kill': - result = kill( None, json.dumps([{'relayID': args[0], 'seed':int(args[1])}]) ) - except : pass + if cmd == 'query': + result = query(None, json.dumps(args)) + elif cmd == 'post': + # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } + # params = json.dumps(params) + # print(post(None, params)) + pass + elif cmd == 'join': + if len(args) == 6: + result = join(None, 1, args[0], int(args[1]), int(args[2]), int(args[3]), int(args[4]), int(args[5])) + elif cmd == 'kill': + result = kill( None, json.dumps([{'relayID': args[0], 'seed':int(args[1])}]) ) if result: print '->', result @@ -165,7 +220,7 @@ def main(): print 'USAGE: query [connname/hid]*' # print ' post ' print ' kill ' - print ' join ' + print ' join ' ############################################################################## if __name__ == '__main__': From d0dd7a2c029677f474efad4967b9f26951a41648 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 8 Nov 2017 08:07:47 -0800 Subject: [PATCH 070/138] new columns required by relay.py's join() --- xwords4/relay/xwrelay.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xwords4/relay/xwrelay.sh b/xwords4/relay/xwrelay.sh index 1665c603f..5422e5259 100755 --- a/xwords4/relay/xwrelay.sh +++ b/xwords4/relay/xwrelay.sh @@ -49,6 +49,8 @@ cid integer ,pub BOOLEAN ,connName VARCHAR(64) UNIQUE PRIMARY KEY ,nTotal INTEGER +,nJoined INTEGER DEFAULT 0 +,jtimes TIMESTAMP(0)[] ,clntVers INTEGER[] ,nPerDevice INTEGER[] ,seeds INTEGER[] @@ -59,7 +61,7 @@ cid integer ,addrs INET[] ,devids INTEGER[] ,tokens INTEGER[] -); +,CHECK (nJoined <= nTotal)) EOF cat <<-EOF | psql $DBNAME --file - From d1e6a0d1d3debae50c7265c54d90326c2c0867bc Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 9 Nov 2017 06:47:33 -0800 Subject: [PATCH 071/138] fix seed comparison: ints vs strings Was coming in as a string when called via curl. This may be a problem for other ints if I go with lots of params instead of a json, which is looking less likely. --- xwords4/android/scripts/relay.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 486f67f37..cea60aa0c 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -28,12 +28,15 @@ PRX_GET_MSGS = 4 # now? [Note: much of this is PENDING] def join(req, devID, room, seed, hid = 0, lang = 1, nInGame = 2, nHere = 1, inviteID = None): + assert hid <= 4 + seed = int(seed) + assert seed != 0 + connname = None + logs = [] # for debugging con = psycopg2.connect(database='xwgames') cur = con.cursor() - - assert hid <= 4 - assert seed != 0 + # cur.execute('LOCK TABLE games IN ACCESS EXCLUSIVE MODE') # First see if there's a game with a space for me. Must match on # room, lang and size. Must have room OR must have already given a @@ -95,7 +98,7 @@ def join(req, devID, room, seed, hid = 0, lang = 1, nInGame = 2, nHere = 1, invi con.commit() con.close() - result = {'connname': connname, 'hid' : hid} + result = {'connname': connname, 'hid' : hid, 'log' : ':'.join(logs)} return json.dumps(result) From ecc247d56dae1858e0d3239f9f28095dfce833af Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 10 Nov 2017 20:30:25 -0800 Subject: [PATCH 072/138] fix a couple of obvious bugs --- xwords4/linux/gamesdb.c | 5 +++-- xwords4/linux/relaycon.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/xwords4/linux/gamesdb.c b/xwords4/linux/gamesdb.c index fd968a22b..377a0dac0 100644 --- a/xwords4/linux/gamesdb.c +++ b/xwords4/linux/gamesdb.c @@ -129,13 +129,14 @@ writeBlobColumnData( const XP_U8* data, gsize len, XP_U16 strVersion, sqlite3* p assertPrintResult( pDb, result, SQLITE_OK ); result = sqlite3_blob_close( blob ); assertPrintResult( pDb, result, SQLITE_OK ); + if ( !!stmt ) { sqlite3_finalize( stmt ); } LOG_RETURNF( "%lld", curRow ); return curRow; -} +} /* writeBlobColumnData */ static sqlite3_int64 writeBlobColumnStream( XWStreamCtxt* stream, sqlite3* pDb, sqlite3_int64 curRow, @@ -326,7 +327,7 @@ getRelayIDsToRowsMap( sqlite3* pDb ) XP_UCHAR relayID[32]; getColumnText( ppStmt, 0, relayID, VSIZE(relayID) ); gpointer key = g_strdup( relayID ); - sqlite3_int64* value = g_malloc( sizeof( *key ) ); + sqlite3_int64* value = g_malloc( sizeof( value ) ); *value = sqlite3_column_int64( ppStmt, 1 ); g_hash_table_insert( table, key, value ); /* XP_LOGF( "%s(): added map %s => %lld", __func__, (char*)key, *value ); */ diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 707c20756..d5bad5e81 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -152,7 +152,7 @@ addJsonParams( CURL* curl, va_list ap ) json_object* param = va_arg(ap, json_object*); XP_ASSERT( !!param ); - const char* asStr = json_object_to_json_string( param ); + const char* asStr = json_object_get_string( param ); XP_LOGF( "%s: adding param (with name %s): %s", __func__, name, asStr ); char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); From f2c4c82129446ac3a1010726eb84c58391d40d34 Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 10 Nov 2017 21:34:02 -0800 Subject: [PATCH 073/138] a shot at no-conn connecting Ideally the comms module wouldn't go through its connecting routine in order to join a game. To that end I added a join() method to relay.py and code to call it. Joins happen (pairing games, starting new ones, etc.), but after that communication doesn't. First part of fixing that would be to make cookieID persistent and transmit it back with the rest of what join sends (since it's used by all the messages currently sent in a connected state), but I suspect there's more to be done, and even that requires a fair number of changes on the relay side. So all that's wrapped in #ifdef RELAY_VIA_HTTP (and turned off.) --- xwords4/android/scripts/relay.py | 25 ++++- xwords4/common/comms.c | 50 +++++++-- xwords4/common/comms.h | 20 +++- xwords4/linux/Makefile | 1 + xwords4/linux/cursesmain.c | 27 ++++- xwords4/linux/gtkboard.c | 61 +++++++++-- xwords4/linux/relaycon.c | 164 +++++++++++++++++++++++++--- xwords4/linux/relaycon.h | 8 ++ xwords4/relay/scripts/showinplay.sh | 2 +- 9 files changed, 315 insertions(+), 43 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index cea60aa0c..8ae901fb1 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -31,9 +31,17 @@ def join(req, devID, room, seed, hid = 0, lang = 1, nInGame = 2, nHere = 1, invi assert hid <= 4 seed = int(seed) assert seed != 0 + nInGame = int(nInGame) + nHere = int(nHere) + assert nHere <= nInGame + assert nInGame <= 4 + + devID = int(devID, 16) connname = None logs = [] # for debugging + # logs.append('vers: ' + platform.python_version()) + con = psycopg2.connect(database='xwgames') cur = con.cursor() # cur.execute('LOCK TABLE games IN ACCESS EXCLUSIVE MODE') @@ -72,24 +80,29 @@ def join(req, devID, room, seed, hid = 0, lang = 1, nInGame = 2, nHere = 1, invi else: print('hid already', hid) query = "UPDATE games SET njoined = njoined + %s, " + query += "devids[%d] = %%s, " % hid query += "seeds[%d] = %%s, " % hid query += "jtimes[%d] = 'now', " % hid query += "nperdevice[%d] = %%s " % hid query += "WHERE connname = %s " print(query) - params = (nHere, seed, nHere, connname) + params = (nHere, devID, seed, nHere, connname) cur.execute(query, params) # If nothing was found, add a new game and add me. Honor my hid # preference if specified if not connname: - connname = str(random.randint(0, 10000000000)) + # This requires python3, which likely requires mod_wsgi + # ts = datetime.datetime.utcnow().timestamp() + # connname = '%s:%d:1' % (xwconfig.k_HOSTNAME, int(ts * 1000)) + connname = '%s:%d:1' % (xwconfig.k_HOSTNAME, random.randint(0, 10000000000)) useHid = hid == 0 and 1 or hid print('not found case; inserting using hid:', useHid) - query = "INSERT INTO games (connname, room, lang, ntotal, njoined, seeds[%d], jtimes[%d], nperdevice[%d]) " % (useHid, useHid, useHid) - query += "VALUES (%s, %s, %s, %s, %s, %s, 'now', %s) " + query = "INSERT INTO games (connname, room, lang, ntotal, njoined, " + \ + "devids[%d], seeds[%d], jtimes[%d], nperdevice[%d]) " % (4 * (useHid,)) + query += "VALUES (%s, %s, %s, %s, %s, %s, %s, 'now', %s) " query += "RETURNING connname, array_length(seeds,1); " - cur.execute(query, (connname, room, lang, nInGame, nHere, seed, nHere)) + cur.execute(query, (connname, room, lang, nInGame, nHere, devID, seed, nHere)) for row in cur: connname, gothid = row break @@ -221,6 +234,8 @@ def main(): print '->', result else: print 'USAGE: query [connname/hid]*' + print ' join ' + print ' query [connname/hid]*' # print ' post ' print ' kill ' print ' join ' diff --git a/xwords4/common/comms.c b/xwords4/common/comms.c index b980441a6..4afd3dba9 100644 --- a/xwords4/common/comms.c +++ b/xwords4/common/comms.c @@ -274,6 +274,9 @@ CommsRelayState2Str( CommsRelayState state ) CASE_STR(COMMS_RELAYSTATE_CONNECTED); CASE_STR(COMMS_RELAYSTATE_RECONNECTED); CASE_STR(COMMS_RELAYSTATE_ALLCONNECTED); +#ifdef RELAY_VIA_HTTP + CASE_STR(COMMS_RELAYSTATE_USING_HTTP); +#endif default: XP_ASSERT(0); } @@ -2375,6 +2378,19 @@ comms_isConnected( const CommsCtxt* const comms ) return result; } +#ifdef RELAY_VIA_HTTP +void +comms_gameJoined( CommsCtxt* comms, const XP_UCHAR* connname, XWHostID hid ) +{ + LOG_FUNC(); + XP_ASSERT( XP_STRLEN( connname ) + 1 < sizeof(comms->rr.connName) ); + XP_STRNCPY( comms->rr.connName, connname, sizeof(comms->rr.connName) ); + comms->rr.myHostID = hid; + comms->forceChannel = hid; + set_relay_state( comms, COMMS_RELAYSTATE_USING_HTTP ); +} +#endif + #if defined COMMS_HEARTBEAT || defined XWFEATURE_COMMSACK static void sendEmptyMsg( CommsCtxt* comms, AddressRecord* rec ) @@ -3097,14 +3113,34 @@ sendNoConn( CommsCtxt* comms, const MsgQueueElem* elem, XWHostID destID ) static XP_Bool relayConnect( CommsCtxt* comms ) { - XP_Bool success = XP_TRUE; LOG_FUNC(); - if ( addr_hasType( &comms->addr, COMMS_CONN_RELAY ) && !comms->rr.connecting ) { - comms->rr.connecting = XP_TRUE; - success = send_via_relay( comms, comms->rr.connName[0]? - XWRELAY_GAME_RECONNECT : XWRELAY_GAME_CONNECT, - comms->rr.myHostID, NULL, 0, NULL ); - comms->rr.connecting = XP_FALSE; + XP_Bool success = XP_TRUE; + if ( addr_hasType( &comms->addr, COMMS_CONN_RELAY ) ) { + if ( 0 ) { +#ifdef RELAY_VIA_HTTP + } else if ( comms->rr.connName[0] ) { + set_relay_state( comms, COMMS_RELAYSTATE_USING_HTTP ); + } else { + CommsAddrRec addr; + comms_getAddr( comms, &addr ); + DevIDType ignored; /* but should it be? */ + (*comms->procs.requestJoin)( comms->procs.closure, + util_getDevID( comms->util, &ignored ), + addr.u.ip_relay.invite, /* room */ + comms->rr.nPlayersHere, + comms->rr.nPlayersTotal, + comms_getChannelSeed(comms), + comms->util->gameInfo->dictLang ); + success = XP_FALSE; +#else + } else if ( !comms->rr.connecting ) { + comms->rr.connecting = XP_TRUE; + success = send_via_relay( comms, comms->rr.connName[0]? + XWRELAY_GAME_RECONNECT : XWRELAY_GAME_CONNECT, + comms->rr.myHostID, NULL, 0, NULL ); + comms->rr.connecting = XP_FALSE; +#endif + } } return success; } /* relayConnect */ diff --git a/xwords4/common/comms.h b/xwords4/common/comms.h index e767b6b92..31ddfae82 100644 --- a/xwords4/common/comms.h +++ b/xwords4/common/comms.h @@ -32,7 +32,7 @@ EXTERN_C_START #define CONN_ID_NONE 0L typedef XP_U32 MsgID; /* this is too big!!! PENDING */ -typedef XP_U8 XWHostID; +typedef XP_U8 XWHostID; typedef enum { COMMS_CONN_NONE /* I want errors on uninited case */ @@ -56,6 +56,9 @@ typedef enum { , COMMS_RELAYSTATE_CONNECTED , COMMS_RELAYSTATE_RECONNECTED , COMMS_RELAYSTATE_ALLCONNECTED +#ifdef RELAY_VIA_HTTP + , COMMS_RELAYSTATE_USING_HTTP /* connection state doesn't matter */ +#endif } CommsRelayState; #ifdef XWFEATURE_BLUETOOTH @@ -90,7 +93,7 @@ typedef struct _CommsAddrRec { XP_U16 port_ip; } ip; struct { - XP_UCHAR invite[MAX_INVITE_LEN + 1]; + XP_UCHAR invite[MAX_INVITE_LEN + 1]; /* room!!!! */ XP_UCHAR hostName[MAX_HOSTNAME_LEN + 1]; XP_U32 ipAddr; /* looked up from above */ XP_U16 port; @@ -135,6 +138,12 @@ typedef void (*RelayErrorProc)( void* closure, XWREASON relayErr ); typedef XP_Bool (*RelayNoConnProc)( const XP_U8* buf, XP_U16 len, const XP_UCHAR* msgNo, const XP_UCHAR* relayID, void* closure ); +# ifdef RELAY_VIA_HTTP +typedef void (*RelayRequestJoinProc)( void* closure, const XP_UCHAR* devID, + const XP_UCHAR* room, XP_U16 nPlayersHere, + XP_U16 nPlayersTotal, XP_U16 seed, + XP_U16 lang ); +# endif #endif typedef enum { @@ -161,6 +170,9 @@ typedef struct _TransportProcs { RelayConndProc rconnd; RelayErrorProc rerror; RelayNoConnProc sendNoConn; +# ifdef RELAY_VIA_HTTP + RelayRequestJoinProc requestJoin; +# endif #endif void* closure; } TransportProcs; @@ -248,6 +260,10 @@ XP_Bool comms_checkComplete( const CommsAddrRec* const addr ); XP_Bool comms_canChat( const CommsCtxt* comms ); XP_Bool comms_isConnected( const CommsCtxt* const comms ); +#ifdef RELAY_VIA_HTTP +void comms_gameJoined( CommsCtxt* comms, const XP_UCHAR* connname, XWHostID hid ); +#endif + CommsConnType addr_getType( const CommsAddrRec* addr ); void addr_setType( CommsAddrRec* addr, CommsConnType type ); void addr_addType( CommsAddrRec* addr, CommsConnType type ); diff --git a/xwords4/linux/Makefile b/xwords4/linux/Makefile index fc85f7f7c..30d0c59dc 100644 --- a/xwords4/linux/Makefile +++ b/xwords4/linux/Makefile @@ -130,6 +130,7 @@ DEFINES += -DCOMMS_XPORT_FLAGSPROC DEFINES += -DINITIAL_CLIENT_VERS=3 DEFINES += -DCOMMON_LAYOUT DEFINES += -DNATIVE_NLI +# DEFINES += -DRELAY_VIA_HTTP # MAX_ROWS controls STREAM_VERS_BIGBOARD and with it move hashing DEFINES += -DMAX_ROWS=32 diff --git a/xwords4/linux/cursesmain.c b/xwords4/linux/cursesmain.c index f76d8fe1f..c02e5fe60 100644 --- a/xwords4/linux/cursesmain.c +++ b/xwords4/linux/cursesmain.c @@ -1023,7 +1023,7 @@ curses_socket_added( void* closure, int newSock, GIOFunc func ) /* XP_ASSERT( !globals->cGlobals.relaySocket ); */ /* globals->cGlobals.relaySocket = newSock; */ #endif -} /* curses_socket_changed */ +} /* curses_socket_added */ static void curses_onGameSaved( void* closure, sqlite3_int64 rowid, @@ -1613,6 +1613,27 @@ relay_sendNoConn_curses( const XP_U8* msg, XP_U16 len, return storeNoConnMsg( &globals->cGlobals, msg, len, relayID ); } /* relay_sendNoConn_curses */ +#ifdef RELAY_VIA_HTTP +static void +onJoined( void* closure, const XP_UCHAR* connname, XWHostID hid ) +{ + LOG_FUNC(); + CursesAppGlobals* globals = (CursesAppGlobals*)closure; + CommsCtxt* comms = globals->cGlobals.game.comms; + comms_gameJoined( comms, connname, hid ); +} + +static void +relay_requestJoin_curses( void* closure, const XP_UCHAR* devID, const XP_UCHAR* room, + XP_U16 nPlayersHere, XP_U16 nPlayersTotal, + XP_U16 seed, XP_U16 lang ) +{ + CursesAppGlobals* globals = (CursesAppGlobals*)closure; + relaycon_join( globals->cGlobals.params, devID, room, nPlayersHere, nPlayersTotal, + seed, lang, onJoined, globals ); +} +#endif + static void relay_status_curses( void* closure, CommsRelayState state ) { @@ -1949,6 +1970,10 @@ cursesmain( XP_Bool isServer, LaunchParams* params ) .rconnd = relay_connd_curses, .rerror = relay_error_curses, .sendNoConn = relay_sendNoConn_curses, +#ifdef RELAY_VIA_HTTP + .requestJoin = relay_requestJoin_curses, +#endif + # ifdef COMMS_XPORT_FLAGSPROC .getFlags = curses_getFlags, # endif diff --git a/xwords4/linux/gtkboard.c b/xwords4/linux/gtkboard.c index 52eb7c820..7abae1409 100644 --- a/xwords4/linux/gtkboard.c +++ b/xwords4/linux/gtkboard.c @@ -430,13 +430,57 @@ relay_sendNoConn_gtk( const XP_U8* msg, XP_U16 len, return success; } /* relay_sendNoConn_gtk */ +static void +tryConnectToServer(CommonGlobals* cGlobals) +{ + LaunchParams* params = cGlobals->params; + XWStreamCtxt* stream = + mem_stream_make( cGlobals->util->mpool, params->vtMgr, + cGlobals, CHANNEL_NONE, + sendOnClose ); + (void)server_initClientConnection( cGlobals->game.server, + stream ); +} + +#ifdef RELAY_VIA_HTTP +static void +onJoined( void* closure, const XP_UCHAR* connname, XWHostID hid ) +{ + GtkGameGlobals* globals = (GtkGameGlobals*)closure; + XWGame* game = &globals->cGlobals.game; + CommsCtxt* comms = game->comms; + comms_gameJoined( comms, connname, hid ); + if ( hid > 1 ) { + globals->cGlobals.gi->serverRole = SERVER_ISCLIENT; + server_reset( game->server, game->comms ); + tryConnectToServer( &globals->cGlobals ); + } +} + +static void +relay_requestJoin_gtk( void* closure, const XP_UCHAR* devID, const XP_UCHAR* room, + XP_U16 nPlayersHere, XP_U16 nPlayersTotal, + XP_U16 seed, XP_U16 lang ) +{ + GtkGameGlobals* globals = (GtkGameGlobals*)closure; + LaunchParams* params = globals->cGlobals.params; + relaycon_join( params, devID, room, nPlayersHere, nPlayersTotal, seed, lang, + onJoined, globals ); +} +#endif + #ifdef COMMS_XPORT_FLAGSPROC static XP_U32 gtk_getFlags( void* closure ) { GtkGameGlobals* globals = (GtkGameGlobals*)closure; +# ifdef RELAY_VIA_HTTP + XP_USE( globals ); + return COMMS_XPORT_FLAGS_HASNOCONN; +# else return (!!globals->draw) ? COMMS_XPORT_FLAGS_NONE : COMMS_XPORT_FLAGS_HASNOCONN; +# endif } #endif @@ -456,6 +500,9 @@ setTransportProcs( TransportProcs* procs, GtkGameGlobals* globals ) procs->rconnd = relay_connd_gtk; procs->rerror = relay_error_gtk; procs->sendNoConn = relay_sendNoConn_gtk; +# ifdef RELAY_VIA_HTTP + procs->requestJoin = relay_requestJoin_gtk; +# endif #endif } @@ -665,12 +712,7 @@ createOrLoadObjects( GtkGameGlobals* globals ) } else { DeviceRole serverRole = cGlobals->gi->serverRole; if ( serverRole == SERVER_ISCLIENT ) { - XWStreamCtxt* stream = - mem_stream_make( MEMPOOL params->vtMgr, - cGlobals, CHANNEL_NONE, - sendOnClose ); - (void)server_initClientConnection( cGlobals->game.server, - stream ); + tryConnectToServer( cGlobals ); } #endif } @@ -1016,12 +1058,7 @@ new_game_impl( GtkGameGlobals* globals, XP_Bool fireConnDlg ) } if ( isClient ) { - XWStreamCtxt* stream = - mem_stream_make( MEMPOOL cGlobals->params->vtMgr, - cGlobals, CHANNEL_NONE, - sendOnClose ); - (void)server_initClientConnection( cGlobals->game.server, - stream ); + tryConnectToServer( cGlobals ); } #endif (void)server_do( cGlobals->game.server ); /* assign tiles, etc. */ diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index d5bad5e81..505bd19dd 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -84,6 +84,9 @@ static size_t writeBytes( XP_U8* buf, size_t len, const XP_U8* bytes, static size_t writeVLI( XP_U8* out, uint32_t nn ); static size_t un2vli( int nn, uint8_t* buf ); static bool vli2un( const uint8_t** inp, uint32_t* outp ); +#ifdef DEBUG +static const char* msgToStr( XWRelayReg msg ); +#endif static void* relayThread( void* arg ); @@ -92,7 +95,11 @@ typedef struct _WriteState { size_t curSize; } WriteState; -typedef enum { POST, QUERY, } TaskType; +typedef enum { +#ifdef RELAY_VIA_HTTP + JOIN, +#endif + POST, QUERY, } TaskType; typedef struct _RelayTask { TaskType typ; @@ -101,6 +108,18 @@ typedef struct _RelayTask { WriteState ws; XP_U32 ctime; union { +#ifdef RELAY_VIA_HTTP + struct { + XP_U16 lang; + XP_U16 nHere; + XP_U16 nTotal; + XP_U16 seed; + XP_UCHAR devID[64]; + XP_UCHAR room[MAX_INVITE_LEN + 1]; + OnJoinedProc proc; + void* closure; + } join; +#endif struct { XP_U8* msgbuf; XP_U16 len; @@ -114,6 +133,9 @@ typedef struct _RelayTask { static RelayTask* makeRelayTask( RelayConStorage* storage, TaskType typ ); static void freeRelayTask(RelayTask* task); +#ifdef RELAY_VIA_HTTP +static void handleJoin( RelayTask* task ); +#endif static void handlePost( RelayTask* task ); static void handleQuery( RelayTask* task ); static void addToGotData( RelayTask* task ); @@ -290,7 +312,7 @@ relaycon_reg( LaunchParams* params, const XP_UCHAR* rDevID, indx += addVLIStr( &tmpbuf[indx], sizeof(tmpbuf) - indx, "linux box" ); indx += addVLIStr( &tmpbuf[indx], sizeof(tmpbuf) - indx, "linux version" ); - sendIt( storage, tmpbuf, indx, 2.0 ); + sendIt( storage, tmpbuf, indx, 0.5 ); } void @@ -327,7 +349,7 @@ relaycon_invite( LaunchParams* params, XP_U32 destDevID, indx += writeBytes( &tmpbuf[indx], sizeof(tmpbuf) - indx, ptr, len ); stream_destroy( stream ); - sendIt( storage, tmpbuf, indx, 2.0 ); + sendIt( storage, tmpbuf, indx, 0.5 ); LOG_RETURN_VOID(); } @@ -344,7 +366,7 @@ relaycon_send( LaunchParams* params, const XP_U8* buf, XP_U16 buflen, indx += writeHeader( storage, tmpbuf, XWPDEV_MSG ); indx += writeLong( &tmpbuf[indx], sizeof(tmpbuf) - indx, gameToken ); indx += writeBytes( &tmpbuf[indx], sizeof(tmpbuf) - indx, buf, buflen ); - nSent = sendIt( storage, tmpbuf, indx, 2.0 ); + nSent = sendIt( storage, tmpbuf, indx, 0.5 ); if ( nSent > buflen ) { nSent = buflen; } @@ -372,7 +394,7 @@ relaycon_sendnoconn( LaunchParams* params, const XP_U8* buf, XP_U16 buflen, (const XP_U8*)relayID, idLen ); tmpbuf[indx++] = '\n'; indx += writeBytes( &tmpbuf[indx], sizeof(tmpbuf) - indx, buf, buflen ); - nSent = sendIt( storage, tmpbuf, indx, 2.0 ); + nSent = sendIt( storage, tmpbuf, indx, 0.5 ); if ( nSent > buflen ) { nSent = buflen; } @@ -391,7 +413,7 @@ relaycon_requestMsgs( LaunchParams* params, const XP_UCHAR* devID ) indx += writeHeader( storage, tmpbuf, XWPDEV_RQSTMSGS ); indx += addVLIStr( &tmpbuf[indx], sizeof(tmpbuf) - indx, devID ); - sendIt( storage, tmpbuf, indx, 2.0 ); + sendIt( storage, tmpbuf, indx, 0.5 ); } void @@ -418,12 +440,19 @@ onMainThread( RelayConStorage* storage ) static const gchar* taskName( const RelayTask* task ) { + const char* str; +# define CASE_STR(c) case c: str = #c; break switch (task->typ) { - case POST: return "POST"; - case QUERY: return "QUERY"; + CASE_STR(POST); + CASE_STR(QUERY); +#ifdef RELAY_VIA_HTTP + CASE_STR(JOIN); +#endif default: XP_ASSERT(0); - return NULL; + str = NULL; } +#undef CASE_STR + return str; } static gchar* @@ -450,6 +479,7 @@ listTasks( GSList* tasks ) static void* relayThread( void* arg ) { + LOG_FUNC(); RelayConStorage* storage = (RelayConStorage*)arg; for ( ; ; ) { pthread_mutex_lock( &storage->relayMutex ); @@ -472,6 +502,11 @@ relayThread( void* arg ) g_free( strs ); switch ( task->typ ) { +#ifdef RELAY_VIA_HTTP + case JOIN: + handleJoin( task ); + break; +#endif case POST: handlePost( task ); break; @@ -535,6 +570,28 @@ freeRelayTask( RelayTask* task ) g_free( task ); } +#ifdef RELAY_VIA_HTTP +void +relaycon_join( LaunchParams* params, const XP_UCHAR* devID, const XP_UCHAR* room, + XP_U16 nPlayersHere, XP_U16 nPlayersTotal, XP_U16 seed, XP_U16 lang, + OnJoinedProc proc, void* closure ) +{ + LOG_FUNC(); + RelayConStorage* storage = getStorage( params ); + XP_ASSERT( onMainThread(storage) ); + RelayTask* task = makeRelayTask( storage, JOIN ); + task->u.join.nHere = nPlayersHere; + XP_STRNCPY( task->u.join.devID, devID, sizeof(task->u.join.devID) ); + XP_STRNCPY( task->u.join.room, room, sizeof(task->u.join.room) ); + task->u.join.nTotal = nPlayersTotal; + task->u.join.lang = lang; + task->u.join.seed = seed; + task->u.join.proc = proc; + task->u.join.closure = closure; + addTask( storage, task ); +} +#endif + static void sendAckIf( RelayConStorage* storage, const MsgHeader* header ) { @@ -556,6 +613,8 @@ process( RelayConStorage* storage, XP_U8* buf, ssize_t nRead ) if ( readHeader( &ptr, &header ) ) { sendAckIf( storage, &header ); + XP_LOGF( "%s(): got %s", __func__, msgToStr(header.cmd) ); + switch( header.cmd ) { case XWPDEV_REGRSP: { uint32_t len; @@ -605,7 +664,7 @@ process( RelayConStorage* storage, XP_U8* buf, ssize_t nRead ) assert( 0 ); } XP_USE( packetID ); - XP_LOGF( "got ack for packetID %d", packetID ); + XP_LOGF( "%s(): got ack for packetID %d", __func__, packetID ); break; } case XWPDEV_ALERT: { @@ -735,10 +794,34 @@ hostNameToIP( const XP_UCHAR* name ) return ip; } -static gboolean -onGotPostData( gpointer user_data ) +#ifdef RELAY_VIA_HTTP +static void +onGotJoinData( RelayTask* task ) +{ + LOG_FUNC(); + RelayConStorage* storage = task->storage; + XP_ASSERT( onMainThread(storage) ); + if ( !!task->ws.ptr ) { + XP_LOGF( "%s(): got json? %s", __func__, task->ws.ptr ); + json_object* reply = json_tokener_parse( task->ws.ptr ); + json_object* jConnname = NULL; + json_object* jHID = NULL; + if ( json_object_object_get_ex( reply, "connname", &jConnname ) + && json_object_object_get_ex( reply, "hid", &jHID ) ) { + const char* connname = json_object_get_string( jConnname ); + XWHostID hid = json_object_get_int( jHID ); + (*task->u.join.proc)( task->u.join.closure, connname, hid ); + } + json_object_put( jConnname ); + json_object_put( jHID ); + } + freeRelayTask( task ); +} +#endif + +static gboolean +onGotPostData( RelayTask* task ) { - RelayTask* task = (RelayTask*)user_data; RelayConStorage* storage = task->storage; /* Now pull any data from the reply */ // got "{"status": "ok", "dataLen": 14, "data": "AYQDiDAyMUEzQ0MyADw=", "err": "none"}" @@ -767,6 +850,23 @@ onGotPostData( gpointer user_data ) return FALSE; } +#ifdef RELAY_VIA_HTTP +static void +handleJoin( RelayTask* task ) +{ + LOG_FUNC(); + runWitCurl( task, "join", + "devID", json_object_new_string( task->u.join.devID ), + "room", json_object_new_string( task->u.join.room ), + "seed", json_object_new_int( task->u.join.seed ), + "lang", json_object_new_int( task->u.join.lang ), + "nInGame", json_object_new_int( task->u.join.nTotal ), + "nHere", json_object_new_int( task->u.join.nHere ), + NULL ); + addToGotData( task ); +} +#endif + static void handlePost( RelayTask* task ) { @@ -799,9 +899,8 @@ post( RelayConStorage* storage, const XP_U8* msgbuf, XP_U16 len, float timeout ) } static gboolean -onGotQueryData( gpointer user_data ) +onGotQueryData( RelayTask* task ) { - RelayTask* task = (RelayTask*)user_data; RelayConStorage* storage = task->storage; XP_Bool foundAny = false; if ( !!task->ws.ptr ) { @@ -902,6 +1001,11 @@ gotDataTimer(gpointer user_data) break; } else { switch ( task->typ ) { +#ifdef RELAY_VIA_HTTP + case JOIN: + onGotJoinData( task ); + break; +#endif case POST: onGotPostData( task ); break; @@ -1184,3 +1288,33 @@ vli2un( const uint8_t** inp, uint32_t* outp ) } return success; } + +#ifdef DEBUG +static const char* +msgToStr( XWRelayReg msg ) +{ + const char* str; +# define CASE_STR(c) case c: str = #c; break + switch( msg ) { + CASE_STR(XWPDEV_UNAVAIL); + CASE_STR(XWPDEV_REG); + CASE_STR(XWPDEV_REGRSP); + CASE_STR(XWPDEV_INVITE); + CASE_STR(XWPDEV_KEEPALIVE); + CASE_STR(XWPDEV_HAVEMSGS); + CASE_STR(XWPDEV_RQSTMSGS); + CASE_STR(XWPDEV_MSG); + CASE_STR(XWPDEV_MSGNOCONN); + CASE_STR(XWPDEV_MSGRSP); + CASE_STR(XWPDEV_BADREG); + CASE_STR(XWPDEV_ALERT); // should not receive this.... + CASE_STR(XWPDEV_ACK); + CASE_STR(XWPDEV_DELGAME); + default: + str = ""; + break; + } +# undef CASE_STR + return str; +} +#endif diff --git a/xwords4/linux/relaycon.h b/xwords4/linux/relaycon.h index 494c86bad..aafab5133 100644 --- a/xwords4/linux/relaycon.h +++ b/xwords4/linux/relaycon.h @@ -60,4 +60,12 @@ XP_U32 makeClientToken( sqlite3_int64 rowid, XP_U16 seed ); void rowidFromToken( XP_U32 clientToken, sqlite3_int64* rowid, XP_U16* seed ); void relaycon_checkMsgs( LaunchParams* params ); + +# ifdef RELAY_VIA_HTTP +typedef void (*OnJoinedProc)( void* closure, const XP_UCHAR* connname, XWHostID hid ); +void relaycon_join( LaunchParams* params, const XP_UCHAR* devID, const XP_UCHAR* room, + XP_U16 nPlayersHere, XP_U16 nPlayersTotal, XP_U16 seed, + XP_U16 lang, OnJoinedProc proc, void* closure ); +# endif + #endif diff --git a/xwords4/relay/scripts/showinplay.sh b/xwords4/relay/scripts/showinplay.sh index a1cc65a6d..957f30470 100755 --- a/xwords4/relay/scripts/showinplay.sh +++ b/xwords4/relay/scripts/showinplay.sh @@ -54,7 +54,7 @@ echo "; relay pid[s]: $(pidof xwrelay)" echo "Row count:" $(psql -t xwgames -c "select count(*) FROM games $QUERY;") # Games -echo "SELECT dead as d,connname,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ +echo "SELECT dead as d,connname,room,lang as lg,clntVers as cv ,ntotal as t,njoined as nj,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ "FROM games $QUERY ORDER BY NOT dead, ctime DESC LIMIT $LIMIT;" \ | psql xwgames From acf668ae8a0f57b4c54e577062b8a6c0315ee6a8 Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 10 Nov 2017 22:16:15 -0800 Subject: [PATCH 074/138] add new prefs controlling web api usage Add, but don't use yet, prefs giving relay.py path and checkbox whether to use the new APIs. --- .../app/src/main/res/values/common_rsrc.xml | 3 +++ .../app/src/main/res/values/strings.xml | 3 +++ .../android/app/src/main/res/xml/xwprefs.xml | 23 +++++++++++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/xwords4/android/app/src/main/res/values/common_rsrc.xml b/xwords4/android/app/src/main/res/values/common_rsrc.xml index 2ef326001..cb123d833 100644 --- a/xwords4/android/app/src/main/res/values/common_rsrc.xml +++ b/xwords4/android/app/src/main/res/values/common_rsrc.xml @@ -36,7 +36,9 @@ key_relay_host key_relay_port2 + key_relay_via_http key_update_url + key_relay_url key_update_prerel key_proxy_port key_sms_port @@ -149,6 +151,7 @@ http://eehouse.org/and_wordlists http://eehouse.org/xw4/info.py + http://eehouse.org/xw4/relay.py diff --git a/xwords4/android/app/src/main/res/values/strings.xml b/xwords4/android/app/src/main/res/values/strings.xml index 4ba195973..d0cea03c3 100644 --- a/xwords4/android/app/src/main/res/values/strings.xml +++ b/xwords4/android/app/src/main/res/values/strings.xml @@ -2474,6 +2474,8 @@ For debugging You should never need these... Relay host + Use Web APIs + (instead of custom protocol to ports below) Wordlist download URL Enable logging (release builds only) @@ -2513,6 +2515,7 @@ gameid Pending packet count Update checks URL + URL for relay web API Fetch default wordlist for language Don\'t try a second time diff --git a/xwords4/android/app/src/main/res/xml/xwprefs.xml b/xwords4/android/app/src/main/res/xml/xwprefs.xml index 6b30a70fb..1ede6a0b9 100644 --- a/xwords4/android/app/src/main/res/xml/xwprefs.xml +++ b/xwords4/android/app/src/main/res/xml/xwprefs.xml @@ -415,11 +415,29 @@ + + + + + + - Date: Sat, 11 Nov 2017 08:10:48 -0800 Subject: [PATCH 075/138] fix failure to notify relay of half-game deletion Wasn't detecting as a relay game something registered on relay but not yet joined by another device, a common case! --- .../main/java/org/eehouse/android/xw4/GameUtils.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/GameUtils.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/GameUtils.java index dba4c57de..92c476792 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/GameUtils.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/GameUtils.java @@ -1196,7 +1196,7 @@ public class GameUtils { for ( CommsConnType typ : conTypes ) { switch ( typ ) { case COMMS_CONN_RELAY: - tellRelayDied( context, summary, informNow ); + // see below break; case COMMS_CONN_BT: BTService.gameDied( context, addr.bt_btAddr, gameID ); @@ -1210,6 +1210,14 @@ public class GameUtils { } } } + + // comms doesn't have a relay address for us until the game's + // in play (all devices registered, at least.) To enable + // deleting on relay half-games that we created but nobody + // joined, special-case this one. + if ( summary.inRelayGame() ) { + tellRelayDied( context, summary, informNow ); + } gamePtr.release(); } From 0831bd8022a833aa7fa6ea42c32b51a79e78ddf8 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 08:22:52 -0800 Subject: [PATCH 076/138] talk to relay.py/kill, first web api use from android And it seems to work! --- .../eehouse/android/xw4/DictsDelegate.java | 4 +- .../org/eehouse/android/xw4/NetUtils.java | 75 ++++++++++++++++--- .../android/xw4/UpdateCheckReceiver.java | 3 +- .../java/org/eehouse/android/xw4/XWPrefs.java | 10 +++ 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/DictsDelegate.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/DictsDelegate.java index 726fe693a..7f1832fe5 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/DictsDelegate.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/DictsDelegate.java @@ -1238,7 +1238,7 @@ public class DictsDelegate extends ListDelegateBase // parse less data String name = null; String proc = String.format( "listDicts?lc=%s", m_lc ); - HttpURLConnection conn = NetUtils.makeHttpConn( m_context, proc ); + HttpURLConnection conn = NetUtils.makeHttpUpdateConn( m_context, proc ); if ( null != conn ) { JSONObject theOne = null; String langName = null; @@ -1320,7 +1320,7 @@ public class DictsDelegate extends ListDelegateBase public Boolean doInBackground( Void... unused ) { boolean success = false; - HttpURLConnection conn = NetUtils.makeHttpConn( m_context, "listDicts" ); + HttpURLConnection conn = NetUtils.makeHttpUpdateConn( m_context, "listDicts" ); if ( null != conn ) { String json = NetUtils.runConn( conn, new JSONObject() ); if ( !isCancelled() ) { diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java index 62ff478da..147cc93d2 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java @@ -25,6 +25,8 @@ import android.text.TextUtils; import junit.framework.Assert; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import java.io.BufferedInputStream; @@ -88,7 +90,41 @@ public class NetUtils { m_obits = obits; } - public void run() { + public void run() + { + if ( XWPrefs.getPreferWebAPI( m_context ) ) { + runViaWeb(); + } else { + runWithProxySocket(); + } + } + + private void runViaWeb() + { + try { + JSONArray params = new JSONArray(); + for ( int ii = 0; ii < m_obits.length; ++ii ) { + JSONObject one = new JSONObject(); + one.put( "relayID", m_obits[ii].m_relayID ); + one.put( "seed", m_obits[ii].m_seed ); + params.put( one ); + } + HttpURLConnection conn = makeHttpRelayConn( m_context, "kill" ); + Log.d( TAG, "kill params: %s", params.toString() ); + String resStr = runConn( conn, params ); + Log.d( TAG, "kill => %s", resStr ); + + JSONObject result = new JSONObject( resStr ); + if ( 0 == result.optInt( "err", -1 ) ) { + DBUtils.clearObits( m_context, m_obits ); + } + } catch ( JSONException ex ) { + Assert.assertFalse( BuildConfig.DEBUG ); + } + } + + private void runWithProxySocket() + { Socket socket = makeProxySocket( m_context, 10000 ); if ( null != socket ) { int strLens = 0; @@ -139,8 +175,7 @@ public class NetUtils { { DBUtils.Obit[] obits = DBUtils.listObits( context ); if ( null != obits && 0 < obits.length ) { - InformThread thread = new InformThread( context, obits ); - thread.start(); + new InformThread( context, obits ).start(); } } @@ -214,14 +249,26 @@ public class NetUtils { return host; } - protected static HttpURLConnection makeHttpConn( Context context, - String proc ) + protected static HttpURLConnection makeHttpRelayConn( Context context, + String proc ) + { + String url = XWPrefs.getDefaultRelayUrl( context ); + return makeHttpConn( context, url, proc ); + } + + protected static HttpURLConnection makeHttpUpdateConn( Context context, + String proc ) + { + String url = XWPrefs.getDefaultUpdateUrl( context ); + return makeHttpConn( context, url, proc ); + } + + private static HttpURLConnection makeHttpConn( Context context, + String path, String proc ) { HttpURLConnection result = null; try { - String url = String.format( "%s/%s", - XWPrefs.getDefaultUpdateUrl( context ), - proc ); + String url = String.format( "%s/%s", path, proc ); result = (HttpURLConnection)new URL(url).openConnection(); } catch ( java.net.MalformedURLException mue ) { Assert.assertNull( result ); @@ -233,11 +280,21 @@ public class NetUtils { return result; } + protected static String runConn( HttpURLConnection conn, JSONArray param ) + { + return runConn( conn, param.toString() ); + } + protected static String runConn( HttpURLConnection conn, JSONObject param ) + { + return runConn( conn, param.toString() ); + } + + private static String runConn( HttpURLConnection conn, String param ) { String result = null; Map params = new HashMap(); - params.put( k_PARAMS, param.toString() ); + params.put( k_PARAMS, param ); String paramsString = getPostDataString( params ); if ( null != paramsString ) { diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/UpdateCheckReceiver.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/UpdateCheckReceiver.java index 1dbd678bc..75efe5abe 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/UpdateCheckReceiver.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/UpdateCheckReceiver.java @@ -258,7 +258,8 @@ public class UpdateCheckReceiver extends BroadcastReceiver { @Override protected String doInBackground( Void... unused ) { - HttpURLConnection conn = NetUtils.makeHttpConn( m_context, "getUpdates" ); + HttpURLConnection conn + = NetUtils.makeHttpUpdateConn( m_context, "getUpdates" ); String json = null; if ( null != conn ) { json = NetUtils.runConn( conn, m_params ); diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWPrefs.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWPrefs.java index 80d16b1d2..813fcede8 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWPrefs.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWPrefs.java @@ -115,6 +115,16 @@ public class XWPrefs { return getPrefsString( context, R.string.key_update_url ); } + public static String getDefaultRelayUrl( Context context ) + { + return getPrefsString( context, R.string.key_relay_url ); + } + + public static boolean getPreferWebAPI( Context context ) + { + return getPrefsBoolean( context, R.string.key_relay_via_http, false ); + } + public static int getDefaultProxyPort( Context context ) { String val = getPrefsString( context, R.string.key_proxy_port ); From 4a3133924a580a6c0bb459226c04127a156132ea Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 09:14:35 -0800 Subject: [PATCH 077/138] name change for clarity --- .../main/java/org/eehouse/android/xw4/RelayService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index 740aa945d..2423c8bce 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -449,7 +449,7 @@ public class RelayService extends XWService case TIMER_FIRED: if ( !NetStateCache.netAvail( this ) ) { Log.w( TAG, "not connecting: no network" ); - } else if ( startFetchThreadIf() ) { + } else if ( startFetchThreadIfNotUDP() ) { // do nothing } else if ( registerWithRelayIfNot() ) { requestMessages(); @@ -510,9 +510,9 @@ public class RelayService extends XWService } } - private boolean startFetchThreadIf() + private boolean startFetchThreadIfNotUDP() { - // DbgUtils.logf( "startFetchThreadIf()" ); + // DbgUtils.logf( "startFetchThreadIfNotUDP()" ); boolean handled = relayEnabled( this ) && !XWApp.UDP_ENABLED; if ( handled && null == m_fetchThread ) { m_fetchThread = new Thread( null, new Runnable() { @@ -1247,7 +1247,7 @@ public class RelayService extends XWService registerWithRelay(); } else { stopUDPThreadsIf(); - startFetchThreadIf(); + startFetchThreadIfNotUDP(); } } From b9800b22f577ffc91e2da2e1ba766a56551d7acf Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 09:56:20 -0800 Subject: [PATCH 078/138] RelayService uses web instead of UDP! As expected, moves are no longer received instantly because the UDP socket isn't available for the relay to write too once the URL handler (relay.py) finishes. --- .../org/eehouse/android/xw4/RelayService.java | 103 ++++++++++++------ xwords4/android/scripts/relay.py | 22 ++-- 2 files changed, 84 insertions(+), 41 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index 2423c8bce..a5c0691f4 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -38,6 +38,10 @@ import org.eehouse.android.xw4.jni.UtilCtxt.DevIDType; import org.eehouse.android.xw4.jni.XwJNI; import org.eehouse.android.xw4.loc.LocUtils; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -46,6 +50,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.DatagramPacket; import java.net.DatagramSocket; +import java.net.HttpURLConnection; import java.net.InetAddress; import java.net.Socket; import java.util.ArrayList; @@ -621,33 +626,24 @@ public class RelayService extends XWService break; } - try { - DatagramPacket outPacket = outData.assemble(); - m_UDPSocket.send( outPacket ); - int pid = outData.m_packetID; - Log.d( TAG, "Sent udp packet, cmd=%s, id=%d," - + " of length %d", - outData.m_cmd.toString(), - pid, outPacket.getLength()); - synchronized( s_packetsSent ) { - s_packetsSent.add( pid ); - } - resetExitTimer(); - ConnStatusHandler.showSuccessOut(); - } catch ( java.net.SocketException se ) { - Log.ex( TAG, se ); - Log.i( TAG, "Restarting threads to force" - + " new socket" ); - m_handler.post( new Runnable() { - public void run() { - stopUDPThreadsIf(); - } - } ); - } catch ( java.io.IOException ioe ) { - Log.ex( TAG, ioe ); - } catch ( NullPointerException npe ) { - Log.w( TAG, "network problem; dropping packet" ); + int sentLen; + byte[] data = outData.assemble(); + if ( XWPrefs.getPreferWebAPI( RelayService.this ) ) { + sentLen = sendViaWeb( data ); + } else { + sentLen = sendViaUDP( data ); } + + int pid = outData.m_packetID; + Log.d( TAG, "Sent udp packet, cmd=%s, id=%d," + + " of length %d", + outData.m_cmd.toString(), + pid, sentLen); + synchronized( s_packetsSent ) { + s_packetsSent.add( pid ); + } + resetExitTimer(); + ConnStatusHandler.showSuccessOut(); } Log.i( TAG, "write thread exiting" ); } @@ -659,6 +655,51 @@ public class RelayService extends XWService } } + private int sendViaWeb( byte[] data ) + { + try { + JSONObject params = new JSONObject(); + String b64Data = Utils.base64Encode(data); + params.put( "data", b64Data ); + HttpURLConnection conn = NetUtils.makeHttpRelayConn(this, "post"); + String result = NetUtils.runConn(conn, params); + JSONObject resultObj = new JSONObject( result ); + JSONArray resData = resultObj.getJSONArray( "data" ); + for ( int ii = 0; ii < resData.length(); ++ii ) { + byte[] datum = Utils.base64Decode( resData.getString( ii ) ); + // PENDING: skip ack or not + gotPacket( datum, false ); + } + } catch ( JSONException ex ) { + Assert.assertFalse( BuildConfig.DEBUG ); + } + return data.length; + } + + private int sendViaUDP( byte[] data ) + { + int sentLen = -1; + try { + DatagramPacket outPacket = new DatagramPacket( data, data.length ); + m_UDPSocket.send( outPacket ); + sentLen = outPacket.getLength(); + } catch ( java.net.SocketException se ) { + Log.ex( TAG, se ); + Log.i( TAG, "Restarting threads to force" + + " new socket" ); + m_handler.post( new Runnable() { + public void run() { + stopUDPThreadsIf(); + } + } ); + } catch ( java.io.IOException ioe ) { + Log.ex( TAG, ioe ); + } catch ( NullPointerException npe ) { + Log.w( TAG, "network problem; dropping packet" ); + } + return sentLen; + } + private void stopUDPThreadsIf() { if ( null != m_UDPWriteThread ) { @@ -1441,13 +1482,13 @@ public class RelayService extends XWService return result; } - public DatagramPacket assemble() + public byte[] assemble() { - byte[] dest = new byte[getLength()]; - System.arraycopy( m_header, 0, dest, 0, m_header.length ); + byte[] data = new byte[getLength()]; + System.arraycopy( m_header, 0, data, 0, m_header.length ); byte[] basData = m_bas.toByteArray(); - System.arraycopy( basData, 0, dest, m_header.length, basData.length ); - return new DatagramPacket( dest, dest.length ); + System.arraycopy( basData, 0, data, m_header.length, basData.length ); + return data; } private void makeHeader() diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 8ae901fb1..4985a20cf 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -143,8 +143,8 @@ def kill(req, params): # winds up in handle_udp_packet() in xwrelay.cpp def post(req, params, timeoutSecs = 1.0): err = 'none' - jobj = json.loads(params) - data = base64.b64decode(jobj['data']) + params = json.loads(params) + data = base64.b64decode(params['data']) udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) udpSock.settimeout(float(timeoutSecs)) # seconds @@ -161,18 +161,20 @@ def post(req, params, timeoutSecs = 1.0): err = 'timeout' break - jobj = {'err' : err, 'data' : responses} - return json.dumps(jobj) + result = {'err' : err, 'data' : responses} + return json.dumps(result) -def query(req, ids, timeoutSecs = 5.0): - print('ids', ids) - ids = json.loads(ids) +def query(req, params): + print('params', params) + params = json.loads(params) + ids = params['ids'] + timeoutSecs = 'timeoutSecs' in params and float(params['timeoutSecs']) or 2.0 idsLen = 0 for id in ids: idsLen += len(id) tcpSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - tcpSock.settimeout(float(timeoutSecs)) + tcpSock.settimeout(timeoutSecs) tcpSock.connect(('127.0.0.1', 10998)) lenShort = 2 + idsLen + len(ids) + 2 @@ -217,8 +219,8 @@ def main(): if len(sys.argv) > 1: cmd = sys.argv[1] args = sys.argv[2:] - if cmd == 'query': - result = query(None, json.dumps(args)) + if cmd == 'query' and len(args) > 0: + result = query(None, json.dumps({'ids':args})) elif cmd == 'post': # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } # params = json.dumps(params) From f428538afc63632954e576c6a5e308284f86ae76 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 10:47:26 -0800 Subject: [PATCH 079/138] send an array of packets to post() Somethimes there are several on the queue, so why not send all at once. Note: this will break the linux implementation which I'll fix next. --- .../org/eehouse/android/xw4/RelayService.java | 113 ++++++++++++------ xwords4/android/scripts/relay.py | 6 +- 2 files changed, 78 insertions(+), 41 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index a5c0691f4..61aa48b25 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -57,6 +57,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.concurrent.LinkedBlockingQueue; public class RelayService extends XWService @@ -613,35 +614,31 @@ public class RelayService extends XWService public void run() { Log.i( TAG, "write thread starting" ); for ( ; ; ) { - PacketData outData; + List dataList = null; try { - outData = m_queue.take(); + PacketData outData = m_queue.take(); + if ( null != outData ) { + dataList = new ArrayList<>(); + dataList.add(outData); + m_queue.drainTo(dataList); + } + Log.d( TAG, "got %d packets; %d more left", dataList.size(), m_queue.size()); } catch ( InterruptedException ie ) { Log.w( TAG, "write thread killed" ); break; } - if ( null == outData - || 0 == outData.getLength() ) { + if ( null == dataList ) { Log.i( TAG, "stopping write thread" ); break; } int sentLen; - byte[] data = outData.assemble(); if ( XWPrefs.getPreferWebAPI( RelayService.this ) ) { - sentLen = sendViaWeb( data ); + sentLen = sendViaWeb( dataList ); } else { - sentLen = sendViaUDP( data ); + sentLen = sendViaUDP( dataList ); } - int pid = outData.m_packetID; - Log.d( TAG, "Sent udp packet, cmd=%s, id=%d," - + " of length %d", - outData.m_cmd.toString(), - pid, sentLen); - synchronized( s_packetsSent ) { - s_packetsSent.add( pid ); - } resetExitTimer(); ConnStatusHandler.showSuccessOut(); } @@ -655,16 +652,28 @@ public class RelayService extends XWService } } - private int sendViaWeb( byte[] data ) + private int sendViaWeb( List packets ) { + Log.d( TAG, "sendViaWeb(): sending %d at once", packets.size() ); + int sentLen = 0; try { + JSONArray dataArray = new JSONArray(); + for ( PacketData packet : packets ) { + byte[] datum = packet.assemble(); + dataArray.put( Utils.base64Encode(datum) ); + sentLen += datum.length; + } JSONObject params = new JSONObject(); - String b64Data = Utils.base64Encode(data); - params.put( "data", b64Data ); - HttpURLConnection conn = NetUtils.makeHttpRelayConn(this, "post"); + params.put( "data", dataArray ); + + HttpURLConnection conn = NetUtils.makeHttpRelayConn( this, "post" ); String result = NetUtils.runConn(conn, params); JSONObject resultObj = new JSONObject( result ); JSONArray resData = resultObj.getJSONArray( "data" ); + Log.d( TAG, "sendViaWeb(): got %d replies", resData.length() ); + + noteSent( packets ); // before we process the acks below :-) + for ( int ii = 0; ii < resData.length(); ++ii ) { byte[] datum = Utils.base64Decode( resData.getString( ii ) ); // PENDING: skip ack or not @@ -673,33 +682,59 @@ public class RelayService extends XWService } catch ( JSONException ex ) { Assert.assertFalse( BuildConfig.DEBUG ); } - return data.length; + return sentLen; } - private int sendViaUDP( byte[] data ) + private int sendViaUDP( List packets ) { - int sentLen = -1; - try { - DatagramPacket outPacket = new DatagramPacket( data, data.length ); - m_UDPSocket.send( outPacket ); - sentLen = outPacket.getLength(); - } catch ( java.net.SocketException se ) { - Log.ex( TAG, se ); - Log.i( TAG, "Restarting threads to force" - + " new socket" ); - m_handler.post( new Runnable() { - public void run() { - stopUDPThreadsIf(); - } - } ); - } catch ( java.io.IOException ioe ) { - Log.ex( TAG, ioe ); - } catch ( NullPointerException npe ) { - Log.w( TAG, "network problem; dropping packet" ); + int sentLen = 0; + for ( PacketData packet : packets ) { + boolean getOut = true; + byte[] data = packet.assemble(); + try { + DatagramPacket udpPacket = new DatagramPacket( data, data.length ); + m_UDPSocket.send( udpPacket ); + sentLen += udpPacket.getLength(); + noteSent( packet ); + getOut = false; + } catch ( java.net.SocketException se ) { + Log.ex( TAG, se ); + Log.i( TAG, "Restarting threads to force" + + " new socket" ); + m_handler.post( new Runnable() { + public void run() { + stopUDPThreadsIf(); + } + } ); + } catch ( java.io.IOException ioe ) { + Log.ex( TAG, ioe ); + } catch ( NullPointerException npe ) { + Log.w( TAG, "network problem; dropping packet" ); + } + if ( getOut ) { + break; + } } return sentLen; } + private void noteSent( PacketData packet ) + { + int pid = packet.m_packetID; + Log.d( TAG, "Sent [udp?] packet: cmd=%s, id=%d", + packet.m_cmd.toString(), pid); + synchronized( s_packetsSent ) { + s_packetsSent.add( pid ); + } + } + + private void noteSent( List packets ) + { + for ( PacketData packet : packets ) { + noteSent( packet ); + } + } + private void stopUDPThreadsIf() { if ( null != m_UDPWriteThread ) { diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index 4985a20cf..cc45dcfbd 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -144,12 +144,14 @@ def kill(req, params): def post(req, params, timeoutSecs = 1.0): err = 'none' params = json.loads(params) - data = base64.b64decode(params['data']) + data = params['data'] + binData = [base64.b64decode(datum) for datum in data] udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) udpSock.settimeout(float(timeoutSecs)) # seconds addr = ("127.0.0.1", 10997) - udpSock.sendto(data, addr) + for binDatum in binData: + udpSock.sendto(binDatum, addr) responses = [] while True: From e2c64809925d2b621d590b2f8636c314728585c5 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 12:57:33 -0800 Subject: [PATCH 080/138] fix linux client everything takes one param now, and post sends an array --- xwords4/android/scripts/relay.py | 3 +- xwords4/linux/relaycon.c | 52 +++++++++++++++----------------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py index cc45dcfbd..7ca6c2fa6 100755 --- a/xwords4/android/scripts/relay.py +++ b/xwords4/android/scripts/relay.py @@ -141,10 +141,11 @@ def kill(req, params): return json.dumps(result) # winds up in handle_udp_packet() in xwrelay.cpp -def post(req, params, timeoutSecs = 1.0): +def post(req, params): err = 'none' params = json.loads(params) data = params['data'] + timeoutSecs = 'timeoutSecs' in params and params['timeoutSecs'] or 1.0 binData = [base64.b64decode(datum) for datum in data] udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 505bd19dd..2e6a6c2ab 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -162,10 +162,10 @@ write_callback(void *contents, size_t size, size_t nmemb, void* data) return newLength; } -static void -addJsonParams( CURL* curl, va_list ap ) +static gchar* +mkJsonParams( CURL* curl, va_list ap ) { - gchar* buf = NULL; + json_object* params = json_object_new_object(); for ( ; ; ) { const char* name = va_arg(ap, const char*); if ( !name ) { @@ -174,31 +174,20 @@ addJsonParams( CURL* curl, va_list ap ) json_object* param = va_arg(ap, json_object*); XP_ASSERT( !!param ); - const char* asStr = json_object_get_string( param ); - XP_LOGF( "%s: adding param (with name %s): %s", __func__, name, asStr ); - - char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); - gchar* tmp = g_strdup_printf( "%s=%s", name, curl_params ); - curl_free( curl_params ); - - if ( !buf ) { - buf = tmp; - } else { - gchar* cur = buf; - buf = g_strdup_printf( "%s&%s", cur, tmp ); - g_free( tmp ); - g_free( cur ); - } - json_object_put( param ); + json_object_object_add( params, name, param ); + // XP_LOGF( "%s: adding param (with name %s): %s", __func__, name, json_object_get_string(param) ); } - XP_LOGF( "%s(): setting params: %s", __func__, buf ); - curl_easy_setopt( curl, CURLOPT_POSTFIELDS, buf ); - curl_easy_setopt( curl, CURLOPT_POSTFIELDSIZE, (long)strlen(buf) ); - // Can't free the buf!! Well, maybe after the send... - // g_free( buf ); + const char* asStr = json_object_get_string( params ); + char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); + gchar* result = g_strdup_printf( "params=%s", curl_params ); + curl_free( curl_params ); + json_object_put( params ); + return result; } +/* relay.py's methods all take one json object param "param" So we wrap + everything else in that then send it. */ static XP_Bool runWitCurl( RelayTask* task, const gchar* proc, ...) { @@ -214,9 +203,12 @@ runWitCurl( RelayTask* task, const gchar* proc, ...) va_list ap; va_start( ap, proc ); - addJsonParams( curl, ap ); + gchar* params = mkJsonParams( curl, ap ); va_end( ap ); + curl_easy_setopt( curl, CURLOPT_POSTFIELDS, params ); + curl_easy_setopt( curl, CURLOPT_POSTFIELDSIZE, (long)strlen(params) ); + curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, write_callback ); curl_easy_setopt( curl, CURLOPT_WRITEDATA, &task->ws ); // curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L ); @@ -233,6 +225,7 @@ runWitCurl( RelayTask* task, const gchar* proc, ...) /* always cleanup */ curl_easy_cleanup(curl); curl_global_cleanup(); + g_free( params ); return success; } @@ -873,13 +866,16 @@ handlePost( RelayTask* task ) XP_LOGF( "%s(task.post.len=%d)", __func__, task->u.post.len ); XP_ASSERT( !onMainThread(task->storage) ); char* data = g_base64_encode( task->u.post.msgbuf, task->u.post.len ); - struct json_object* jobj = json_object_new_object(); struct json_object* jstr = json_object_new_string(data); g_free( data ); - json_object_object_add( jobj, "data", jstr ); + + /* The protocol takes an array of messages so they can be combined. Do + that soon. */ + json_object* dataArr = json_object_new_array(); + json_object_array_add( dataArr, jstr); json_object* jTimeout = json_object_new_double( task->u.post.timeoutSecs ); - runWitCurl( task, "post", "params", jobj, "timeoutSecs", jTimeout, NULL ); + runWitCurl( task, "post", "data", dataArr, "timeoutSecs", jTimeout, NULL ); // Put the data on the main thread for processing addToGotData( task ); From c2d5f2d253d5dda457da3bc006062a6b5a6182fd Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 13:05:25 -0800 Subject: [PATCH 081/138] cleanup --- .../src/main/java/org/eehouse/android/xw4/NetUtils.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java index 147cc93d2..7dc4a215a 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java @@ -342,17 +342,18 @@ public class NetUtils { return result; } + // This handles multiple params but only every gets passed one! private static String getPostDataString( Map params ) { String result = null; try { ArrayList pairs = new ArrayList(); // StringBuilder sb = new StringBuilder(); - String[] pair = { null, null }; + // String[] pair = { null, null }; for ( Map.Entry entry : params.entrySet() ){ - pair[0] = URLEncoder.encode( entry.getKey(), "UTF-8" ); - pair[1] = URLEncoder.encode( entry.getValue(), "UTF-8" ); - pairs.add( TextUtils.join( "=", pair ) ); + pairs.add( URLEncoder.encode( entry.getKey(), "UTF-8" ) + + "=" + + URLEncoder.encode( entry.getValue(), "UTF-8" ) ); } result = TextUtils.join( "&", pairs ); } catch ( java.io.UnsupportedEncodingException uee ) { From 878875dd3480e6ac33153ca37f1d60b764b1625e Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 13:26:44 -0800 Subject: [PATCH 082/138] revert DB changes -- don't need 'em now It'll be a lot easier moving the relay forward if I can not change the DB. --- xwords4/relay/scripts/showinplay.sh | 2 +- xwords4/relay/xwrelay.sh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/xwords4/relay/scripts/showinplay.sh b/xwords4/relay/scripts/showinplay.sh index 957f30470..a1cc65a6d 100755 --- a/xwords4/relay/scripts/showinplay.sh +++ b/xwords4/relay/scripts/showinplay.sh @@ -54,7 +54,7 @@ echo "; relay pid[s]: $(pidof xwrelay)" echo "Row count:" $(psql -t xwgames -c "select count(*) FROM games $QUERY;") # Games -echo "SELECT dead as d,connname,room,lang as lg,clntVers as cv ,ntotal as t,njoined as nj,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ +echo "SELECT dead as d,connname,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ "FROM games $QUERY ORDER BY NOT dead, ctime DESC LIMIT $LIMIT;" \ | psql xwgames diff --git a/xwords4/relay/xwrelay.sh b/xwords4/relay/xwrelay.sh index 5422e5259..1665c603f 100755 --- a/xwords4/relay/xwrelay.sh +++ b/xwords4/relay/xwrelay.sh @@ -49,8 +49,6 @@ cid integer ,pub BOOLEAN ,connName VARCHAR(64) UNIQUE PRIMARY KEY ,nTotal INTEGER -,nJoined INTEGER DEFAULT 0 -,jtimes TIMESTAMP(0)[] ,clntVers INTEGER[] ,nPerDevice INTEGER[] ,seeds INTEGER[] @@ -61,7 +59,7 @@ cid integer ,addrs INET[] ,devids INTEGER[] ,tokens INTEGER[] -,CHECK (nJoined <= nTotal)) +); EOF cat <<-EOF | psql $DBNAME --file - From bd4ddb0adf07d47f627d05782ba0cfd9f52dd7dd Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 15:26:09 -0800 Subject: [PATCH 083/138] name change --- xwords4/relay/xwrelay.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index 269fa2eab..44ce0f586 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -1998,7 +1998,7 @@ maint_str_loop( int udpsock, const char* str ) } // maint_str_loop static uint32_t -getIPAddr( void ) +getUDPIPAddr( void ) { uint32_t result = INADDR_ANY; char iface[16] = {0}; @@ -2233,7 +2233,7 @@ main( int argc, char** argv ) struct sockaddr_in saddr; g_udpsock = socket( AF_INET, SOCK_DGRAM, IPPROTO_UDP ); saddr.sin_family = PF_INET; - saddr.sin_addr.s_addr = getIPAddr(); + saddr.sin_addr.s_addr = getUDPIPAddr(); saddr.sin_port = htons(udpport); int err = bind( g_udpsock, (struct sockaddr*)&saddr, sizeof(saddr) ); if ( 0 == err ) { From 3da5f237c6c1a87adca35745ed14a2812f774391 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 15:31:36 -0800 Subject: [PATCH 084/138] add asserts to catch non-web holdouts; don't crash Getting nulls when e.g. the relay isn't up. Shouldn't crash in that case. --- .../org/eehouse/android/xw4/NetUtils.java | 14 ++++++--- .../org/eehouse/android/xw4/RelayService.java | 31 ++++++++++++++----- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java index 7dc4a215a..3919eaf8e 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java @@ -110,13 +110,17 @@ public class NetUtils { params.put( one ); } HttpURLConnection conn = makeHttpRelayConn( m_context, "kill" ); - Log.d( TAG, "kill params: %s", params.toString() ); + Log.d( TAG, "runViaWeb(): kill params: %s", params.toString() ); String resStr = runConn( conn, params ); - Log.d( TAG, "kill => %s", resStr ); + Log.d( TAG, "runViaWeb(): kill => %s", resStr ); - JSONObject result = new JSONObject( resStr ); - if ( 0 == result.optInt( "err", -1 ) ) { - DBUtils.clearObits( m_context, m_obits ); + if ( null != resStr ) { + JSONObject result = new JSONObject( resStr ); + if ( 0 == result.optInt( "err", -1 ) ) { + DBUtils.clearObits( m_context, m_obits ); + } + } else { + Log.e( TAG, "runViaWeb(): KILL => null" ); } } catch ( JSONException ex ) { Assert.assertFalse( BuildConfig.DEBUG ); diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index 61aa48b25..9603c13ae 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -668,16 +668,25 @@ public class RelayService extends XWService HttpURLConnection conn = NetUtils.makeHttpRelayConn( this, "post" ); String result = NetUtils.runConn(conn, params); - JSONObject resultObj = new JSONObject( result ); - JSONArray resData = resultObj.getJSONArray( "data" ); - Log.d( TAG, "sendViaWeb(): got %d replies", resData.length() ); + if ( null != result ) { + Log.d( TAG, "sendViaWeb(): POST => %s", result ); + JSONObject resultObj = new JSONObject( result ); + JSONArray resData = resultObj.getJSONArray( "data" ); + int nReplies = resData.length(); + Log.d( TAG, "sendViaWeb(): got %d replies", nReplies ); - noteSent( packets ); // before we process the acks below :-) + noteSent( packets ); // before we process the acks below :-) - for ( int ii = 0; ii < resData.length(); ++ii ) { - byte[] datum = Utils.base64Decode( resData.getString( ii ) ); - // PENDING: skip ack or not - gotPacket( datum, false ); + if ( nReplies > 0 ) { + resetExitTimer(); + } + for ( int ii = 0; ii < nReplies; ++ii ) { + byte[] datum = Utils.base64Decode( resData.getString( ii ) ); + // PENDING: skip ack or not + gotPacket( datum, false ); + } + } else { + Log.e( TAG, "sendViaWeb(): failed result for POST" ); } } catch ( JSONException ex ) { Assert.assertFalse( BuildConfig.DEBUG ); @@ -694,6 +703,9 @@ public class RelayService extends XWService try { DatagramPacket udpPacket = new DatagramPacket( data, data.length ); m_UDPSocket.send( udpPacket ); + if ( BuildConfig.DEBUG ) { + Assert.assertFalse( XWPrefs.getPreferWebAPI( this ) ); + } sentLen += udpPacket.getLength(); noteSent( packet ); getOut = false; @@ -1158,6 +1170,7 @@ public class RelayService extends XWService @Override protected Void doInBackground( Void... ignored ) { + Assert.assertFalse( XWPrefs.getPreferWebAPI( m_context ) ); // format: total msg lenth: 2 // number-of-relayIDs: 2 // for-each-relayid: relayid + '\n': varies @@ -1205,6 +1218,8 @@ public class RelayService extends XWService } // Now open a real socket, write size and proto, and // copy in the formatted buffer + + Assert.assertFalse( XWPrefs.getPreferWebAPI( m_context ) ); Socket socket = NetUtils.makeProxySocket( m_context, 8000 ); if ( null != socket ) { DataOutputStream outStream = From fa52c5091aeae36c4b7d87c90d0f63a836ff6a30 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 15:43:07 -0800 Subject: [PATCH 085/138] fix tap down lower not opening game Making the right_side elem match its parent height prevents the lower-right region of game list items from falling through and triggering a toggle-selection event. --- xwords4/android/app/src/main/res/layout/game_list_item.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwords4/android/app/src/main/res/layout/game_list_item.xml b/xwords4/android/app/src/main/res/layout/game_list_item.xml index 2591b7afe..49e924837 100644 --- a/xwords4/android/app/src/main/res/layout/game_list_item.xml +++ b/xwords4/android/app/src/main/res/layout/game_list_item.xml @@ -64,7 +64,7 @@ From cb87a849ab9a267793e824c1658657aad6a847a7 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 16:06:43 -0800 Subject: [PATCH 086/138] rename dict symlink It breaks rematch that "dict" is being passed to the Android client from the linux side, and this is easier than figuring out how and when to dereference the link. --- xwords4/linux/{dict.xwd => CollegeEng_2to8.xwd} | 0 xwords4/linux/linuxmain.c | 2 +- xwords4/linux/scripts/discon_ok2.sh | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename xwords4/linux/{dict.xwd => CollegeEng_2to8.xwd} (100%) diff --git a/xwords4/linux/dict.xwd b/xwords4/linux/CollegeEng_2to8.xwd similarity index 100% rename from xwords4/linux/dict.xwd rename to xwords4/linux/CollegeEng_2to8.xwd diff --git a/xwords4/linux/linuxmain.c b/xwords4/linux/linuxmain.c index 5edeea05f..ba38e5143 100644 --- a/xwords4/linux/linuxmain.c +++ b/xwords4/linux/linuxmain.c @@ -2659,7 +2659,7 @@ main( int argc, char** argv ) if ( mainParams.needsNewGame ) { /* curses doesn't have newgame dialog */ usage( argv[0], "game params required for curses version, e.g. --name Eric --room MyRoom" - " --remote-player --dict-dir ../ --game-dict dict.xwd"); + " --remote-player --dict-dir ../ --game-dict CollegeEng_2to8.xwd"); } else { #if defined PLATFORM_NCURSES cursesmain( isServer, &mainParams ); diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index 5990beccb..bb5ce7958 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -753,7 +753,7 @@ done # Assign defaults #[ 0 -eq ${#DICTS[@]} ] && DICTS=(dict.xwd) -[ 0 -eq ${#DICTS} ] && DICTS=(dict.xwd) +[ 0 -eq ${#DICTS} ] && DICTS=(CollegeEng_2to8.xwd) [ -z "$APP_NEW" ] && APP_NEW=./obj_linux_memdbg/xwords [ -z "$MINDEVS" ] && MINDEVS=2 [ -z "$MAXDEVS" ] && MAXDEVS=4 From 057728c2871411b299f1319f6b66e3a51555ea8d Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 16:39:09 -0800 Subject: [PATCH 087/138] make gtk include an invite ID in rematch invitations otherwise Android refuses to accept the second "" it receives. --- .../src/main/java/org/eehouse/android/xw4/XWService.java | 2 +- xwords4/common/nli.c | 7 +++++++ xwords4/common/nli.h | 1 + xwords4/linux/gtkboard.c | 3 +++ xwords4/linux/linuxmain.c | 6 +++--- xwords4/linux/linuxmain.h | 2 ++ 6 files changed, 17 insertions(+), 4 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWService.java index c1f305fb1..fc2e666ae 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWService.java @@ -82,7 +82,7 @@ class XWService extends Service { s_seen.add( inviteID ); } } - Log.d( TAG, "checkNotDupe(%s) => %b", inviteID, !isDupe ); + Log.d( TAG, "checkNotDupe('%s') => %b", inviteID, !isDupe ); return !isDupe; } diff --git a/xwords4/common/nli.c b/xwords4/common/nli.c index e54c72a8f..29ea12285 100644 --- a/xwords4/common/nli.c +++ b/xwords4/common/nli.c @@ -61,6 +61,13 @@ nli_setDevID( NetLaunchInfo* nli, XP_U32 devID ) types_addType( &nli->_conTypes, COMMS_CONN_RELAY ); } +void +nli_setInviteID( NetLaunchInfo* nli, const XP_UCHAR* inviteID ) +{ + nli->inviteID[0] = '\0'; + XP_STRCAT( nli->inviteID, inviteID ); +} + void nli_saveToStream( const NetLaunchInfo* nli, XWStreamCtxt* stream ) { diff --git a/xwords4/common/nli.h b/xwords4/common/nli.h index 721a839c4..47f572585 100644 --- a/xwords4/common/nli.h +++ b/xwords4/common/nli.h @@ -76,6 +76,7 @@ void nli_saveToStream( const NetLaunchInfo* invit, XWStreamCtxt* stream ); void nli_makeAddrRec( const NetLaunchInfo* invit, CommsAddrRec* addr ); void nli_setDevID( NetLaunchInfo* invit, XP_U32 devID ); +void nli_setInviteID( NetLaunchInfo* invit, const XP_UCHAR* inviteID ); #endif diff --git a/xwords4/linux/gtkboard.c b/xwords4/linux/gtkboard.c index 7abae1409..0930e3935 100644 --- a/xwords4/linux/gtkboard.c +++ b/xwords4/linux/gtkboard.c @@ -1640,6 +1640,9 @@ send_invites( CommonGlobals* cGlobals, XP_U16 nPlayers, NetLaunchInfo nli = {0}; nli_init( &nli, cGlobals->gi, &addr, nPlayers, forceChannel ); + XP_UCHAR buf[32]; + snprintf( buf, sizeof(buf), "%X", makeRandomInt() ); + nli_setInviteID( &nli, buf ); nli_setDevID( &nli, linux_getDevIDRelay( cGlobals->params ) ); #ifdef DEBUG diff --git a/xwords4/linux/linuxmain.c b/xwords4/linux/linuxmain.c index ba38e5143..948054cd6 100644 --- a/xwords4/linux/linuxmain.c +++ b/xwords4/linux/linuxmain.c @@ -1558,8 +1558,8 @@ linuxChangeRoles( CommonGlobals* cGlobals ) } #endif -static unsigned int -defaultRandomSeed() +unsigned int +makeRandomInt() { /* use kernel device rather than time() so can run multiple times/second without getting the same results. */ @@ -2034,7 +2034,7 @@ main( int argc, char** argv ) XP_Bool isServer = XP_FALSE; // char* portNum = NULL; // char* hostName = "localhost"; - unsigned int seed = defaultRandomSeed(); + unsigned int seed = makeRandomInt(); LaunchParams mainParams; XP_U16 nPlayerDicts = 0; XP_U16 robotCount = 0; diff --git a/xwords4/linux/linuxmain.h b/xwords4/linux/linuxmain.h index 5f902c810..a3622257e 100644 --- a/xwords4/linux/linuxmain.h +++ b/xwords4/linux/linuxmain.h @@ -111,6 +111,8 @@ const XP_UCHAR* linux_getDevID( LaunchParams* params, DevIDType* typ ); void linux_doInitialReg( LaunchParams* params, XP_Bool idIsNew ); XP_Bool linux_setupDevidParams( LaunchParams* params ); +unsigned int makeRandomInt(); + /* void initParams( LaunchParams* params ); */ /* void freeParams( LaunchParams* params ); */ From 5bd71ba7c51d4bef9ce97ee2b8701fc19100221b Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 18:46:02 -0800 Subject: [PATCH 088/138] put cids back into the database The hack I came up with for storing them in memory isn't working. Even if it's just that I don't understand C++ maps they need to be cleared when the DB's wiped (my favorite test these days) and I don't want to spend the time now. --- xwords4/relay/dbmgr.cpp | 89 +++++++++++------------------ xwords4/relay/dbmgr.h | 6 +- xwords4/relay/scripts/showinplay.sh | 2 +- xwords4/relay/xwrelay.cpp | 2 +- 4 files changed, 38 insertions(+), 61 deletions(-) diff --git a/xwords4/relay/dbmgr.cpp b/xwords4/relay/dbmgr.cpp index eb55898cd..d1cc3aca1 100644 --- a/xwords4/relay/dbmgr.cpp +++ b/xwords4/relay/dbmgr.cpp @@ -69,7 +69,6 @@ DBMgr::DBMgr() pthread_key_create( &m_conn_key, destr_function ); pthread_mutex_init( &m_haveNoMessagesMutex, NULL ); - pthread_mutex_init( &m_cidsMutex, NULL ); srand( time( NULL ) ); } @@ -90,13 +89,11 @@ DBMgr::AddNew( const char* cookie, const char* connName, CookieID cid, if ( !cookie ) cookie = ""; if ( !connName ) connName = ""; - MutexLock ml( &m_cidsMutex ); - AddCIDImpl( connName, cid ); - QueryBuilder qb; qb.appendQueryf( "INSERT INTO " GAMES_TABLE - " (room, connName, nTotal, lang, pub)" - " VALUES( $$, $$, $$, $$, $$ )" ) + " (cid, room, connName, nTotal, lang, pub)" + " VALUES( $$, $$, $$, $$, $$, $$ )" ) + .appendParam(cid) .appendParam(cookie) .appendParam(connName) .appendParam(nPlayersT) @@ -125,7 +122,7 @@ DBMgr::FindGameFor( const char* connName, char* cookieBuf, int bufLen, { bool found = false; - const char* fmt = "SELECT room, lang, dead FROM " + const char* fmt = "SELECT cid, room, lang, dead FROM " GAMES_TABLE " WHERE connName = '%s' AND nTotal = %d " "AND %d = seeds[%d] AND 'A' = ack[%d] " ; @@ -133,18 +130,15 @@ DBMgr::FindGameFor( const char* connName, char* cookieBuf, int bufLen, query.catf( fmt, connName, nPlayersS, seed, hid, hid ); logf( XW_LOGINFO, "query: %s", query.c_str() ); - MutexLock ml( &m_cidsMutex ); // not sure I need this.... - PGresult* result = PQexec( getThreadConn(), query.c_str() ); assert( 1 >= PQntuples( result ) ); found = 1 == PQntuples( result ); if ( found ) { int col = 0; + *cidp = atoi( PQgetvalue( result, 0, col++ ) ); snprintf( cookieBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); *langP = atoi( PQgetvalue( result, 0, col++ ) ); *isDead = 't' == PQgetvalue( result, 0, col++ )[0]; - - *cidp = GetCIDImpl(connName); } PQclear( result ); @@ -156,11 +150,9 @@ CookieID DBMgr::FindGame( const char* connName, HostID hid, char* roomBuf, int roomBufLen, int* langP, int* nPlayersTP, int* nPlayersHP, bool* isDead ) { - MutexLock ml( &m_cidsMutex ); - CookieID cid = 0; - const char* fmt = "SELECT room, lang, nTotal, nPerDevice[%d], dead FROM " + const char* fmt = "SELECT cid, room, lang, nTotal, nPerDevice[%d], dead FROM " GAMES_TABLE " WHERE connName = '%s'" // " LIMIT 1" ; @@ -172,7 +164,7 @@ DBMgr::FindGame( const char* connName, HostID hid, char* roomBuf, int roomBufLen assert( 1 >= PQntuples( result ) ); if ( 1 == PQntuples( result ) ) { int col = 0; - cid = GetCIDImpl(connName); + cid = atoi( PQgetvalue( result, 0, col++ ) ); snprintf( roomBuf, roomBufLen, "%s", PQgetvalue( result, 0, col++ ) ); *langP = atoi( PQgetvalue( result, 0, col++ ) ); *nPlayersTP = atoi( PQgetvalue( result, 0, col++ ) ); @@ -192,7 +184,7 @@ DBMgr::FindGame( const AddrInfo::ClientToken clientToken, HostID hid, int* langP, int* nPlayersTP, int* nPlayersHP ) { CookieID cid = 0; - const char* fmt = "SELECT room, lang, nTotal, nPerDevice[%d], connname FROM " + const char* fmt = "SELECT cid, room, lang, nTotal, nPerDevice[%d], connname FROM " GAMES_TABLE " WHERE tokens[%d] = %d and NOT dead"; // " LIMIT 1" ; @@ -204,6 +196,7 @@ DBMgr::FindGame( const AddrInfo::ClientToken clientToken, HostID hid, assert( 1 >= PQntuples( result ) ); if ( 1 == PQntuples( result ) ) { int col = 0; + cid = atoi( PQgetvalue( result, 0, col++ ) ); // room snprintf( roomBuf, roomBufLen, "%s", PQgetvalue( result, 0, col++ ) ); // lang @@ -211,7 +204,6 @@ DBMgr::FindGame( const AddrInfo::ClientToken clientToken, HostID hid, *nPlayersTP = atoi( PQgetvalue( result, 0, col++ ) ); *nPlayersHP = atoi( PQgetvalue( result, 0, col++ ) ); snprintf( connNameBuf, connNameBufLen, "%s", PQgetvalue( result, 0, col++ ) ); - cid = GetCIDImpl(connNameBuf); } PQclear( result ); @@ -302,7 +294,7 @@ DBMgr::SeenSeed( const char* cookie, unsigned short seed, CookieID* cid ) { QueryBuilder qb; - qb.appendQueryf( "SELECT connName, seeds, sum_array(nPerDevice) FROM " + qb.appendQueryf( "SELECT cid, connName, seeds, sum_array(nPerDevice) FROM " GAMES_TABLE " WHERE NOT dead" " AND room ILIKE $$" @@ -319,8 +311,6 @@ DBMgr::SeenSeed( const char* cookie, unsigned short seed, .appendParam(wantsPublic?"TRUE":"FALSE" ) .finish(); - MutexLock ml( &m_cidsMutex ); - PGresult* result = PQexecParams( getThreadConn(), qb.c_str(), qb.paramCount(), NULL, qb.paramValues(), @@ -328,8 +318,8 @@ DBMgr::SeenSeed( const char* cookie, unsigned short seed, bool found = 1 == PQntuples( result ); if ( found ) { int col = 0; + *cid = atoi( PQgetvalue( result, 0, col++ ) ); snprintf( connNameBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); - *cid = GetCIDImpl(connNameBuf); const char* seeds = PQgetvalue( result, 0, col++ ); int perDeviceSum = atoi( PQgetvalue( result, 0, col++ ) ); @@ -346,7 +336,7 @@ DBMgr::FindOpen( const char* cookie, int lang, int nPlayersT, int nPlayersH, int* nPlayersHP ) { QueryBuilder qb; - qb.appendQueryf("SELECT connName, sum_array(nPerDevice) FROM " + qb.appendQueryf("SELECT cid, connName, sum_array(nPerDevice) FROM " GAMES_TABLE " WHERE NOT dead" " AND room ILIKE $$" @@ -362,8 +352,6 @@ DBMgr::FindOpen( const char* cookie, int lang, int nPlayersT, int nPlayersH, .appendParam(wantsPublic?"TRUE":"FALSE" ) .finish(); - MutexLock ml( &m_cidsMutex ); - PGresult* result = PQexecParams( getThreadConn(), qb.c_str(), qb.paramCount(), NULL, qb.paramValues(), @@ -371,8 +359,8 @@ DBMgr::FindOpen( const char* cookie, int lang, int nPlayersT, int nPlayersH, CookieID cid = 0; if ( 1 == PQntuples( result ) ) { int col = 0; + cid = atoi( PQgetvalue( result, 0, col++ ) ); snprintf( connNameBuf, bufLen, "%s", PQgetvalue( result, 0, col++ ) ); - cid = GetCIDImpl(connNameBuf); *nPlayersHP = atoi( PQgetvalue( result, 0, col++ ) ); /* cid may be 0, but should use game anyway */ } @@ -676,45 +664,30 @@ DBMgr::HaveDevice( const char* connName, HostID hid, int seed ) return found; } -bool -DBMgr::AddCIDImpl( const char* const connName, CookieID cid ) -{ - logf( XW_LOGINFO, "%s(%s, %d)", __func__, connName, cid ); - assert( cid != 0 ); - assert( m_cidsMap.find(connName) == m_cidsMap.end() ); - m_cidsMap.insert( pair(connName, cid) ); - assert( m_cidsMap.find(connName) != m_cidsMap.end() ); - return TRUE; -} - bool DBMgr::AddCID( const char* const connName, CookieID cid ) { - MutexLock ml( &m_cidsMutex ); - return AddCIDImpl( connName, cid ); -} + const char* fmt = "UPDATE " GAMES_TABLE " SET cid = %d " + " WHERE connName = '%s' AND cid IS NULL"; + StrWPF query; + query.catf( fmt, cid, connName ); + logf( XW_LOGINFO, "%s: query: %s", __func__, query.c_str() ); -CookieID -DBMgr::GetCIDImpl( const char* const connName ) -{ - CookieID cid = 0; - map::const_iterator iter = m_cidsMap.find(connName); - if (iter != m_cidsMap.end()) { - cid = iter->second; - assert( cid != 0 ); - } - logf( XW_LOGINFO, "%s(%s) => %d", __func__, connName, cid ); - return cid; + bool result = execSql( query ); + logf( XW_LOGINFO, "%s(cid=%d)=>%d", __func__, cid, result ); + return result; } void DBMgr::ClearCID( const char* connName ) { - logf( XW_LOGINFO, "%s(%s)", __func__, connName ); - MutexLock ml( &m_cidsMutex ); - assert( 0 != GetCIDImpl(connName) ); - m_cidsMap.erase( m_cidsMap.find( connName )); - assert( m_cidsMap.find(connName) == m_cidsMap.end() ); + const char* fmt = "UPDATE " GAMES_TABLE " SET cid = null " + "WHERE connName = '%s'"; + StrWPF query; + query.catf( fmt, connName ); + logf( XW_LOGINFO, "%s: query: %s", __func__, query.c_str() ); + + execSql( query ); } void @@ -833,6 +806,12 @@ DBMgr::WaitDBConn( void ) logf( XW_LOGERROR, "%s() done", __func__ ); } +void +DBMgr::ClearCIDs( void ) +{ + execSql( "UPDATE " GAMES_TABLE " set cid = null" ); +} + void DBMgr::PublicRooms( int lang, int nPlayers, int* nNames, string& names ) { diff --git a/xwords4/relay/dbmgr.h b/xwords4/relay/dbmgr.h index 1185b7d7c..d23622c7e 100644 --- a/xwords4/relay/dbmgr.h +++ b/xwords4/relay/dbmgr.h @@ -65,6 +65,8 @@ class DBMgr { void WaitDBConn( void ); + void ClearCIDs( void ); + void AddNew( const char* cookie, const char* connName, CookieID cid, int langCode, int nPlayersT, bool isPublic ); @@ -173,8 +175,6 @@ class DBMgr { int clientVersion, const char* const model, const char* const osVers, DevIDRelay relayID ); - bool AddCIDImpl( const char* const connName, CookieID cid ); - CookieID GetCIDImpl( const char* const connName ); PGconn* getThreadConn( void ); void clearThreadConn(); @@ -195,8 +195,6 @@ class DBMgr { pthread_mutex_t m_haveNoMessagesMutex; set m_haveNoMessagesDevID; set m_haveNoMessagesConnname; - pthread_mutex_t m_cidsMutex; - map m_cidsMap; }; /* DBMgr */ diff --git a/xwords4/relay/scripts/showinplay.sh b/xwords4/relay/scripts/showinplay.sh index a1cc65a6d..8a593ea02 100755 --- a/xwords4/relay/scripts/showinplay.sh +++ b/xwords4/relay/scripts/showinplay.sh @@ -54,7 +54,7 @@ echo "; relay pid[s]: $(pidof xwrelay)" echo "Row count:" $(psql -t xwgames -c "select count(*) FROM games $QUERY;") # Games -echo "SELECT dead as d,connname,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ +echo "SELECT dead as d,connname,cid,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ "FROM games $QUERY ORDER BY NOT dead, ctime DESC LIMIT $LIMIT;" \ | psql xwgames diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index 44ce0f586..def2b44f1 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -2274,7 +2274,7 @@ main( int argc, char** argv ) exit( 1 ); } - // DBMgr::Get()->ClearCIDs(); /* get prev boot's state in db */ + DBMgr::Get()->ClearCIDs(); /* get prev boot's state in db */ vector::const_iterator iter_game; for ( iter_game = ints_game.begin(); iter_game != ints_game.end(); From ec075eff18f8ebf667c2482359ffc5217409d110 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 11 Nov 2017 18:49:23 -0800 Subject: [PATCH 089/138] comment out assertion that's firing when db wiped --- xwords4/common/comms.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xwords4/common/comms.c b/xwords4/common/comms.c index 4afd3dba9..19a0c7986 100644 --- a/xwords4/common/comms.c +++ b/xwords4/common/comms.c @@ -462,7 +462,10 @@ reset_internal( CommsCtxt* comms, XP_Bool isServer, if ( 0 != comms->nextChannelNo ) { XP_LOGF( "%s: comms->nextChannelNo: %d", __func__, comms->nextChannelNo ); } - XP_ASSERT( 0 == comms->nextChannelNo ); /* firing... */ + /* This tends to fire when games reconnect to the relay after the DB's + been wiped and connect in a different order from that in which they did + originally. So comment it out. */ + // XP_ASSERT( 0 == comms->nextChannelNo ); // comms->nextChannelNo = 0; if ( resetRelay ) { comms->channelSeed = 0; From 8650795a8117c8c2963a13bfcd4c4223b2b2ab56 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 12 Nov 2017 08:53:19 -0800 Subject: [PATCH 090/138] don't crash when can't reach relay via http --- .../org/eehouse/android/xw4/RelayService.java | 73 ++++++++++--------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index 9603c13ae..fd483c73c 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -166,7 +166,6 @@ public class RelayService extends XWService { boolean enabled = ! XWPrefs .getPrefsBoolean( context, R.string.key_disable_relay, false ); - Log.d( TAG, "relayEnabled() => %b", enabled ); return enabled; } @@ -656,40 +655,44 @@ public class RelayService extends XWService { Log.d( TAG, "sendViaWeb(): sending %d at once", packets.size() ); int sentLen = 0; - try { - JSONArray dataArray = new JSONArray(); - for ( PacketData packet : packets ) { - byte[] datum = packet.assemble(); - dataArray.put( Utils.base64Encode(datum) ); - sentLen += datum.length; - } - JSONObject params = new JSONObject(); - params.put( "data", dataArray ); - - HttpURLConnection conn = NetUtils.makeHttpRelayConn( this, "post" ); - String result = NetUtils.runConn(conn, params); - if ( null != result ) { - Log.d( TAG, "sendViaWeb(): POST => %s", result ); - JSONObject resultObj = new JSONObject( result ); - JSONArray resData = resultObj.getJSONArray( "data" ); - int nReplies = resData.length(); - Log.d( TAG, "sendViaWeb(): got %d replies", nReplies ); - - noteSent( packets ); // before we process the acks below :-) - - if ( nReplies > 0 ) { - resetExitTimer(); + HttpURLConnection conn = NetUtils.makeHttpRelayConn( this, "post" ); + if ( null == conn ) { + Log.e( TAG, "sendViaWeb(): null conn for POST" ); + } else { + try { + JSONArray dataArray = new JSONArray(); + for ( PacketData packet : packets ) { + byte[] datum = packet.assemble(); + dataArray.put( Utils.base64Encode(datum) ); + sentLen += datum.length; } - for ( int ii = 0; ii < nReplies; ++ii ) { - byte[] datum = Utils.base64Decode( resData.getString( ii ) ); - // PENDING: skip ack or not - gotPacket( datum, false ); + JSONObject params = new JSONObject(); + params.put( "data", dataArray ); + + String result = NetUtils.runConn(conn, params); + if ( null != result ) { + Log.d( TAG, "sendViaWeb(): POST => %s", result ); + JSONObject resultObj = new JSONObject( result ); + JSONArray resData = resultObj.getJSONArray( "data" ); + int nReplies = resData.length(); + Log.d( TAG, "sendViaWeb(): got %d replies", nReplies ); + + noteSent( packets ); // before we process the acks below :-) + + if ( nReplies > 0 ) { + resetExitTimer(); + } + for ( int ii = 0; ii < nReplies; ++ii ) { + byte[] datum = Utils.base64Decode( resData.getString( ii ) ); + // PENDING: skip ack or not + gotPacket( datum, false ); + } + } else { + Log.e( TAG, "sendViaWeb(): failed result for POST" ); } - } else { - Log.e( TAG, "sendViaWeb(): failed result for POST" ); + } catch ( JSONException ex ) { + Assert.assertFalse( BuildConfig.DEBUG ); } - } catch ( JSONException ex ) { - Assert.assertFalse( BuildConfig.DEBUG ); } return sentLen; } @@ -962,11 +965,15 @@ public class RelayService extends XWService { ByteArrayOutputStream bas = new ByteArrayOutputStream(); try { - String devid = getDevID( null ); + DevIDType[] typp = new DevIDType[1]; + String devid = getDevID( typp ); if ( null != devid ) { DataOutputStream out = new DataOutputStream( bas ); writeVLIString( out, devid ); + Log.d(TAG, "requestMessagesImpl(): devid: %s; type: " + typp[0], devid ); postPacket( bas, reg ); + } else { + Log.d(TAG, "requestMessagesImpl(): devid is null" ); } } catch ( java.io.IOException ioe ) { Log.ex( TAG, ioe ); From fb4f44b0fc529a645ac11dcc0c6ae4cf6a691d30 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 12 Nov 2017 19:59:20 -0800 Subject: [PATCH 091/138] remove mistaken assert I'm not quite sure why it was firing, but the pattern of a FindGame failing due to a race condition and requiring a retry exists elsewhere in the code. Lots of tests pass once this change is made. :-) --- xwords4/relay/dbmgr.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/xwords4/relay/dbmgr.cpp b/xwords4/relay/dbmgr.cpp index d1cc3aca1..99492a87a 100644 --- a/xwords4/relay/dbmgr.cpp +++ b/xwords4/relay/dbmgr.cpp @@ -193,7 +193,6 @@ DBMgr::FindGame( const AddrInfo::ClientToken clientToken, HostID hid, logf( XW_LOGINFO, "query: %s", query.c_str() ); PGresult* result = PQexec( getThreadConn(), query.c_str() ); - assert( 1 >= PQntuples( result ) ); if ( 1 == PQntuples( result ) ) { int col = 0; cid = atoi( PQgetvalue( result, 0, col++ ) ); From 373c0249bce2d0ad879a87fcefda6056dcef3598 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 12 Nov 2017 20:04:22 -0800 Subject: [PATCH 092/138] start backoff timer at 5 second, not 30. Makes the http case nearly as quick as the udp one when both games are on the same device. Which is all I've tested so far. --- .../org/eehouse/android/xw4/RelayService.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index fd483c73c..389ddb604 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -66,6 +66,7 @@ public class RelayService extends XWService private static final int MAX_SEND = 1024; private static final int MAX_BUF = MAX_SEND - 2; private static final int REG_WAIT_INTERVAL = 10; + private static final int INITIAL_BACKOFF = 5; // One day, in seconds. Probably should be configurable. private static final long MAX_KEEPALIVE_SECS = 24 * 60 * 60; @@ -1494,18 +1495,19 @@ public class RelayService extends XWService long now = Utils.getCurSeconds(); if ( s_curNextTimer <= now ) { if ( 0 == s_curBackoff ) { - s_curBackoff = 15; + s_curBackoff = INITIAL_BACKOFF; + } else { + s_curBackoff = Math.min( 2 * s_curBackoff, result ); } - s_curBackoff = Math.min( 2 * s_curBackoff, result ); s_curNextTimer += s_curBackoff; } diff = s_curNextTimer - now; } Assert.assertTrue( diff < Integer.MAX_VALUE ); - Log.d( TAG, "figureBackoffSeconds() => %d", diff ); - result = (int)diff; + result = (int)diff; } + Log.d( TAG, "figureBackoffSeconds() => %d", result ); return result; } @@ -1519,6 +1521,11 @@ public class RelayService extends XWService } private class PacketData { + public ByteArrayOutputStream m_bas; + public XWRelayReg m_cmd; + public byte[] m_header; + public int m_packetID; + public PacketData() { m_bas = null; } public PacketData( ByteArrayOutputStream bas, XWRelayReg cmd ) @@ -1564,10 +1571,5 @@ public class RelayService extends XWService Log.ex( TAG, ioe ); } } - - public ByteArrayOutputStream m_bas; - public XWRelayReg m_cmd; - public byte[] m_header; - public int m_packetID; } } From 35d172225fb119437fe6b60613b2d19ab5a3bb4e Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 12 Nov 2017 20:10:51 -0800 Subject: [PATCH 093/138] Use AtomicInteger rather than synchronization --- .../main/java/org/eehouse/android/xw4/RelayService.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index 389ddb604..0db6c86ad 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -59,6 +59,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; public class RelayService extends XWService implements NetStateCache.StateChangedIf { @@ -98,7 +99,7 @@ public class RelayService extends XWService private static final String BINBUFFER = "BINBUFFER"; private static HashSet s_packetsSent = new HashSet(); - private static int s_nextPacketID = 1; + private static AtomicInteger s_nextPacketID = new AtomicInteger(); private static boolean s_gcmWorking = false; private static boolean s_registered = false; private static CommsAddrRec s_addr = @@ -1304,9 +1305,7 @@ public class RelayService extends XWService { int nextPacketID = 0; if ( XWRelayReg.XWPDEV_ACK != cmd ) { - synchronized( s_packetsSent ) { - nextPacketID = ++s_nextPacketID; - } + nextPacketID = s_nextPacketID.incrementAndGet(); } return nextPacketID; } From 033ceab9d143d634360d66a4564c0b70872ecaf0 Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 13 Nov 2017 07:26:47 -0800 Subject: [PATCH 094/138] log outgoing params --- xwords4/linux/relaycon.c | 1 + 1 file changed, 1 insertion(+) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 2e6a6c2ab..17825281c 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -181,6 +181,7 @@ mkJsonParams( CURL* curl, va_list ap ) const char* asStr = json_object_get_string( params ); char* curl_params = curl_easy_escape( curl, asStr, strlen(asStr) ); gchar* result = g_strdup_printf( "params=%s", curl_params ); + XP_LOGF( "%s: adding: params=%s (%s)", __func__, asStr, curl_params ); curl_free( curl_params ); json_object_put( params ); return result; From 2879055ca9516851fe28ebe88fadfcf69f2ca578 Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 13 Nov 2017 08:17:26 -0800 Subject: [PATCH 095/138] log url when response code isn't 200 --- .../app/src/main/java/org/eehouse/android/xw4/NetUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java index f3c584279..fa8e4df5c 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java @@ -334,7 +334,8 @@ public class NetUtils { } result = new String( bas.toByteArray() ); } else { - Log.w( TAG, "runConn: responseCode: %d", responseCode ); + Log.w( TAG, "runConn: responseCode: %d for url: %s", + responseCode, conn.getURL() ); } } catch ( java.net.ProtocolException pe ) { Log.ex( TAG, pe ); From 768b63df246ea7149f939a3f9fc7acec2ac917c8 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 15 Nov 2017 05:48:45 -0800 Subject: [PATCH 096/138] move free() after last use of ptr --- xwords4/linux/relaycon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 17825281c..7e91bf715 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -725,12 +725,12 @@ relaycon_receive( GIOChannel* source, GIOCondition XP_UNUSED_DBG(condition), gpo gchar* b64 = g_base64_encode( (const guchar*)buf, ((0 <= nRead)? nRead : 0) ); XP_LOGF( "%s: read %zd bytes ('%s')", __func__, nRead, b64 ); - g_free( b64 ); #ifdef COMMS_CHECKSUM gchar* sum = g_compute_checksum_for_data( G_CHECKSUM_MD5, buf, nRead ); XP_LOGF( "%s: read %zd bytes ('%s')(sum=%s)", __func__, nRead, b64, sum ); g_free( sum ); #endif + g_free( b64 ); return process( storage, buf, nRead ); } From 8b4042a082b3445a98efc03f90cb26e763785f62 Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 16 Nov 2017 07:13:10 -0800 Subject: [PATCH 097/138] add assertion to catch too many players --- xwords4/linux/linuxmain.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/xwords4/linux/linuxmain.c b/xwords4/linux/linuxmain.c index 948054cd6..ffdc68917 100644 --- a/xwords4/linux/linuxmain.c +++ b/xwords4/linux/linuxmain.c @@ -2290,6 +2290,7 @@ main( int argc, char** argv ) break; case CMD_PLAYERNAME: index = mainParams.pgi.nPlayers++; + XP_ASSERT( index < MAX_NUM_PLAYERS ); ++mainParams.nLocalPlayers; mainParams.pgi.players[index].robotIQ = 0; /* means human */ mainParams.pgi.players[index].isLocal = XP_TRUE; @@ -2298,6 +2299,7 @@ main( int argc, char** argv ) break; case CMD_REMOTEPLAYER: index = mainParams.pgi.nPlayers++; + XP_ASSERT( index < MAX_NUM_PLAYERS ); mainParams.pgi.players[index].isLocal = XP_FALSE; ++mainParams.info.serverInfo.nRemotePlayers; break; @@ -2308,6 +2310,7 @@ main( int argc, char** argv ) case CMD_ROBOTNAME: ++robotCount; index = mainParams.pgi.nPlayers++; + XP_ASSERT( index < MAX_NUM_PLAYERS ); ++mainParams.nLocalPlayers; mainParams.pgi.players[index].robotIQ = 1; /* real smart by default */ mainParams.pgi.players[index].isLocal = XP_TRUE; From 236124319ae237fd20ad87845cb3287f6f875ae1 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 15 Nov 2017 08:34:04 -0800 Subject: [PATCH 098/138] rewrite of shell script in python3 translate the most-used features of my too-big-for-bash script into python3, which is clearly much better suited. Tried to keep the structure and variable names intact so that diff has a chance of showing something, but when a class replaces a bunch of arrays that were being kept in sync there's only so much you can to. Currently doesn't support stuff like app upgrades and switching from tcp to udp, but those should be relatively easy to bring over from the .sh when/if I need them. --- xwords4/linux/scripts/discon_ok2.py | 986 ++++++++++++++++++++++++++++ 1 file changed, 986 insertions(+) create mode 100755 xwords4/linux/scripts/discon_ok2.py diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py new file mode 100755 index 000000000..82bfa6c33 --- /dev/null +++ b/xwords4/linux/scripts/discon_ok2.py @@ -0,0 +1,986 @@ +#!/usr/bin/env python3 + +import re, os, sys, getopt, shutil, threading, requests, json, glob +import argparse, datetime, random, subprocess, time + +# LOGDIR=./$(basename $0)_logs +# APP_NEW="" +# DO_CLEAN="" +# APP_NEW_PARAMS="" +# NGAMES = 1 +g_UDP_PCT_START = 100 +# UDP_PCT_INCR=10 +# UPGRADE_ODDS="" +# NROOMS="" +# HOST="" +# PORT="" +# TIMEOUT="" +# SAVE_GOOD="" +# MINDEVS="" +# MAXDEVS="" +# ONEPER="" +# RESIGN_PCT=0 +g_DROP_N=0 +# MINRUN=2 # seconds +# ONE_PER_ROOM="" # don't run more than one device at a time per room +# USE_GTK="" +# UNDO_PCT=0 +# ALL_VIA_RQ=${ALL_VIA_RQ:-FALSE} +# SEED="" +# BOARD_SIZES_OLD=(15) +# BOARD_SIZES_NEW=(15) +g_NAMES = [None, 'Brynn', 'Ariela', 'Kati', 'Eric'] +# SEND_CHAT='' +# CORE_COUNT=$(ls core.* 2>/dev/null | wc -l) +# DUP_PACKETS='' +# HTTP_PCT=0 + +# declare -A PIDS +# declare -A APPS +# declare -A NEW_ARGS +# declare -a ARGS +# declare -A ARGS_DEVID +# declare -A ROOMS +# declare -A FILES +# declare -A LOGS +# declare -A MINEND +# ROOM_PIDS = {} +# declare -a APPS_OLD=() +# declare -a DICTS= # wants to be =() too? +# declare -A CHECKED_ROOMS + +# function cleanup() { +# APP="$(basename $APP_NEW)" +# while pidof $APP; do +# echo "killing existing $APP instances..." +# killall -9 $APP +# sleep 1 +# done +# echo "cleaning everything up...." +# if [ -d $LOGDIR ]; then +# mv $LOGDIR /tmp/${LOGDIR}_$$ +# fi +# if [ -e $(dirname $0)/../../relay/xwrelay.log ]; then +# mkdir -p /tmp/${LOGDIR}_$$ +# mv $(dirname $0)/../../relay/xwrelay.log /tmp/${LOGDIR}_$$ +# fi + +# echo "DELETE FROM games WHERE room LIKE 'ROOM_%';" | psql -q -t xwgames +# echo "DELETE FROM msgs WHERE NOT devid in (SELECT unnest(devids) from games);" | psql -q -t xwgames +# } + +# function connName() { +# LOG=$1 +# grep -a 'got_connect_cmd: connName' $LOG | \ +# tail -n 1 | \ +# sed 's,^.*connName: \"\(.*\)\" (reconnect=.)$,\1,' +# } + +# function check_room() { +# ROOM=$1 +# if [ -z ${CHECKED_ROOMS[$ROOM]:-""} ]; then +# NUM=$(echo "SELECT COUNT(*) FROM games "\ +# "WHERE NOT dead "\ +# "AND ntotal!=sum_array(nperdevice) "\ +# "AND ntotal != -sum_array(nperdevice) "\ +# "AND room='$ROOM'" | +# psql -q -t xwgames) +# NUM=$((NUM+0)) +# if [ "$NUM" -gt 0 ]; then +# echo "$ROOM in the DB has unconsummated games. Remove them." +# exit 1 +# else +# CHECKED_ROOMS[$ROOM]=1 +# fi +# fi +# } + +# print_cmdline() { +# local COUNTER=$1 +# local LOG=${LOGS[$COUNTER]} +# echo -n "New cmdline: " >> $LOG +# echo "${APPS[$COUNTER]} ${NEW_ARGS[$COUNTER]} ${ARGS[$COUNTER]}" >> $LOG +# } + +def pick_ndevs(args): + RNUM = random.randint(0, 99) + if RNUM > 90 and args.MAXDEVS >= 4: + NDEVS = 4 + elif RNUM > 75 and args.MAXDEVS >= 3: + NDEVS = 3 + else: + NDEVS = 2 + if NDEVS < args.MINDEVS: + NDEVS = args.MINDEVS + return NDEVS + +# # Given a device count, figure out how many local players per device. +# # "1 1" would be a two-device game with 1 each. "1 2 1" a +# # three-device game with four players total +def figure_locals(args, NDEVS): + NPLAYERS = pick_ndevs(args) + if NPLAYERS < NDEVS: NPLAYERS = NDEVS + + EXTRAS = 0 + if not args.ONEPER: + EXTRAS = NPLAYERS - NDEVS + + LOCALS = [] + for IGNORE in range(NDEVS): + COUNT = 1 + if EXTRAS > 0: + EXTRA = random.randint(0, EXTRAS) + if EXTRA > 0: + COUNT += EXTRA + EXTRAS -= EXTRA + LOCALS.append(COUNT) + assert 0 < sum(LOCALS) <= 4 + return LOCALS + +def player_params(args, NLOCALS, NPLAYERS, NAME_INDX): + assert 0 < NPLAYERS <= 4 + NREMOTES = NPLAYERS - NLOCALS + PARAMS = [] + while NLOCALS > 0 or NREMOTES > 0: + if 0 == random.randint(0, 2) and 0 < NLOCALS: + PARAMS += ['--robot', g_NAMES[NAME_INDX], '--robot-iq', str(random.randint(1,100))] + NLOCALS -= 1 + NAME_INDX += 1 + elif 0 < NREMOTES: + PARAMS += ['--remote-player'] + NREMOTES -= 1 + return PARAMS + +def logReaderStub(dev): dev.logReaderMain() + +class Device(): + sConnnameMap = {} + sHasLDevIDMap = {} + sConnNamePat = re.compile('.*got_connect_cmd: connName: "([^"]+)".*$') + sGameOverPat = re.compile('.*\[unused tiles\].*') + sTilesLeftPat = re.compile('.*pool_removeTiles: (\d+) tiles left in pool') + sRelayIDPat = re.compile('.*UPDATE games.*seed=(\d+),.*relayid=\'([^\']+)\'.*') + + def __init__(self, args, indx, app, params, room, db, log, nInGame): + self.indx = indx + self.args = args + self.pid = 0 + self.gameOver = False + self.app = app + self.params = params + self.room = room + self.db = db + self.logPath = log + self.nInGame = nInGame + # runtime stuff; init now + self.proc = None + self.connname = None + self.devID = '' + self.launchCount = 0 + self.allDone = False # when true, can be killed + self.nTilesLeft = -1 # negative means don't know + self.relayID = None + self.relaySeed = 0 + + with open(self.logPath, "w") as log: + log.write('New cmdline: ' + self.app + ' ' + (' '.join([str(p) for p in self.params]))) + log.write(os.linesep) + + def logReaderMain(self): + assert self and self.proc + stdout, stderr = self.proc.communicate() + # print('logReaderMain called; opening:', self.logPath, 'flag:', flag) + nLines = 0 + with open(self.logPath, 'a') as log: + for line in stderr.splitlines(): + nLines += 1 + log.write(line + os.linesep) + + # check for connname + if not self.connname: + match = Device.sConnNamePat.match(line) + if match: + self.connname = match.group(1) + if not self.connname in Device.sConnnameMap: + Device.sConnnameMap[self.connname] = set() + Device.sConnnameMap[self.connname].add(self) + + # check for game over + if not self.gameOver: + match = Device.sGameOverPat.match(line) + if match: self.gameOver = True + + # Check every line for tiles left + match = Device.sTilesLeftPat.match(line) + if match: self.nTilesLeft = int(match.group(1)) + + if not self.relayID: + match = Device.sRelayIDPat.match(line) + if match: + self.relaySeed = int(match.group(1)) + self.relayID = match.group(2) + + # print('logReaderMain done, wrote lines:', nLines, 'to', self.logPath); + + def launch(self): + args = [self.app] + [str(p) for p in self.params] + if self.devID: args.extend( ' '.split(self.devID)) + self.launchCount += 1 + # self.logStream = open(self.logPath, flag) + self.proc = subprocess.Popen(args, stdout = subprocess.DEVNULL, + stderr = subprocess.PIPE, universal_newlines = True) + self.pid = self.proc.pid + self.minEnd = datetime.datetime.now() + datetime.timedelta(seconds = self.args.MINRUN) + + # Now start a thread to read stdio + self.reader = threading.Thread(target = logReaderStub, args=(self,)) + self.reader.isDaemon = True + self.reader.start() + + def running(self): + return self.proc and not self.proc.poll() + + def minTimeExpired(self): + assert self.proc + return self.minEnd < datetime.datetime.now() + + def kill(self): + if self.proc.poll() is None: + self.proc.terminate() + self.proc.wait() + assert self.proc.poll() is not None + + self.reader.join() + self.reader = None + else: + print('NOT killing') + self.proc = None + self.check_game() + + def moveFiles(self): + assert not self.running() + shutil.move(self.logPath, self.args.LOGDIR + '/done') + shutil.move(self.db, self.args.LOGDIR + '/done') + + def send_dead(self): + JSON = json.dumps([{'relayID': self.relayID, 'seed': self.relaySeed}]) + url = 'http://%s/xw4/relay.py/kill' % (self.args.HOST) + req = requests.get(url, params = {'params' : JSON}) + + def getTilesCount(self): + result = None + if self.nTilesLeft != -1: + result = '%.2d:%.2d' % (self.indx, self.nTilesLeft) + return result + + def update_ldevid(self): + if not self.app in Device.sHasLDevIDMap: + hasLDevID = False + proc = subprocess.Popen([self.app, '--help'], stderr=subprocess.PIPE) + # output, err, = proc.communicate() + for line in proc.stderr.readlines(): + if b'--ldevid' in line: + hasLDevID = True + break + print('found --ldevid:', hasLDevID); + Device.sHasLDevIDMap[self.app] = hasLDevID + + if Device.sHasLDevIDMap[self.app]: + RNUM = random.randint(0, 99) + if not self.devID: + if RNUM < 30: + self.devID = '--ldevid LINUX_TEST_%.5d_' % (self.indx) + elif RNUM < 10: + self.devID += 'x' + + def check_game(self): + if self.gameOver and not self.allDone: + allDone = False + if len(Device.sConnnameMap[self.connname]) == self.nInGame: + allDone = True + for dev in Device.sConnnameMap[self.connname]: + if dev == self: continue + if not dev.gameOver: + allDone = False + break + + if allDone: + for dev in Device.sConnnameMap[self.connname]: + dev.allDone = True + + # print('Closing', self.connname, datetime.datetime.now()) + # for dev in Device.sConnnameMap[self.connname]: + # dev.kill() +# # kill_from_logs $OTHERS $KEY +# for ID in $OTHERS $KEY; do +# echo -n "${ID}:${LOGS[$ID]}, " +# kill_from_log ${LOGS[$ID]} || /bin/true +# send_dead $ID +# close_device $ID $DONEDIR "game over" +# done +# echo "" +# # XWRELAY_ERROR_DELETED may be old +# elif grep -aq 'relay_error_curses(XWRELAY_ERROR_DELETED)' $LOG; then +# echo "deleting $LOG $(connName $LOG) b/c another resigned" +# kill_from_log $LOG || /bin/true +# close_device $KEY $DEADDIR "other resigned" +# elif grep -aq 'relay_error_curses(XWRELAY_ERROR_DEADGAME)' $LOG; then +# echo "deleting $LOG $(connName $LOG) b/c another resigned" +# kill_from_log $LOG || /bin/true +# close_device $KEY $DEADDIR "other resigned" +# else +# maybe_resign $KEY +# fi +# } + + +def build_cmds(args): + devs = [] + COUNTER = 0 + PLAT_PARMS = [] + if not args.USE_GTK: + PLAT_PARMS += ['--curses', '--close-stdin'] + + for GAME in range(1, args.NGAMES + 1): + ROOM = 'ROOM_%.3d' % (GAME % args.NROOMS) + # check_room $ROOM + NDEVS = pick_ndevs(args) + LOCALS = figure_locals(args, NDEVS) # as array + NPLAYERS = sum(LOCALS) + assert(len(LOCALS) == NDEVS) + DICT = args.DICTS[GAME % len(args.DICTS)] + # make one in three games public + PUBLIC = [] + if random.randint(0, 3) == 0: PUBLIC = ['--make-public', '--join-public'] + DEV = 0 + for NLOCALS in LOCALS: + DEV += 1 + FILE="%s/GAME_%d_%d.sql3" % (args.LOGDIR, GAME, DEV) + LOG='%s/%d_%d_LOG.txt' % (args.LOGDIR, GAME, DEV) + # os.system("rm -f $LOG") # clear the log + + # APPS[$COUNTER]="$APP_NEW" + # NEW_ARGS[$COUNTER]="$APP_NEW_PARAMS" + BOARD_SIZE = ['--board-size', '15'] + # if [ 0 -lt ${#APPS_OLD[@]} ]; then + # # 50% chance of starting out with old app + # NAPPS=$((1+${#APPS_OLD[*]})) + # if [ 0 -lt $((RANDOM%$NAPPS)) ]; then + # APPS[$COUNTER]=${APPS_OLD[$((RANDOM%${#APPS_OLD[*]}))]} + # BOARD_SIZE="--board-size ${BOARD_SIZES_OLD[$((RANDOM%${#BOARD_SIZES_OLD[*]}))]}" + # NEW_ARGS[$COUNTER]="" + # fi + # fi + + PARAMS = player_params(args, NLOCALS, NPLAYERS, DEV) + PARAMS += PLAT_PARMS + PARAMS += BOARD_SIZE + ['--room', ROOM, '--trade-pct', args.TRADE_PCT, '--sort-tiles'] + if args.UNDO_PCT > 0: + PARAMS += ['--undo-pct', args.UNDO_PCT] + PARAMS += [ '--game-dict', DICT, '--relay-port', args.PORT, '--host', args.HOST] + PARAMS += ['--slow-robot', '1:3', '--skip-confirm'] + PARAMS += ['--db', FILE] + if random.randint(0,100) % 100 < g_UDP_PCT_START: + PARAMS += ['--use-udp'] + + PARAMS += ['--drop-nth-packet', g_DROP_N] + if random.randint(0, 100) < args.HTTP_PCT: + PARAMS += ['--use-http'] + + PARAMS += ['--split-packets', '2'] + if args.SEND_CHAT: + PARAMS += ['--send-chat', args.SEND_CHAT] + + if args.DUP_PACKETS: + PARAMS += ['--dup-packets'] + # PARAMS += ['--my-port', '1024'] + # PARAMS += ['--savefail-pct', 10] + + # With the --seed param passed, games with more than 2 + # devices don't get going. No idea why. This param is NOT + # passed in the old bash version of this script, so fixing + # it isn't a priority. + # PARAMS += ['--seed', args.SEED] + PARAMS += PUBLIC + if DEV > 1: + PARAMS += ['--force-channel', DEV - 1] + else: + PARAMS += ['--server'] + + # print('PARAMS:', PARAMS) + + dev = Device(args, COUNTER, args.APP_NEW, PARAMS, ROOM, FILE, LOG, len(LOCALS)) + dev.update_ldevid() + devs.append(dev) + + COUNTER += 1 + return devs + +# read_resume_cmds() { +# COUNTER=0 +# for LOG in $(ls $LOGDIR/*.txt); do +# echo "need to parse cmd and deal with changes" +# exit 1 +# CMD=$(head -n 1 $LOG) + +# ARGS[$COUNTER]=$CMD +# LOGS[$COUNTER]=$LOG +# PIDS[$COUNTER]=0 + +# set $CMD +# while [ $# -gt 0 ]; do +# case $1 in +# --file) +# FILES[$COUNTER]=$2 +# shift +# ;; +# --room) +# ROOMS[$COUNTER]=$2 +# shift +# ;; +# esac +# shift +# done +# COUNTER=$((COUNTER+1)) +# done +# ROOM_PIDS[$ROOM]=0 +# } + +# launch() { +# KEY=$1 +# LOG=${LOGS[$KEY]} +# APP="${APPS[$KEY]}" +# if [ -z "$APP" ]; then +# echo "error: no app set" +# exit 1 +# fi +# PARAMS="${NEW_ARGS[$KEY]} ${ARGS[$KEY]} ${ARGS_DEVID[$KEY]}" +# exec $APP $PARAMS >/dev/null 2>>$LOG +# } + +# # launch_via_rq() { +# # KEY=$1 +# # RELAYID=$2 +# # PIPE=${PIPES[$KEY]} +# # ../relay/rq -f $RELAYID -o $PIPE & +# # CMD="${CMDS[$KEY]}" +# # exec $CMD >/dev/null 2>>$LOG +# # } + +# send_dead() { +# ID=$1 +# DB=${FILES[$ID]} +# while :; do +# [ -f $DB ] || break # it's gone +# RES=$(echo 'select relayid, seed from games limit 1;' | sqlite3 -separator ' ' $DB || /bin/true) +# [ -n "$RES" ] && break +# sleep 0.2 +# done +# RELAYID=$(echo $RES | awk '{print $1}') +# SEED=$(echo $RES | awk '{print $2}') +# JSON="[{\"relayID\":\"$RELAYID\", \"seed\":$SEED}]" +# curl -G --data-urlencode params="$JSON" http://$HOST/xw4/relay.py/kill >/dev/null 2>&1 +# } + +# close_device() { +# ID=$1 +# MVTO=$2 +# REASON="$3" +# PID=${PIDS[$ID]} +# if [ $PID -ne 0 ]; then +# kill ${PIDS[$ID]} 2>/dev/null +# wait ${PIDS[$ID]} +# ROOM=${ROOMS[$ID]} +# [ ${ROOM_PIDS[$ROOM]} -eq $PID ] && ROOM_PIDS[$ROOM]=0 +# fi +# unset PIDS[$ID] +# unset ARGS[$ID] +# echo "closing game: $REASON" >> ${LOGS[$ID]} +# if [ -n "$MVTO" ]; then +# [ -f "${FILES[$ID]}" ] && mv ${FILES[$ID]} $MVTO +# mv ${LOGS[$ID]} $MVTO +# else +# rm -f ${FILES[$ID]} +# rm -f ${LOGS[$ID]} +# fi +# unset FILES[$ID] +# unset LOGS[$ID] +# unset ROOMS[$ID] +# unset APPS[$ID] +# unset ARGS_DEVID[$ID] + +# COUNT=${#ARGS[*]} +# echo "$COUNT devices left playing..." +# } + +# OBITS="" + +# kill_from_log() { +# LOG=$1 +# RELAYID=$(./scripts/relayID.sh --long $LOG) +# if [ -n "$RELAYID" ]; then +# OBITS="$OBITS -d $RELAYID" +# if [ 0 -eq $(($RANDOM%2)) ]; then +# ../relay/rq -a $HOST $OBITS 2>/dev/null || /bin/true +# OBITS="" +# fi +# return 0 # success +# fi +# echo "unable to send kill command for $LOG" +# return 1 +# } + +# maybe_resign() { +# if [ "$RESIGN_PCT" -gt 0 ]; then +# KEY=$1 +# LOG=${LOGS[$KEY]} +# if grep -aq XWRELAY_ALLHERE $LOG; then +# if [ $((${RANDOM}%100)) -lt $RESIGN_PCT ]; then +# echo "making $LOG $(connName $LOG) resign..." +# kill_from_log $LOG && close_device $KEY $DEADDIR "resignation forced" || /bin/true +# fi +# fi +# fi +# } + +# try_upgrade() { +# KEY=$1 +# if [ 0 -lt ${#APPS_OLD[@]} ]; then +# if [ $APP_NEW != "${APPS[$KEY]}" ]; then +# # one in five chance of upgrading +# if [ 0 -eq $((RANDOM % UPGRADE_ODDS)) ]; then +# APPS[$KEY]=$APP_NEW +# NEW_ARGS[$KEY]="$APP_NEW_PARAMS" +# print_cmdline $KEY +# fi +# fi +# fi +# } + +# try_upgrade_upd() { +# KEY=$1 +# CMD=${ARGS[$KEY]} +# if [ "${CMD/--use-udp/}" = "${CMD}" ]; then +# if [ $((RANDOM % 100)) -lt $UDP_PCT_INCR ]; then +# ARGS[$KEY]="$CMD --use-udp" +# echo -n "$(date +%r): " +# echo "upgrading key $KEY to use UDP" +# fi +# fi +# } + +# check_game() { +# KEY=$1 +# LOG=${LOGS[$KEY]} +# CONNNAME="$(connName $LOG)" +# OTHERS="" +# if [ -n "$CONNNAME" ]; then +# if grep -aq '\[unused tiles\]' $LOG ; then +# for INDX in ${!LOGS[*]}; do +# [ $INDX -eq $KEY ] && continue +# ALOG=${LOGS[$INDX]} +# CONNNAME2="$(connName $ALOG)" +# if [ "$CONNNAME2" = "$CONNNAME" ]; then +# if ! grep -aq '\[unused tiles\]' $ALOG; then +# OTHERS="" +# break +# fi +# OTHERS="$OTHERS $INDX" +# fi +# done +# fi +# fi + +# if [ -n "$OTHERS" ]; then +# echo -n "Closing $CONNNAME [$(date)]: " +# # kill_from_logs $OTHERS $KEY +# for ID in $OTHERS $KEY; do +# echo -n "${ID}:${LOGS[$ID]}, " +# kill_from_log ${LOGS[$ID]} || /bin/true +# send_dead $ID +# close_device $ID $DONEDIR "game over" +# done +# echo "" +# # XWRELAY_ERROR_DELETED may be old +# elif grep -aq 'relay_error_curses(XWRELAY_ERROR_DELETED)' $LOG; then +# echo "deleting $LOG $(connName $LOG) b/c another resigned" +# kill_from_log $LOG || /bin/true +# close_device $KEY $DEADDIR "other resigned" +# elif grep -aq 'relay_error_curses(XWRELAY_ERROR_DEADGAME)' $LOG; then +# echo "deleting $LOG $(connName $LOG) b/c another resigned" +# kill_from_log $LOG || /bin/true +# close_device $KEY $DEADDIR "other resigned" +# else +# maybe_resign $KEY +# fi +# } + +# increment_drop() { +# KEY=$1 +# CMD=${ARGS[$KEY]} +# if [ "$CMD" != "${CMD/drop-nth-packet//}" ]; then +# DROP_N=$(echo $CMD | sed 's,^.*drop-nth-packet \(-*[0-9]*\) .*$,\1,') +# if [ $DROP_N -gt 0 ]; then +# NEXT_N=$((DROP_N+1)) +# ARGS[$KEY]=$(echo $CMD | sed "s,^\(.*drop-nth-packet \)$DROP_N\(.*\)$,\1$NEXT_N\2,") +# fi +# fi +# } + +def summarizeTileCounts(devs): + nDevs = len(devs) + strs = [dev.getTilesCount() for dev in devs] + strs = [s for s in strs if s] + nWithTiles = len(strs) + print('%s %d/%d %s' % (datetime.datetime.now().strftime("%H:%M:%S"), nDevs, nWithTiles, ' '.join(strs))) + +def countCores(): + return len(glob.glob1('/tmp',"core*")) + +def run_cmds(args, devs): + nCores = countCores() + endTime = datetime.datetime.now() + datetime.timedelta(seconds = args.TIMEOUT) + LOOPCOUNT = 0 + + while len(devs) > 0: + if countCores() > nCores: + print('core file count increased; exiting') + break + if datetime.datetime.now() > endTime: + print('outta time; outta here') + break + + LOOPCOUNT += 1 + if 0 == LOOPCOUNT % 20: summarizeTileCounts(devs) + + dev = random.choice(devs) + if not dev.running(): + if dev.allDone: + dev.moveFiles() + dev.send_dead() + devs.remove(dev) + else: +# if [ -n "$ONE_PER_ROOM" -a 0 -ne ${ROOM_PIDS[$ROOM]} ]; then +# continue +# fi +# try_upgrade $KEY +# try_upgrade_upd $KEY + dev.launch() +# PID=$! +# # renice doesn't work on one of my machines... +# renice -n 1 -p $PID >/dev/null 2>&1 || /bin/true +# PIDS[$KEY]=$PID +# ROOM_PIDS[$ROOM]=$PID +# MINEND[$KEY]=$(($NOW + $MINRUN)) + elif not dev.minTimeExpired(): + # print('sleeping...') + time.sleep(2) + else: + dev.kill() + # if g_DROP_N >= 0: dev.increment_drop() + # update_ldevid $KEY + + + # if we get here via a break, kill any remaining games + if devs: + print('stopping %d remaining games' % (len(devs))) + for dev in devs: + if dev.running(): dev.kill() + +# run_via_rq() { +# # launch then kill all games to give chance to hook up +# for KEY in ${!ARGS[*]}; do +# echo "launching $KEY" +# launch $KEY & +# PID=$! +# sleep 1 +# kill $PID +# wait $PID +# # add_pipe $KEY +# done + +# echo "now running via rq" +# # then run them +# while :; do +# COUNT=${#ARGS[*]} +# [ 0 -ge $COUNT ] && break + +# INDX=$(($RANDOM%COUNT)) +# KEYS=( ${!ARGS[*]} ) +# KEY=${KEYS[$INDX]} +# CMD=${ARGS[$KEY]} + +# RELAYID=$(./scripts/relayID.sh --short ${LOGS[$KEY]}) +# MSG_COUNT=$(../relay/rq -a $HOST -m $RELAYID 2>/dev/null | sed 's,^.*-- ,,') +# if [ $MSG_COUNT -gt 0 ]; then +# launch $KEY & +# PID=$! +# sleep 2 +# kill $PID || /bin/true +# wait $PID +# fi +# [ "$DROP_N" -ge 0 ] && increment_drop $KEY +# check_game $KEY +# done +# } # run_via_rq + +# function getArg() { +# [ 1 -lt "$#" ] || usage "$1 requires an argument" +# echo $2 +# } + +def mkParser(): + parser = argparse.ArgumentParser() + parser.add_argument('--send-chat', dest = 'SEND_CHAT', type = str, default = None, + help = 'the message to send') + + parser.add_argument('--app-new', dest = 'APP_NEW', default = './obj_linux_memdbg/xwords', + help = 'the app we\'ll use') + parser.add_argument('--num-games', dest = 'NGAMES', type = int, default = 1, help = 'number of games') + parser.add_argument('--num-rooms', dest = 'NROOMS', type = int, default = 0, + help = 'number of roooms (default to --num-games)') + parser.add_argument('--no-timeout', dest = 'TIMEOUT', default = False, action = 'store_true', + help = 'run forever (default proportional to number of games') + parser.add_argument('--log-root', dest='LOGROOT', default = '.', help = 'where logfiles go') + parser.add_argument('--dup-packets', dest = 'DUP_PACKETS', default = False, help = 'send all packet twice') + parser.add_argument('--use-gtk', dest = 'USE_GTK', default = False, action = 'store_true', + help = 'run games using gtk instead of ncurses') + # # + # # echo " [--clean-start] \\" >&2 + parser.add_argument('--game-dict', dest = 'DICTS', action = 'append', default = []) + # # echo " [--help] \\" >&2 + parser.add_argument('--host', dest = 'HOST', default = 'localhost', + help = 'relay hostname') + # # echo " [--max-devs ] \\" >&2 + parser.add_argument('--min-devs', dest = 'MINDEVS', type = int, default = 2, + help = 'No game will have fewer devices than this') + parser.add_argument('--max-devs', dest = 'MAXDEVS', type = int, default = 4, + help = 'No game will have more devices than this') + parser.add_argument('--min-run', dest = 'MINRUN', type = int, default = 2, + help = 'Keep each run alive at least this many seconds') + # # echo " [--new-app &2 + # # echo " [--new-app-args [arg*]] # passed only to new app \\" >&2 + # # echo " [--num-rooms ] \\" >&2 + # # echo " [--old-app &2 + parser.add_argument('--one-per', dest = 'ONEPER', default = False, + action = 'store_true', help = 'force one player per device') + parser.add_argument('--port', dest = 'PORT', default = 10997, type = int, \ + help = 'Port relay\'s on') + parser.add_argument('--resign-pct', dest = 'RESIGN_PCT', default = 0, type = int, \ + help = 'Odds of resigning [0..100]') + # # echo " [--no-timeout] # run until all games done \\" >&2 + parser.add_argument('--seed', type = int, dest = 'SEED', + default = random.randint(1, 1000000000)) + # # echo " [--send-chat \\" >&2 + # # echo " [--udp-incr ] \\" >&2 + # # echo " [--udp-start ] # default: $UDP_PCT_START \\" >&2 + # # echo " [--undo-pct ] \\" >&2 + parser.add_argument('--http-pct', dest = 'HTTP_PCT', default = 0, type = int, + help = 'pct of games to be using web api') + + parser.add_argument('--undo-pct', dest = 'UNDO_PCT', default = 0, type = int) + parser.add_argument('--trade-pct', dest = 'TRADE_PCT', default = 0, type = int) + + return parser + +# ####################################################### +# ##################### MAIN begins ##################### +# ####################################################### + +def parseArgs(): + args = mkParser().parse_args() + assignDefaults(args) + print(args) + return args + # print(options) + +# while [ "$#" -gt 0 ]; do +# case $1 in +# --udp-start) +# UDP_PCT_START=$(getArg $*) +# shift +# ;; +# --udp-incr) +# UDP_PCT_INCR=$(getArg $*) +# shift +# ;; +# --clean-start) +# DO_CLEAN=1 +# ;; +# --num-games) +# NGAMES=$(getArg $*) +# shift +# ;; +# --num-rooms) +# NROOMS=$(getArg $*) +# shift +# ;; +# --old-app) +# APPS_OLD[${#APPS_OLD[@]}]=$(getArg $*) +# shift +# ;; +# --log-root) +# [ -d $2 ] || usage "$1: no such directory $2" +# LOGDIR=$2/$(basename $0)_logs +# shift +# ;; +# --dup-packets) + # DUP_PACKETS=1 +# ;; +# --new-app) +# APP_NEW=$(getArg $*) +# shift +# ;; +# --new-app-args) +# APP_NEW_PARAMS="${2}" +# echo "got $APP_NEW_PARAMS" +# shift +# ;; +# --game-dict) +# DICTS[${#DICTS[@]}]=$(getArg $*) +# shift +# ;; +# --min-devs) +# MINDEVS=$(getArg $*) +# shift +# ;; +# --max-devs) +# MAXDEVS=$(getArg $*) +# shift +# ;; +# --min-run) +# MINRUN=$(getArg $*) +# [ $MINRUN -ge 2 -a $MINRUN -le 60 ] || usage "$1: n must be 2 <= n <= 60" +# shift +# ;; +# --one-per) +# ONEPER=TRUE +# ;; +# --host) +# HOST=$(getArg $*) +# shift +# ;; +# --port) +# PORT=$(getArg $*) +# shift +# ;; +# --seed) +# SEED=$(getArg $*) +# shift +# ;; +# --undo-pct) +# UNDO_PCT=$(getArg $*) +# shift +# ;; +# --http-pct) +# HTTP_PCT=$(getArg $*) +# [ $HTTP_PCT -ge 0 -a $HTTP_PCT -le 100 ] || usage "$1: n must be 0 <= n <= 100" +# shift +# ;; +# --send-chat) +# SEND_CHAT=$(getArg $*) +# shift +# ;; +# --resign-pct) +# RESIGN_PCT=$(getArg $*) +# [ $RESIGN_PCT -ge 0 -a $RESIGN_PCT -le 100 ] || usage "$1: n must be 0 <= n <= 100" +# shift +# ;; +# --no-timeout) +# TIMEOUT=0x7FFFFFFF +# ;; +# --help) +# usage +# ;; +# *) usage "unrecognized option $1" +# ;; +# esac +# shift +# done + +def assignDefaults(args): + if not args.NROOMS: args.NROOMS = args.NGAMES + if args.TIMEOUT: args.TIMEOUT = 100000000000 # huge number + if len(args.DICTS) == 0: args.DICTS.append('CollegeEng_2to8.xwd') + else: args.TIMEOUT = args.NGAMES * 60 + 500 + args.LOGDIR = os.path.basename(sys.argv[0]) + '_logs' + # Move an existing logdir aside + if os.path.exists(args.LOGDIR): + shutil.move(args.LOGDIR, '/tmp/' + args.LOGDIR + '_' + str(random.randint(0, 100000))) + for d in ['', 'done', 'dead',]: + os.mkdir(args.LOGDIR + '/' + d) +# [ -z "$SAVE_GOOD" ] && SAVE_GOOD=YES +# # [ -z "$RESIGN_PCT" -a "$NGAMES" -gt 1 ] && RESIGN_RATIO=1000 || RESIGN_RATIO=0 +# [ -z "$DROP_N" ] && DROP_N=0 +# [ -z "$USE_GTK" ] && USE_GTK=FALSE +# [ -z "$UPGRADE_ODDS" ] && UPGRADE_ODDS=10 +# #$((NGAMES/50)) +# [ 0 -eq $UPGRADE_ODDS ] && UPGRADE_ODDS=1 +# [ -n "$SEED" ] && RANDOM=$SEED +# [ -z "$ONEPER" -a $NROOMS -lt $NGAMES ] && usage "use --one-per if --num-rooms < --num-games" + +# [ -n "$DO_CLEAN" ] && cleanup + +# RESUME="" +# for FILE in $(ls $LOGDIR/*.{xwg,txt} 2>/dev/null); do +# if [ -e $FILE ]; then +# echo "Unfinished games found in $LOGDIR; continue with them (or discard)?" +# read -p " " ANSWER +# case "$ANSWER" in +# y|yes|Y|YES) +# RESUME=1 +# ;; +# *) +# ;; +# esac +# fi +# break +# done + +# if [ -z "$RESUME" -a -d $LOGDIR ]; then +# NEWNAME="$(basename $LOGDIR)_$$" +# (cd $(dirname $LOGDIR) && mv $(basename $LOGDIR) /tmp/${NEWNAME}) +# fi +# mkdir -p $LOGDIR + +# if [ "$SAVE_GOOD" = YES ]; then +# DONEDIR=$LOGDIR/done +# mkdir -p $DONEDIR +# fi +# DEADDIR=$LOGDIR/dead +# mkdir -p $DEADDIR + +# for VAR in NGAMES NROOMS USE_GTK TIMEOUT HOST PORT SAVE_GOOD \ +# MINDEVS MAXDEVS ONEPER RESIGN_PCT DROP_N ALL_VIA_RQ SEED \ +# APP_NEW; do +# echo "$VAR:" $(eval "echo \$${VAR}") 1>&2 +# done +# echo "DICTS: ${DICTS[*]}" +# echo -n "APPS_OLD: "; [ xx = "${APPS_OLD[*]+xx}" ] && echo "${APPS_OLD[*]}" || echo "" + +# echo "*********$0 starting: $(date)**************" +# STARTTIME=$(date +%s) +# [ -z "$RESUME" ] && build_cmds || read_resume_cmds +# if [ TRUE = "$ALL_VIA_RQ" ]; then +# run_via_rq +# else +# run_cmds +# fi + +# wait + +# SECONDS=$(($(date +%s)-$STARTTIME)) +# HOURS=$((SECONDS/3600)) +# SECONDS=$((SECONDS%3600)) +# MINUTES=$((SECONDS/60)) +# SECONDS=$((SECONDS%60)) +# echo "*********$0 finished: $(date) (took $HOURS:$MINUTES:$SECONDS)**************" + +def main(): + args = parseArgs() + devs = build_cmds(args) + run_cmds(args, devs) + +############################################################################## +if __name__ == '__main__': + main() From e7d00cbad490f560583854dfccd99bf06b6b708c Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 16 Nov 2017 08:27:51 -0800 Subject: [PATCH 099/138] fix TIMEOUT calculation --- xwords4/linux/scripts/discon_ok2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index 82bfa6c33..87c1d5c52 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -900,9 +900,8 @@ def parseArgs(): def assignDefaults(args): if not args.NROOMS: args.NROOMS = args.NGAMES - if args.TIMEOUT: args.TIMEOUT = 100000000000 # huge number + args.TIMEOUT = not args.TIMEOUT and (args.NGAMES * 60 + 500) or 100000000000 if len(args.DICTS) == 0: args.DICTS.append('CollegeEng_2to8.xwd') - else: args.TIMEOUT = args.NGAMES * 60 + 500 args.LOGDIR = os.path.basename(sys.argv[0]) + '_logs' # Move an existing logdir aside if os.path.exists(args.LOGDIR): From ae98425c21600bf632b14b07413f260e133fd76c Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 17 Nov 2017 08:06:23 -0800 Subject: [PATCH 100/138] output game progress as a table --- xwords4/linux/scripts/discon_ok2.py | 39 +++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index 87c1d5c52..a322e90d8 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -178,7 +178,7 @@ class Device(): self.devID = '' self.launchCount = 0 self.allDone = False # when true, can be killed - self.nTilesLeft = -1 # negative means don't know + self.nTilesLeft = None self.relayID = None self.relaySeed = 0 @@ -268,10 +268,7 @@ class Device(): req = requests.get(url, params = {'params' : JSON}) def getTilesCount(self): - result = None - if self.nTilesLeft != -1: - result = '%.2d:%.2d' % (self.indx, self.nTilesLeft) - return result + return (self.indx, self.nTilesLeft, self.launchCount) def update_ldevid(self): if not self.app in Device.sHasLDevIDMap: @@ -628,11 +625,33 @@ def build_cmds(args): # } def summarizeTileCounts(devs): - nDevs = len(devs) - strs = [dev.getTilesCount() for dev in devs] - strs = [s for s in strs if s] - nWithTiles = len(strs) - print('%s %d/%d %s' % (datetime.datetime.now().strftime("%H:%M:%S"), nDevs, nWithTiles, ' '.join(strs))) + data = [dev.getTilesCount() for dev in devs] + nDevs = len(data) + totalTiles = 0 + colWidth = nDevs < 100 and 2 or 3 + headWidth = 0 + fmtData = [{'head' : 'dev', }, + {'head' : 'tls left', }, + {'head' : 'launches', }, + ] + for datum in fmtData: + headWidth = max(headWidth, len(datum['head'])) + datum['data'] = [] + + for tupl in data: + fmtData[0]['data'].append('{:{width}d}'.format(tupl[0], width=colWidth)) + + nTiles = tupl[1] + fmtData[1]['data'].append(nTiles is None and ('-' * colWidth) or '{:{width}d}'.format(nTiles, width=colWidth)) + if not nTiles is None: totalTiles += int(nTiles) + + fmtData[2]['data'].append('{:{width}d}'.format(tupl[2], width=colWidth)) + + print('devs left: {}; tiles left: {}'.format(nDevs, totalTiles)) + fmt = '{head:>%d} {data}' % headWidth + for datum in fmtData: datum['data'] = ' '.join(datum['data']) + for datum in fmtData: + print(fmt.format(**datum)) def countCores(): return len(glob.glob1('/tmp',"core*")) From 039dcdf0cc935aa00aca09e0feb33feb7e5d2e65 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 19 Nov 2017 19:40:29 -0800 Subject: [PATCH 101/138] print how long run has left --- xwords4/linux/scripts/discon_ok2.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index a322e90d8..b1b2f8aa1 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -624,7 +624,7 @@ def build_cmds(args): # fi # } -def summarizeTileCounts(devs): +def summarizeTileCounts(devs, endTime): data = [dev.getTilesCount() for dev in devs] nDevs = len(data) totalTiles = 0 @@ -647,7 +647,8 @@ def summarizeTileCounts(devs): fmtData[2]['data'].append('{:{width}d}'.format(tupl[2], width=colWidth)) - print('devs left: {}; tiles left: {}'.format(nDevs, totalTiles)) + print('') + print('devs left: {}; tiles left: {}; {}/{}'.format(nDevs, totalTiles, datetime.datetime.now(), endTime )) fmt = '{head:>%d} {data}' % headWidth for datum in fmtData: datum['data'] = ' '.join(datum['data']) for datum in fmtData: @@ -659,6 +660,7 @@ def countCores(): def run_cmds(args, devs): nCores = countCores() endTime = datetime.datetime.now() + datetime.timedelta(seconds = args.TIMEOUT) + print('will run until', endTime) LOOPCOUNT = 0 while len(devs) > 0: @@ -670,7 +672,7 @@ def run_cmds(args, devs): break LOOPCOUNT += 1 - if 0 == LOOPCOUNT % 20: summarizeTileCounts(devs) + if 0 == LOOPCOUNT % 20: summarizeTileCounts(devs, endTime) dev = random.choice(devs) if not dev.running(): From 0d8aef06f3d2d6fc2af4c36ad89e89cd597b1339 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 26 Nov 2017 20:11:25 -0800 Subject: [PATCH 102/138] keep packets until ack'd for retry and better logging --- .../org/eehouse/android/xw4/RelayService.java | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index 03e7aaba1..4ece08c91 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -55,9 +55,9 @@ import java.net.InetAddress; import java.net.Socket; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; @@ -98,7 +98,7 @@ public class RelayService extends XWService private static final String ROWID = "ROWID"; private static final String BINBUFFER = "BINBUFFER"; - private static HashSet s_packetsSent = new HashSet(); + private static Map s_packetsSent = new HashMap<>(); private static AtomicInteger s_nextPacketID = new AtomicInteger(); private static boolean s_gcmWorking = false; private static boolean s_registered = false; @@ -741,7 +741,7 @@ public class RelayService extends XWService Log.d( TAG, "Sent [udp?] packet: cmd=%s, id=%d", packet.m_cmd.toString(), pid); synchronized( s_packetsSent ) { - s_packetsSent.add( pid ); + s_packetsSent.put( pid, packet ); } } @@ -1310,15 +1310,17 @@ public class RelayService extends XWService private static void noteAck( int packetID ) { + PacketData packet; synchronized( s_packetsSent ) { - if ( s_packetsSent.contains( packetID ) ) { - s_packetsSent.remove( packetID ); - } else { - Log.w( TAG, "Weird: got ack %d but never sent", packetID ); - } - Log.d( TAG, "noteAck(): Got ack for %d; there are %d unacked packets", - packetID, s_packetsSent.size() ); + packet = s_packetsSent.remove( packetID ); } + if ( packet != null ) { + Log.w( TAG, "noteAck(): removed for id %d: %s", packetID, packet ); + } else { + Log.w( TAG, "Weird: got ack %d but never sent", packetID ); + } + Log.d( TAG, "noteAck(): Got ack for %d; there are %d unacked packets", + packetID, s_packetsSent.size() ); } // Called from any thread @@ -1522,15 +1524,27 @@ public class RelayService extends XWService public XWRelayReg m_cmd; public byte[] m_header; public int m_packetID; + private long m_created; - public PacketData() { m_bas = null; } + public PacketData() { + m_bas = null; + m_created = System.currentTimeMillis(); + } public PacketData( ByteArrayOutputStream bas, XWRelayReg cmd ) { + this(); m_bas = bas; m_cmd = cmd; } + @Override + public String toString() + { + return String.format( "cmd: %s; age: %d ms", m_cmd, + System.currentTimeMillis() - m_created ); + } + public int getLength() { int result = 0; From 503702c0e459a2af599324daf6069150beed06fc Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 26 Nov 2017 20:56:38 -0800 Subject: [PATCH 103/138] log unack'd packets --- .../org/eehouse/android/xw4/RelayService.java | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index 4ece08c91..29ffd94f2 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -26,6 +26,7 @@ import android.content.Intent; import android.os.AsyncTask; import android.os.Build; import android.os.Handler; +import android.text.TextUtils; import junit.framework.Assert; @@ -740,8 +741,10 @@ public class RelayService extends XWService int pid = packet.m_packetID; Log.d( TAG, "Sent [udp?] packet: cmd=%s, id=%d", packet.m_cmd.toString(), pid); - synchronized( s_packetsSent ) { - s_packetsSent.put( pid, packet ); + if ( packet.m_cmd != XWRelayReg.XWPDEV_ACK ) { + synchronized( s_packetsSent ) { + s_packetsSent.put( pid, packet ); + } } } @@ -1313,14 +1316,20 @@ public class RelayService extends XWService PacketData packet; synchronized( s_packetsSent ) { packet = s_packetsSent.remove( packetID ); + if ( packet != null ) { + Log.d( TAG, "noteAck(): removed for id %d: %s", packetID, packet ); + } else { + Log.w( TAG, "Weird: got ack %d but never sent", packetID ); + } + if ( BuildConfig.DEBUG ) { + ArrayList pstrs = new ArrayList<>(); + for ( Integer pkid : s_packetsSent.keySet() ) { + pstrs.add( s_packetsSent.get(pkid).toString() ); + } + Log.d( TAG, "noteAck(): Got ack for %d; there are %d unacked packets: %s", + packetID, s_packetsSent.size(), TextUtils.join( ",", pstrs ) ); + } } - if ( packet != null ) { - Log.w( TAG, "noteAck(): removed for id %d: %s", packetID, packet ); - } else { - Log.w( TAG, "Weird: got ack %d but never sent", packetID ); - } - Log.d( TAG, "noteAck(): Got ack for %d; there are %d unacked packets", - packetID, s_packetsSent.size() ); } // Called from any thread @@ -1541,7 +1550,7 @@ public class RelayService extends XWService @Override public String toString() { - return String.format( "cmd: %s; age: %d ms", m_cmd, + return String.format( "{cmd: %s; age: %d ms}", m_cmd, System.currentTimeMillis() - m_created ); } From af8134202135837a1a3ac7b795fe43ca111d89fa Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 27 Nov 2017 20:35:21 -0800 Subject: [PATCH 104/138] create new fdroid variant --- xwords4/android/app/build.gradle | 21 +++++++++++ .../org/eehouse/android/xw4/CrashTrack.java | 27 ++++++++++++++ .../eehouse/android/xw4/GCMIntentService.java | 37 +++++++++++++++++++ xwords4/android/jni/andutils.c | 2 +- xwords4/android/jni/xwjni.c | 2 +- 5 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/CrashTrack.java create mode 100644 xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/GCMIntentService.java diff --git a/xwords4/android/app/build.gradle b/xwords4/android/app/build.gradle index 563cce10e..013bbad2f 100644 --- a/xwords4/android/app/build.gradle +++ b/xwords4/android/app/build.gradle @@ -74,6 +74,19 @@ android { buildConfigField "String", "GCM_SENDER_ID", "\"$GCM_SENDER_ID\"" } + + xw4fdroid { + dimension "variant" + applicationId "org.eehouse.android.xw4" + manifestPlaceholders = [ APP_ID: applicationId ] + resValue "string", "app_name", "CrossWords" + resValue "string", "nbs_port", "3344" + resValue "string", "invite_prefix", "/and/" + buildConfigField "boolean", "WIDIR_ENABLED", "false" + buildConfigField "boolean", "RELAYINVITE_SUPPORTED", "false" + + buildConfigField "String", "GCM_SENDER_ID", "\"\"" + } xw4d { dimension "variant" minSdkVersion 8 @@ -158,6 +171,14 @@ android { jniLibs.srcDir "../libs-xw4dDebug" } } + xw4fdroid { + release { + jniLibs.srcDir "../libs-xw4fdroidRelease" + } + debug { + jniLibs.srcDir "../libs-xw4fdroidDebug" + } + } } lintOptions { diff --git a/xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/CrashTrack.java b/xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/CrashTrack.java new file mode 100644 index 000000000..5b1d90b1c --- /dev/null +++ b/xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/CrashTrack.java @@ -0,0 +1,27 @@ +/* -*- compile-command: "find-and-gradle.sh insXw4Deb"; -*- */ +/* + * Copyright 2009 - 2012 by Eric House (xwords@eehouse.org). All + * rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package org.eehouse.android.xw4; + +import android.content.Context; + +public class CrashTrack { + public static void init( Context context ) {} // does nothing here +} diff --git a/xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/GCMIntentService.java b/xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/GCMIntentService.java new file mode 100644 index 000000000..52ed99206 --- /dev/null +++ b/xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/GCMIntentService.java @@ -0,0 +1,37 @@ +/* -*- compile-command: "find-and-gradle.sh -PuseCrashlytics insXw4dDeb"; -*- */ +/* + * Copyright 2017 by Eric House (xwords@eehouse.org). All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package org.eehouse.android.xw4; + +import android.app.Application; + +/** + * The ancient GCMIntentService I copied from sample code seems to have + * trouble (burns battery using the WAKELOCK, specifically) when used with an + * app that doesn't have a registration ID. So let's not use that code. + */ + +public class GCMIntentService { + private static final String TAG = GCMIntentService.class.getSimpleName(); + + public static void init( Application app ) + { + Log.d( TAG, "doing nothing" ); + } +} diff --git a/xwords4/android/jni/andutils.c b/xwords4/android/jni/andutils.c index c3e26f02e..a05931d71 100644 --- a/xwords4/android/jni/andutils.c +++ b/xwords4/android/jni/andutils.c @@ -754,7 +754,7 @@ android_debugf( const char* format, ... ) } (void)__android_log_write( ANDROID_LOG_DEBUG, -# if defined VARIANT_xw4 +# if defined VARIANT_xw4 || defined VARIANT_xw4fdroid "xw4" # elif defined VARIANT_xw4d "x4bg" diff --git a/xwords4/android/jni/xwjni.c b/xwords4/android/jni/xwjni.c index 6aaf43f31..9d09568a9 100644 --- a/xwords4/android/jni/xwjni.c +++ b/xwords4/android/jni/xwjni.c @@ -648,7 +648,7 @@ Java_org_eehouse_android_xw4_jni_XwJNI_comms_1getUUID { jstring jstr = #ifdef XWFEATURE_BLUETOOTH -# if defined VARIANT_xw4 +# if defined VARIANT_xw4 || defined VARIANT_xw4fdroid (*env)->NewStringUTF( env, XW_BT_UUID ) # elif defined VARIANT_xw4d (*env)->NewStringUTF( env, XW_BT_UUID_DBG ) From 455373343adae2de58d7fbc73d6ec7499a86ec95 Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 27 Nov 2017 20:23:14 -0800 Subject: [PATCH 105/138] remove requirement for gcm.jar remove from xw4d version too since it can't work anyway. --- xwords4/android/app/build.gradle | 2 +- .../java/org/eehouse/android/xw4/DevID.java | 4 +-- .../java/org/eehouse/android/xw4/GCMStub.java | 32 +++++++++++++++++++ .../java/org/eehouse/android/xw4/GCMStub.java | 29 +++++++++++++++++ .../java/org/eehouse/android/xw4/GCMStub.java | 1 + 5 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 xwords4/android/app/src/xw4/java/org/eehouse/android/xw4/GCMStub.java create mode 100644 xwords4/android/app/src/xw4d/java/org/eehouse/android/xw4/GCMStub.java create mode 120000 xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/GCMStub.java diff --git a/xwords4/android/app/build.gradle b/xwords4/android/app/build.gradle index 013bbad2f..69f18db9d 100644 --- a/xwords4/android/app/build.gradle +++ b/xwords4/android/app/build.gradle @@ -197,7 +197,7 @@ android { dependencies { // Look into replacing this with a fetch too PENDING - compile files('../libs/gcm.jar') + xw4Compile files('../libs/gcm.jar') compile 'com.android.support:support-v4:23.4.0' diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/DevID.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/DevID.java index 2ba88cb9d..8a39bac03 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/DevID.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/DevID.java @@ -31,8 +31,6 @@ package org.eehouse.android.xw4; import android.content.Context; -import com.google.android.gcm.GCMRegistrar; - public class DevID { private static final String TAG = DevID.class.getSimpleName(); @@ -137,7 +135,7 @@ public class DevID { if ( 0 != storedVers && storedVers < curVers ) { result = ""; // Don't trust what registrar has } else { - result = GCMRegistrar.getRegistrationId( context ); + result = GCMStub.getRegistrationId( context ); } return result; } diff --git a/xwords4/android/app/src/xw4/java/org/eehouse/android/xw4/GCMStub.java b/xwords4/android/app/src/xw4/java/org/eehouse/android/xw4/GCMStub.java new file mode 100644 index 000000000..1fcc593be --- /dev/null +++ b/xwords4/android/app/src/xw4/java/org/eehouse/android/xw4/GCMStub.java @@ -0,0 +1,32 @@ +/* -*- compile-command: "find-and-gradle.sh insXw4Deb"; -*- */ +/* + * Copyright 2010 - 2015 by Eric House (xwords@eehouse.org). All rights + * reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package org.eehouse.android.xw4; + +import android.content.Context; +import com.google.android.gcm.GCMRegistrar; + +class GCMStub { + + public static String getRegistrationId( Context context ) + { + return GCMRegistrar.getRegistrationId( context ); + } +} diff --git a/xwords4/android/app/src/xw4d/java/org/eehouse/android/xw4/GCMStub.java b/xwords4/android/app/src/xw4d/java/org/eehouse/android/xw4/GCMStub.java new file mode 100644 index 000000000..94e61ed2c --- /dev/null +++ b/xwords4/android/app/src/xw4d/java/org/eehouse/android/xw4/GCMStub.java @@ -0,0 +1,29 @@ +/* -*- compile-command: "find-and-gradle.sh insXw4dDeb"; -*- */ +/* + * Copyright 2017 by Eric House (xwords@eehouse.org). All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package org.eehouse.android.xw4; + +import android.content.Context; + +class GCMStub { + public static String getRegistrationId( Context context ) + { + return ""; + } +} diff --git a/xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/GCMStub.java b/xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/GCMStub.java new file mode 120000 index 000000000..d409e129f --- /dev/null +++ b/xwords4/android/app/src/xw4fdroid/java/org/eehouse/android/xw4/GCMStub.java @@ -0,0 +1 @@ +../../../../../../xw4d/java/org/eehouse/android/xw4/GCMStub.java \ No newline at end of file From 8e37551f8a945f4209f0c32bf6984f621cf03dbe Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 27 Nov 2017 20:38:21 -0800 Subject: [PATCH 106/138] up changelog for new release --- xwords4/android/app/build.gradle | 4 ++-- xwords4/android/app/src/main/assets/changes.html | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/xwords4/android/app/build.gradle b/xwords4/android/app/build.gradle index 69f18db9d..b170c5e9a 100644 --- a/xwords4/android/app/build.gradle +++ b/xwords4/android/app/build.gradle @@ -1,6 +1,6 @@ def INITIAL_CLIENT_VERS = 8 -def VERSION_CODE_BASE = 126 -def VERSION_NAME = '4.4.130' +def VERSION_CODE_BASE = 127 +def VERSION_NAME = '4.4.131' def FABRIC_API_KEY = System.getenv("FABRIC_API_KEY") def GCM_SENDER_ID = System.getenv("GCM_SENDER_ID") def BUILD_INFO_NAME = "build-info.txt" diff --git a/xwords4/android/app/src/main/assets/changes.html b/xwords4/android/app/src/main/assets/changes.html index 09d30d22e..5ee93b9c0 100644 --- a/xwords4/android/app/src/main/assets/changes.html +++ b/xwords4/android/app/src/main/assets/changes.html @@ -13,9 +13,9 @@ -

CrossWords 4.4.130 release

+

CrossWords 4.4.131 release

-

This release makes a couple of small UI tweaks.

+

An F-Droid-only release meeting new requirements

Please take @@ -25,12 +25,10 @@

New with this release

    -
  • Offer to "Archive" finished games
  • -
  • Make tap on thumbnail toggle whether game's selected rather - than open it. (Tap to the right still opens)
  • -
  • Bug fix: don't allow duplicate group names
  • -
  • Fix battery-hogging behavior on non-Google-play - installs
  • +
  • F-Droid has stiffened their prohibition against including + proprietary Google components. This release complies by removing + "Google Cloud Messaging" – which never worked on + f-droid installs anyway.

(The full changelog From f9c92ca34f0435cc55517830ca1ea52853ae9c09 Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 28 Nov 2017 06:45:42 -0800 Subject: [PATCH 107/138] tweak device-in-game summary Group devices by game so connectivity is easier to watch. --- xwords4/linux/scripts/discon_ok2.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index b1b2f8aa1..c62f7833e 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -161,7 +161,8 @@ class Device(): sTilesLeftPat = re.compile('.*pool_removeTiles: (\d+) tiles left in pool') sRelayIDPat = re.compile('.*UPDATE games.*seed=(\d+),.*relayid=\'([^\']+)\'.*') - def __init__(self, args, indx, app, params, room, db, log, nInGame): + def __init__(self, args, game, indx, app, params, room, db, log, nInGame): + self.game = game self.indx = indx self.args = args self.pid = 0 @@ -268,7 +269,9 @@ class Device(): req = requests.get(url, params = {'params' : JSON}) def getTilesCount(self): - return (self.indx, self.nTilesLeft, self.launchCount) + return {'index': self.indx, 'nTilesLeft': self.nTilesLeft, + 'launchCount': self.launchCount, 'game': self.game, + } def update_ldevid(self): if not self.app in Device.sHasLDevIDMap: @@ -303,6 +306,7 @@ class Device(): if allDone: for dev in Device.sConnnameMap[self.connname]: + assert self.game == dev.game dev.allDone = True # print('Closing', self.connname, datetime.datetime.now()) @@ -340,7 +344,6 @@ def build_cmds(args): for GAME in range(1, args.NGAMES + 1): ROOM = 'ROOM_%.3d' % (GAME % args.NROOMS) - # check_room $ROOM NDEVS = pick_ndevs(args) LOCALS = figure_locals(args, NDEVS) # as array NPLAYERS = sum(LOCALS) @@ -406,7 +409,7 @@ def build_cmds(args): # print('PARAMS:', PARAMS) - dev = Device(args, COUNTER, args.APP_NEW, PARAMS, ROOM, FILE, LOG, len(LOCALS)) + dev = Device(args, GAME, COUNTER, args.APP_NEW, PARAMS, ROOM, FILE, LOG, len(LOCALS)) dev.update_ldevid() devs.append(dev) @@ -638,14 +641,23 @@ def summarizeTileCounts(devs, endTime): headWidth = max(headWidth, len(datum['head'])) datum['data'] = [] - for tupl in data: - fmtData[0]['data'].append('{:{width}d}'.format(tupl[0], width=colWidth)) + # Group devices by game + games = [] + prev = -1 + for datum in data: + gameNo = datum['game'] + if gameNo != prev: + games.append([]) + prev = gameNo + games[-1].append('{:0{width}d}'.format(datum['index'], width=colWidth)) + fmtData[0]['data'] = ['+'.join(game) for game in games] - nTiles = tupl[1] + for datum in data: + nTiles = datum['nTilesLeft'] fmtData[1]['data'].append(nTiles is None and ('-' * colWidth) or '{:{width}d}'.format(nTiles, width=colWidth)) if not nTiles is None: totalTiles += int(nTiles) - fmtData[2]['data'].append('{:{width}d}'.format(tupl[2], width=colWidth)) + fmtData[2]['data'].append('{:{width}d}'.format(datum['launchCount'], width=colWidth)) print('') print('devs left: {}; tiles left: {}; {}/{}'.format(nDevs, totalTiles, datetime.datetime.now(), endTime )) From db0d5ef746f8aa0c7503d118e2c5bc84f8e54614 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 29 Nov 2017 19:32:27 -0800 Subject: [PATCH 108/138] tweak display; add signal handler for clean shutdown --- xwords4/linux/scripts/discon_ok2.py | 30 +++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index c62f7833e..bb54b86fd 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import re, os, sys, getopt, shutil, threading, requests, json, glob -import argparse, datetime, random, subprocess, time +import argparse, datetime, random, signal, subprocess, time # LOGDIR=./$(basename $0)_logs # APP_NEW="" @@ -631,11 +631,11 @@ def summarizeTileCounts(devs, endTime): data = [dev.getTilesCount() for dev in devs] nDevs = len(data) totalTiles = 0 - colWidth = nDevs < 100 and 2 or 3 + colWidth = max(2, len(str(nDevs))) headWidth = 0 fmtData = [{'head' : 'dev', }, - {'head' : 'tls left', }, {'head' : 'launches', }, + {'head' : 'tls left', }, ] for datum in fmtData: headWidth = max(headWidth, len(datum['head'])) @@ -652,15 +652,20 @@ def summarizeTileCounts(devs, endTime): games[-1].append('{:0{width}d}'.format(datum['index'], width=colWidth)) fmtData[0]['data'] = ['+'.join(game) for game in games] + nLaunches = 0 for datum in data: + launchCount = datum['launchCount'] + nLaunches += launchCount + fmtData[1]['data'].append('{:{width}d}'.format(launchCount, width=colWidth)) + nTiles = datum['nTilesLeft'] - fmtData[1]['data'].append(nTiles is None and ('-' * colWidth) or '{:{width}d}'.format(nTiles, width=colWidth)) + fmtData[2]['data'].append(nTiles is None and ('-' * colWidth) or '{:{width}d}'.format(nTiles, width=colWidth)) if not nTiles is None: totalTiles += int(nTiles) - fmtData[2]['data'].append('{:{width}d}'.format(datum['launchCount'], width=colWidth)) print('') - print('devs left: {}; tiles left: {}; {}/{}'.format(nDevs, totalTiles, datetime.datetime.now(), endTime )) + print('devs left: {}; tiles left: {}; total launches: {}; {}/{}' + .format(nDevs, totalTiles, nLaunches, datetime.datetime.now(), endTime )) fmt = '{head:>%d} {data}' % headWidth for datum in fmtData: datum['data'] = ' '.join(datum['data']) for datum in fmtData: @@ -669,13 +674,15 @@ def summarizeTileCounts(devs, endTime): def countCores(): return len(glob.glob1('/tmp',"core*")) +gDone = False + def run_cmds(args, devs): nCores = countCores() endTime = datetime.datetime.now() + datetime.timedelta(seconds = args.TIMEOUT) print('will run until', endTime) LOOPCOUNT = 0 - while len(devs) > 0: + while len(devs) > 0 and not gDone: if countCores() > nCores: print('core file count increased; exiting') break @@ -707,7 +714,7 @@ def run_cmds(args, devs): # MINEND[$KEY]=$(($NOW + $MINRUN)) elif not dev.minTimeExpired(): # print('sleeping...') - time.sleep(2) + time.sleep(1.0) else: dev.kill() # if g_DROP_N >= 0: dev.increment_drop() @@ -1008,7 +1015,14 @@ def assignDefaults(args): # SECONDS=$((SECONDS%60)) # echo "*********$0 finished: $(date) (took $HOURS:$MINUTES:$SECONDS)**************" +def termHandler(signum, frame): + global gDone + print('termHandler() called') + gDone = True + def main(): + signal.signal(signal.SIGINT, termHandler) + args = parseArgs() devs = build_cmds(args) run_cmds(args, devs) From 12ab4bdc5c689f42adc17a1163b3ea1c1b97f323 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 29 Nov 2017 20:02:27 -0800 Subject: [PATCH 109/138] print total runtime on exit --- xwords4/linux/scripts/discon_ok2.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index bb54b86fd..a48eced8b 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -258,6 +258,12 @@ class Device(): self.proc = None self.check_game() + def handleAllDone(self): + if self.allDone: + self.moveFiles() + self.send_dead() + return self.allDone + def moveFiles(self): assert not self.running() shutil.move(self.logPath, self.args.LOGDIR + '/done') @@ -695,9 +701,7 @@ def run_cmds(args, devs): dev = random.choice(devs) if not dev.running(): - if dev.allDone: - dev.moveFiles() - dev.send_dead() + if dev.handleAllDone(): devs.remove(dev) else: # if [ -n "$ONE_PER_ROOM" -a 0 -ne ${ROOM_PIDS[$ROOM]} ]; then @@ -717,6 +721,8 @@ def run_cmds(args, devs): time.sleep(1.0) else: dev.kill() + if dev.handleAllDone(): + devs.remove(dev) # if g_DROP_N >= 0: dev.increment_drop() # update_ldevid $KEY @@ -1021,11 +1027,14 @@ def termHandler(signum, frame): gDone = True def main(): + startTime = datetime.datetime.now() signal.signal(signal.SIGINT, termHandler) args = parseArgs() devs = build_cmds(args) + nDevs = len(devs) run_cmds(args, devs) + print('{} finished; took {} for {} devices'.format(sys.argv[0], datetime.datetime.now() - startTime, nDevs)) ############################################################################## if __name__ == '__main__': From f1b91f4df25be9e298bf4157480b58096f20386f Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 29 Nov 2017 20:39:15 -0800 Subject: [PATCH 110/138] exit if tile line hasn't change in 2 minutes --- xwords4/linux/scripts/discon_ok2.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index a48eced8b..1105eb3a7 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -633,7 +633,8 @@ def build_cmds(args): # fi # } -def summarizeTileCounts(devs, endTime): +def summarizeTileCounts(devs, endTime, state): + shouldGoOn = True data = [dev.getTilesCount() for dev in devs] nDevs = len(data) totalTiles = 0 @@ -677,6 +678,18 @@ def summarizeTileCounts(devs, endTime): for datum in fmtData: print(fmt.format(**datum)) + # Now let's see if things are stuck: if the tile string hasn't + # changed in two minutes bail. Note that the count of tiles left + # isn't enough because it's zero for a long time as devices are + # using up what's left in their trays and getting killed. + now = datetime.datetime.now() + tilesStr = fmtData[2]['data'] + if not 'tilesStr' in state or state['tilesStr'] != tilesStr: + state['lastChange'] = now + state['tilesStr'] = tilesStr + + return now - state['lastChange'] < datetime.timedelta(minutes = 1) + def countCores(): return len(glob.glob1('/tmp',"core*")) @@ -687,6 +700,7 @@ def run_cmds(args, devs): endTime = datetime.datetime.now() + datetime.timedelta(seconds = args.TIMEOUT) print('will run until', endTime) LOOPCOUNT = 0 + printState = {} while len(devs) > 0 and not gDone: if countCores() > nCores: @@ -697,7 +711,10 @@ def run_cmds(args, devs): break LOOPCOUNT += 1 - if 0 == LOOPCOUNT % 20: summarizeTileCounts(devs, endTime) + if 0 == LOOPCOUNT % 20: + if not summarizeTileCounts(devs, endTime, printState): + print('no change in too long; exiting') + break dev = random.choice(devs) if not dev.running(): From 45bb36e66c9098ec51c91fd0894979754541b717 Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 30 Nov 2017 07:43:04 -0800 Subject: [PATCH 111/138] pad log and db file names for better sorting --- xwords4/linux/scripts/discon_ok2.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index 1105eb3a7..d472164b9 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -361,12 +361,9 @@ def build_cmds(args): DEV = 0 for NLOCALS in LOCALS: DEV += 1 - FILE="%s/GAME_%d_%d.sql3" % (args.LOGDIR, GAME, DEV) - LOG='%s/%d_%d_LOG.txt' % (args.LOGDIR, GAME, DEV) - # os.system("rm -f $LOG") # clear the log + DB = '{}/{:02d}_{:02d}_DB.sql3'.format(args.LOGDIR, GAME, DEV) + LOG = '{}/{:02d}_{:02d}_LOG.txt'.format(args.LOGDIR, GAME, DEV) - # APPS[$COUNTER]="$APP_NEW" - # NEW_ARGS[$COUNTER]="$APP_NEW_PARAMS" BOARD_SIZE = ['--board-size', '15'] # if [ 0 -lt ${#APPS_OLD[@]} ]; then # # 50% chance of starting out with old app @@ -385,7 +382,7 @@ def build_cmds(args): PARAMS += ['--undo-pct', args.UNDO_PCT] PARAMS += [ '--game-dict', DICT, '--relay-port', args.PORT, '--host', args.HOST] PARAMS += ['--slow-robot', '1:3', '--skip-confirm'] - PARAMS += ['--db', FILE] + PARAMS += ['--db', DB] if random.randint(0,100) % 100 < g_UDP_PCT_START: PARAMS += ['--use-udp'] @@ -415,7 +412,7 @@ def build_cmds(args): # print('PARAMS:', PARAMS) - dev = Device(args, GAME, COUNTER, args.APP_NEW, PARAMS, ROOM, FILE, LOG, len(LOCALS)) + dev = Device(args, GAME, COUNTER, args.APP_NEW, PARAMS, ROOM, DB, LOG, len(LOCALS)) dev.update_ldevid() devs.append(dev) From e485af925f2b741873d4ff8a921c8beb80711c88 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 3 Dec 2017 16:07:30 -0800 Subject: [PATCH 112/138] log milliseconds: seconds not enough. --- xwords4/relay/xwrelay.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index def2b44f1..4d65ae45b 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -124,8 +124,6 @@ logf( XW_LogLevel level, const char* format, ... ) va_end(ap); #else FILE* where = NULL; - struct tm* timp; - struct timeval tv; bool useFile; char logFile[256]; @@ -143,13 +141,14 @@ logf( XW_LogLevel level, const char* format, ... ) if ( !!where ) { static int tm_yday = 0; + struct timeval tv; gettimeofday( &tv, NULL ); struct tm result; - timp = localtime_r( &tv.tv_sec, &result ); + struct tm* timp = localtime_r( &tv.tv_sec, &result ); char timeBuf[64]; - sprintf( timeBuf, "%.2d:%.2d:%.2d", timp->tm_hour, - timp->tm_min, timp->tm_sec ); + sprintf( timeBuf, "%.2d:%.2d:%.2d.%03ld", timp->tm_hour, + timp->tm_min, timp->tm_sec, tv.tv_usec / 1000 ); /* log the date once/day. This isn't threadsafe so may be repeated but that's harmless. */ From 17d3f14d9eedb0a0814200ac89a56a2d15bacd9a Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 3 Dec 2017 17:05:02 -0800 Subject: [PATCH 113/138] kill script when no change for two minutes by default -- there's a flag for that too. --- xwords4/linux/scripts/discon_ok2.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index d472164b9..127eba615 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -694,8 +694,7 @@ gDone = False def run_cmds(args, devs): nCores = countCores() - endTime = datetime.datetime.now() + datetime.timedelta(seconds = args.TIMEOUT) - print('will run until', endTime) + endTime = datetime.datetime.now() + datetime.timedelta(minutes = args.TIMEOUT_MINS) LOOPCOUNT = 0 printState = {} @@ -799,8 +798,8 @@ def mkParser(): parser.add_argument('--num-games', dest = 'NGAMES', type = int, default = 1, help = 'number of games') parser.add_argument('--num-rooms', dest = 'NROOMS', type = int, default = 0, help = 'number of roooms (default to --num-games)') - parser.add_argument('--no-timeout', dest = 'TIMEOUT', default = False, action = 'store_true', - help = 'run forever (default proportional to number of games') + parser.add_argument('--timeout-mins', dest = 'TIMEOUT_MINS', default = 10000, type = int, + help = 'minutes after which to timeout') parser.add_argument('--log-root', dest='LOGROOT', default = '.', help = 'where logfiles go') parser.add_argument('--dup-packets', dest = 'DUP_PACKETS', default = False, help = 'send all packet twice') parser.add_argument('--use-gtk', dest = 'USE_GTK', default = False, action = 'store_true', @@ -828,7 +827,6 @@ def mkParser(): help = 'Port relay\'s on') parser.add_argument('--resign-pct', dest = 'RESIGN_PCT', default = 0, type = int, \ help = 'Odds of resigning [0..100]') - # # echo " [--no-timeout] # run until all games done \\" >&2 parser.add_argument('--seed', type = int, dest = 'SEED', default = random.randint(1, 1000000000)) # # echo " [--send-chat \\" >&2 @@ -960,7 +958,6 @@ def parseArgs(): def assignDefaults(args): if not args.NROOMS: args.NROOMS = args.NGAMES - args.TIMEOUT = not args.TIMEOUT and (args.NGAMES * 60 + 500) or 100000000000 if len(args.DICTS) == 0: args.DICTS.append('CollegeEng_2to8.xwd') args.LOGDIR = os.path.basename(sys.argv[0]) + '_logs' # Move an existing logdir aside From fd5549014a3ce4d1fbc72c1c1b51abd607ab9a73 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 3 Dec 2017 18:52:06 -0800 Subject: [PATCH 114/138] fix logging --- xwords4/relay/tpool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwords4/relay/tpool.cpp b/xwords4/relay/tpool.cpp index 9e402c744..e4abae347 100644 --- a/xwords4/relay/tpool.cpp +++ b/xwords4/relay/tpool.cpp @@ -265,7 +265,6 @@ XWThreadPool::real_tpool_main( ThreadInfo* tip ) if ( gotOne ) { sock = pr.m_info.m_addr.getSocket(); - logf( XW_LOGINFO, "worker thread got socket %d from queue", socket ); switch ( pr.m_act ) { case Q_READ: assert( 0 ); @@ -275,6 +274,7 @@ XWThreadPool::real_tpool_main( ThreadInfo* tip ) // } break; case Q_KILL: + logf( XW_LOGINFO, "worker thread got socket %d from queue (to close it)", sock ); (*m_kFunc)( &pr.m_info.m_addr ); CloseSocket( &pr.m_info.m_addr ); break; From d5c4ecabcef0d590c1dfaf678501f2a9b34342bf Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 3 Dec 2017 19:18:36 -0800 Subject: [PATCH 115/138] rename class; add some logging Rename class that's no longer just about UDP as it began. Add a bit of logging. This commit *should* not change behavior at all. --- xwords4/relay/addrinfo.cpp | 2 +- xwords4/relay/tpool.cpp | 30 +++++++++++++++++++----------- xwords4/relay/udpqueue.cpp | 32 +++++++++++++++++++------------- xwords4/relay/udpqueue.h | 14 +++++++------- xwords4/relay/xwrelay.cpp | 31 ++++++++++++++++--------------- 5 files changed, 62 insertions(+), 47 deletions(-) diff --git a/xwords4/relay/addrinfo.cpp b/xwords4/relay/addrinfo.cpp index 32839d1fd..ead4105ca 100644 --- a/xwords4/relay/addrinfo.cpp +++ b/xwords4/relay/addrinfo.cpp @@ -68,7 +68,7 @@ AddrInfo::equals( const AddrInfo& other ) const if ( isTCP() ) { equal = m_socket == other.m_socket; if ( equal && created() != other.created() ) { - logf( XW_LOGINFO, "%s: rejecting on time mismatch (%lx vs %lx)", + logf( XW_LOGINFO, "%s(): rejecting on time mismatch (%lx vs %lx)", __func__, created(), other.created() ); equal = false; } diff --git a/xwords4/relay/tpool.cpp b/xwords4/relay/tpool.cpp index e4abae347..9823acebf 100644 --- a/xwords4/relay/tpool.cpp +++ b/xwords4/relay/tpool.cpp @@ -119,13 +119,14 @@ XWThreadPool::Stop() void XWThreadPool::AddSocket( SockType stype, QueueCallback proc, const AddrInfo* from ) { + int sock = from->getSocket(); + logf( XW_LOGVERBOSE0, "%s(sock=%d, isTCP=%d)", __func__, sock, from->isTCP() ); + SockInfo si = { .m_type = stype, + .m_proc = proc, + .m_addr = *from + }; { - int sock = from->getSocket(); RWWriteLock ml( &m_activeSocketsRWLock ); - SockInfo si; - si.m_type = stype; - si.m_proc = proc; - si.m_addr = *from; m_activeSockets.insert( pair( sock, si ) ); } interrupt_poll(); @@ -158,13 +159,14 @@ XWThreadPool::RemoveSocket( const AddrInfo* addr ) size_t prevSize = m_activeSockets.size(); - map::iterator iter = m_activeSockets.find( addr->getSocket() ); + int sock = addr->getSocket(); + map::iterator iter = m_activeSockets.find( sock ); if ( m_activeSockets.end() != iter && iter->second.m_addr.equals( *addr ) ) { m_activeSockets.erase( iter ); found = true; } - logf( XW_LOGINFO, "%s: AFTER: %d sockets active (was %d)", __func__, - m_activeSockets.size(), prevSize ); + logf( XW_LOGINFO, "%s(): AFTER closing %d: %d sockets active (was %d)", __func__, + sock, m_activeSockets.size(), prevSize ); } return found; } /* RemoveSocket */ @@ -172,6 +174,7 @@ XWThreadPool::RemoveSocket( const AddrInfo* addr ) void XWThreadPool::CloseSocket( const AddrInfo* addr ) { + int sock = addr->getSocket(); if ( addr->isTCP() ) { if ( !RemoveSocket( addr ) ) { MutexLock ml( &m_queueMutex ); @@ -184,8 +187,13 @@ XWThreadPool::CloseSocket( const AddrInfo* addr ) ++iter; } } - logf( XW_LOGINFO, "CLOSING socket %d", addr->getSocket() ); - close( addr->getSocket() ); + int err = close( sock ); + if ( 0 != err ) { + logf( XW_LOGERROR, "%s(): close(socket=%d) => %d/%s", __func__, + sock, errno, strerror(errno) ); + } else { + logf( XW_LOGINFO, "%s(): close(socket=%d) succeeded", __func__, sock ); + } /* We always need to interrupt the poll because the socket we're closing will be in the list being listened to. That or we need to drop sockets @@ -198,7 +206,7 @@ XWThreadPool::CloseSocket( const AddrInfo* addr ) void XWThreadPool::EnqueueKill( const AddrInfo* addr, const char* const why ) { - logf( XW_LOGINFO, "%s(%d) reason: %s", __func__, addr->getSocket(), why ); + logf( XW_LOGINFO, "%s(socket = %d) reason: %s", __func__, addr->getSocket(), why ); if ( addr->isTCP() ) { SockInfo si; si.m_type = STYPE_UNKNOWN; diff --git a/xwords4/relay/udpqueue.cpp b/xwords4/relay/udpqueue.cpp index 675c28ec3..b59e237b0 100644 --- a/xwords4/relay/udpqueue.cpp +++ b/xwords4/relay/udpqueue.cpp @@ -28,7 +28,7 @@ static UdpQueue* s_instance = NULL; void -UdpThreadClosure::logStats() +PacketThreadClosure::logStats() { time_t now = time( NULL ); if ( 1 < now - m_created ) { @@ -48,6 +48,7 @@ PartialPacket::stillGood() const bool PartialPacket::readAtMost( int len ) { + assert( len > 0 ); bool success = false; uint8_t tmp[len]; ssize_t nRead = recv( m_sock, tmp, len, 0 ); @@ -58,9 +59,11 @@ PartialPacket::readAtMost( int len ) len, m_sock, m_errno, strerror(m_errno) ); } } else if ( 0 == nRead ) { // remote socket closed - logf( XW_LOGINFO, "%s: remote closed (socket=%d)", __func__, m_sock ); + logf( XW_LOGINFO, "%s(): remote closed (socket=%d)", __func__, m_sock ); m_errno = -1; // so stillGood will fail } else { + // logf( XW_LOGVERBOSE0, "%s(): read %d bytes on socket %d", __func__, + // nRead, m_sock ); m_errno = 0; success = len == nRead; int curSize = m_buf.size(); @@ -152,17 +155,20 @@ void UdpQueue::handle( const AddrInfo* addr, const uint8_t* buf, int len, QueueCallback cb ) { - UdpThreadClosure* utc = new UdpThreadClosure( addr, buf, len, cb ); + PacketThreadClosure* ptc = new PacketThreadClosure( addr, buf, len, cb ); MutexLock ml( &m_queueMutex ); int id = ++m_nextID; - utc->setID( id ); + ptc->setID( id ); logf( XW_LOGINFO, "%s: enqueuing packet %d (socket %d, len %d)", __func__, id, addr->getSocket(), len ); - m_queue.push_back( utc ); + m_queue.push_back( ptc ); pthread_cond_signal( &m_queueCondVar ); } +// Remove any PartialPacket record with the same socket/fd. This makes sense +// when the socket's being reused or when we have just dealt with a single +// packet and might be getting more. void UdpQueue::newSocket_locked( int sock ) { @@ -194,25 +200,25 @@ UdpQueue::thread_main() while ( m_queue.size() == 0 ) { pthread_cond_wait( &m_queueCondVar, &m_queueMutex ); } - UdpThreadClosure* utc = m_queue.front(); + PacketThreadClosure* ptc = m_queue.front(); m_queue.pop_front(); pthread_mutex_unlock( &m_queueMutex ); - utc->noteDequeued(); + ptc->noteDequeued(); - time_t age = utc->ageInSeconds(); + time_t age = ptc->ageInSeconds(); if ( 30 > age ) { logf( XW_LOGINFO, "%s: dispatching packet %d (socket %d); " - "%d seconds old", __func__, utc->getID(), - utc->addr()->getSocket(), age ); - (*utc->cb())( utc ); - utc->logStats(); + "%d seconds old", __func__, ptc->getID(), + ptc->addr()->getSocket(), age ); + (*ptc->cb())( ptc ); + ptc->logStats(); } else { logf( XW_LOGINFO, "%s: dropping packet %d; it's %d seconds old!", __func__, age ); } - delete utc; + delete ptc; } return NULL; } diff --git a/xwords4/relay/udpqueue.h b/xwords4/relay/udpqueue.h index cc467af75..3926b3903 100644 --- a/xwords4/relay/udpqueue.h +++ b/xwords4/relay/udpqueue.h @@ -30,13 +30,13 @@ using namespace std; -class UdpThreadClosure; +class PacketThreadClosure; -typedef void (*QueueCallback)( UdpThreadClosure* closure ); +typedef void (*QueueCallback)( PacketThreadClosure* closure ); -class UdpThreadClosure { +class PacketThreadClosure { public: - UdpThreadClosure( const AddrInfo* addr, const uint8_t* buf, + PacketThreadClosure( const AddrInfo* addr, const uint8_t* buf, int len, QueueCallback cb ) : m_buf(new uint8_t[len]) , m_len(len) @@ -47,7 +47,7 @@ public: memcpy( m_buf, buf, len ); } - ~UdpThreadClosure() { delete[] m_buf; } + ~PacketThreadClosure() { delete[] m_buf; } const uint8_t* buf() const { return m_buf; } int len() const { return m_len; } @@ -109,8 +109,8 @@ class UdpQueue { pthread_mutex_t m_partialsMutex; pthread_mutex_t m_queueMutex; pthread_cond_t m_queueCondVar; - deque m_queue; - // map > m_bySocket; + deque m_queue; + // map > m_bySocket; int m_nextID; map m_partialPackets; }; diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index 4d65ae45b..82595390d 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -1475,23 +1475,24 @@ handleProxyMsgs( int sock, const AddrInfo* addr, const uint8_t* bufp, } // handleProxyMsgs static void -game_thread_proc( UdpThreadClosure* utc ) +game_thread_proc( PacketThreadClosure* ptc ) { - if ( !processMessage( utc->buf(), utc->len(), utc->addr(), 0 ) ) { - XWThreadPool::GetTPool()->CloseSocket( utc->addr() ); + logf( XW_LOGVERBOSE0, "%s()", __func__ ); + if ( !processMessage( ptc->buf(), ptc->len(), ptc->addr(), 0 ) ) { + XWThreadPool::GetTPool()->CloseSocket( ptc->addr() ); } } static void -proxy_thread_proc( UdpThreadClosure* utc ) +proxy_thread_proc( PacketThreadClosure* ptc ) { - const int len = utc->len(); - const AddrInfo* addr = utc->addr(); + const int len = ptc->len(); + const AddrInfo* addr = ptc->addr(); if ( len > 0 ) { assert( addr->isTCP() ); int sock = addr->getSocket(); - const uint8_t* bufp = utc->buf(); + const uint8_t* bufp = ptc->buf(); const uint8_t* end = bufp + len; if ( (0 == *bufp++) ) { /* protocol */ XWPRXYCMD cmd = (XWPRXYCMD)*bufp++; @@ -1725,10 +1726,10 @@ ackPacketIf( const UDPHeader* header, const AddrInfo* addr ) } static void -handle_udp_packet( UdpThreadClosure* utc ) +handle_udp_packet( PacketThreadClosure* ptc ) { - const uint8_t* ptr = utc->buf(); - const uint8_t* end = ptr + utc->len(); + const uint8_t* ptr = ptc->buf(); + const uint8_t* end = ptr + ptc->len(); UDPHeader header; if ( getHeader( &ptr, end, &header ) ) { @@ -1751,7 +1752,7 @@ handle_udp_packet( UdpThreadClosure* utc ) if ( 3 >= clientVers ) { checkAllAscii( model, "bad model" ); } - registerDevice( relayID, &devID, utc->addr(), + registerDevice( relayID, &devID, ptc->addr(), clientVers, devDesc, model, osVers ); } } @@ -1764,7 +1765,7 @@ handle_udp_packet( UdpThreadClosure* utc ) ptr += sizeof(clientToken); clientToken = ntohl( clientToken ); if ( AddrInfo::NULL_TOKEN != clientToken ) { - AddrInfo addr( g_udpsock, clientToken, utc->saddr() ); + AddrInfo addr( g_udpsock, clientToken, ptc->saddr() ); (void)processMessage( ptr, end - ptr, &addr, clientToken ); } else { logf( XW_LOGERROR, "%s: dropping packet with token of 0", @@ -1785,7 +1786,7 @@ handle_udp_packet( UdpThreadClosure* utc ) } SafeCref scr( connName, hid ); if ( scr.IsValid() ) { - AddrInfo addr( g_udpsock, clientToken, utc->saddr() ); + AddrInfo addr( g_udpsock, clientToken, ptc->saddr() ); handlePutMessage( scr, hid, &addr, end - ptr, &ptr, end ); assert( ptr == end ); // DON'T CHECK THIS IN!!! } else { @@ -1820,7 +1821,7 @@ handle_udp_packet( UdpThreadClosure* utc ) case XWPDEV_RQSTMSGS: { DevID devID( ID_TYPE_RELAY ); if ( getVLIString( &ptr, end, devID.m_devIDString ) ) { - const AddrInfo* addr = utc->addr(); + const AddrInfo* addr = ptc->addr(); DevMgr::Get()->rememberDevice( devID.asRelayID(), addr ); if ( XWPDEV_RQSTMSGS == header.cmd ) { @@ -1861,7 +1862,7 @@ handle_udp_packet( UdpThreadClosure* utc ) } // Do this after the device and address are registered - ackPacketIf( &header, utc->addr() ); + ackPacketIf( &header, ptc->addr() ); } } From 70dea02efce1f9d42d91f7584f6e8e60f4c3ee0c Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 4 Dec 2017 20:40:44 -0800 Subject: [PATCH 116/138] cleanup; fix possible socket drop Haven't seen it happen, but I think there was a bug that could have led to all the sockets coming back as ready from poll() being dropped. Fixed that and added/cleaned up some logging. --- xwords4/relay/tpool.cpp | 21 +++++++++++++-------- xwords4/relay/xwrelay.cpp | 19 +++++++++++++------ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/xwords4/relay/tpool.cpp b/xwords4/relay/tpool.cpp index 9823acebf..804be3de5 100644 --- a/xwords4/relay/tpool.cpp +++ b/xwords4/relay/tpool.cpp @@ -400,35 +400,40 @@ XWThreadPool::real_listener() curfd = 1; int ii; - for ( ii = 0; ii < nSockets && nEvents > 0; ++ii ) { + for ( ii = 0; ii < nSockets && nEvents > 0; ++ii, ++curfd ) { if ( fds[curfd].revents != 0 ) { // int socket = fds[curfd].fd; SockInfo* sinfo = &sinfos[curfd]; const AddrInfo* addr = &sinfo->m_addr; - assert( fds[curfd].fd == addr->getSocket() ); + int sock = addr->getSocket(); + assert( fds[curfd].fd == sock ); if ( !SocketFound( addr ) ) { + logf( XW_LOGINFO, "%s(): dropping socket %d: not found", + __func__, addr->getSocket() ); /* no further processing if it's been removed while - we've been sleeping in poll */ + we've been sleeping in poll. BUT: shouldn't curfd + be incremented?? */ --nEvents; continue; } if ( 0 != (fds[curfd].revents & (POLLIN | POLLPRI)) ) { if ( !UdpQueue::get()->handle( addr, sinfo->m_proc ) ) { + // This is likely wrong!!! return of 0 means + // remote closed, not error. RemoveSocket( addr ); - EnqueueKill( addr, "bad packet" ); + EnqueueKill( addr, "got EOF" ); } } else { - logf( XW_LOGERROR, "odd revents: %x", - fds[curfd].revents ); + logf( XW_LOGERROR, "%s(): odd revents: %x; bad socket %d", + __func__, fds[curfd].revents, sock ); RemoveSocket( addr ); - EnqueueKill( addr, "error/hup in poll()" ); + EnqueueKill( addr, "error/hup in poll()" ); } --nEvents; } - ++curfd; } assert( nEvents == 0 ); } diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index 82595390d..f241382e9 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -1303,14 +1303,16 @@ handleMsgsMsg( const AddrInfo* addr, bool sendFull, const uint8_t* bufp, const uint8_t* end ) { unsigned short nameCount; - int ii; if ( getNetShort( &bufp, end, &nameCount ) ) { DBMgr* dbmgr = DBMgr::Get(); vector out(4); /* space for len and n_msgs */ assert( out.size() == 4 ); vector msgIDs; - for ( ii = 0; ii < nameCount && bufp < end; ++ii ) { - + for ( int ii = 0; ii < nameCount; ++ii ) { + if ( bufp >= end ) { + logf( XW_LOGERROR, "%s(): ran off the end", __func__ ); + break; + } // See NetUtils.java for reply format // message-length: 2 // nameCount: 2 @@ -1344,9 +1346,14 @@ handleMsgsMsg( const AddrInfo* addr, bool sendFull, memcpy( &out[0], &tmp, sizeof(tmp) ); tmp = htons( nameCount ); memcpy( &out[2], &tmp, sizeof(tmp) ); - ssize_t nwritten = write( addr->getSocket(), &out[0], out.size() ); - logf( XW_LOGVERBOSE0, "%s: wrote %d bytes", __func__, nwritten ); - if ( sendFull && nwritten >= 0 && (size_t)nwritten == out.size() ) { + int sock = addr->getSocket(); + ssize_t nWritten = write( sock, &out[0], out.size() ); + if ( nWritten < 0 ) { + logf( XW_LOGERROR, "%s(): write to socket %d failed: %d/%s", __func__, + sock, errno, strerror(errno) ); + } else if ( sendFull && (size_t)nWritten == out.size() ) { + logf( XW_LOGVERBOSE0, "%s(): wrote %d bytes to socket %d", __func__, + nWritten, sock ); dbmgr->RecordSent( &msgIDs[0], msgIDs.size() ); // This is wrong: should be removed when ACK returns and not // before. But for some reason if I make that change apps wind up From 243eb213bbd835b340da2c5be402d3c6fcd6cc77 Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 6 Dec 2017 18:57:48 -0800 Subject: [PATCH 117/138] make reply messages a param within result Makes it easier to debug by adding stuff on the relay end --- xwords4/linux/relaycon.c | 50 +++++++++++++++++++++------------- xwords4/relay/scripts/relay.py | 13 ++++++--- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/xwords4/linux/relaycon.c b/xwords4/linux/relaycon.c index 7e91bf715..6f0c2a534 100644 --- a/xwords4/linux/relaycon.c +++ b/xwords4/linux/relaycon.c @@ -216,7 +216,7 @@ runWitCurl( RelayTask* task, const gchar* proc, ...) res = curl_easy_perform(curl); XP_Bool success = res == CURLE_OK; - XP_LOGF( "%s(): curl_easy_perform() => %d", __func__, res ); + XP_LOGF( "%s(): curl_easy_perform(%s) => %d", __func__, proc, res ); /* Check for errors */ if ( ! success ) { XP_LOGF( "curl_easy_perform() failed: %s", curl_easy_strerror(res)); @@ -906,26 +906,37 @@ onGotQueryData( RelayTask* task ) CommsAddrRec addr = {0}; addr_addType( &addr, COMMS_CONN_RELAY ); - /* Currently there's an array of arrays for each relayID (value) */ - json_object_object_foreach(reply, relayID, arrOfArrOfMoves) { - int len1 = json_object_array_length( arrOfArrOfMoves ); - if ( len1 > 0 ) { - sqlite3_int64 rowid = *(sqlite3_int64*)g_hash_table_lookup( task->u.query.map, relayID ); - for ( int ii = 0; ii < len1; ++ii ) { - json_object* forGameArray = json_object_array_get_idx( arrOfArrOfMoves, ii ); - int len2 = json_object_array_length( forGameArray ); - for ( int jj = 0; jj < len2; ++jj ) { - json_object* oneMove = json_object_array_get_idx( forGameArray, jj ); - const char* asStr = json_object_get_string( oneMove ); - gsize out_len; - guchar* buf = g_base64_decode( asStr, &out_len ); - (*storage->procs.msgForRow)( storage->procsClosure, &addr, - rowid, buf, out_len ); - g_free(buf); - foundAny = XP_TRUE; + GList* ids = g_hash_table_get_keys( task->u.query.map ); + const char* xxx = ids->data; + + json_object* jMsgs; + if ( json_object_object_get_ex( reply, "msgs", &jMsgs ) ) { + /* Currently there's an array of arrays for each relayID (value) */ + XP_LOGF( "%s: got result of len %d", __func__, json_object_object_length(jMsgs) ); + XP_ASSERT( json_object_object_length(jMsgs) <= 1 ); + json_object_object_foreach(jMsgs, relayID, arrOfArrOfMoves) { + XP_ASSERT( 0 == strcmp( relayID, xxx ) ); + int len1 = json_object_array_length( arrOfArrOfMoves ); + if ( len1 > 0 ) { + sqlite3_int64 rowid = *(sqlite3_int64*)g_hash_table_lookup( task->u.query.map, relayID ); + XP_LOGF( "%s(): got row %lld for relayID %s", __func__, rowid, relayID ); + for ( int ii = 0; ii < len1; ++ii ) { + json_object* forGameArray = json_object_array_get_idx( arrOfArrOfMoves, ii ); + int len2 = json_object_array_length( forGameArray ); + for ( int jj = 0; jj < len2; ++jj ) { + json_object* oneMove = json_object_array_get_idx( forGameArray, jj ); + const char* asStr = json_object_get_string( oneMove ); + gsize out_len; + guchar* buf = g_base64_decode( asStr, &out_len ); + (*storage->procs.msgForRow)( storage->procsClosure, &addr, + rowid, buf, out_len ); + g_free(buf); + foundAny = XP_TRUE; + } } } } + json_object_put( jMsgs ); } json_object_put( reply ); } @@ -955,6 +966,7 @@ handleQuery( RelayTask* task ) for ( GList* iter = ids; !!iter; iter = iter->next ) { json_object* idstr = json_object_new_string( iter->data ); json_object_array_add(jIds, idstr); + XP_ASSERT( !iter->next ); /* for curses case there should be only one */ } g_list_free( ids ); @@ -1043,7 +1055,7 @@ getFromGotData( RelayConStorage* storage ) storage->gotDataTaskList ); task = head->data; g_slist_free( head ); - XP_LOGF( "%s(): got id %d!", __func__, task->id ); + XP_LOGF( "%s(): got task id %d", __func__, task->id ); } // XP_LOGF( "%s(): len now %d", __func__, g_slist_length(storage->gotDataTaskList) ); pthread_mutex_unlock( &storage->gotDataMutex ); diff --git a/xwords4/relay/scripts/relay.py b/xwords4/relay/scripts/relay.py index 7ca6c2fa6..5cf1c615c 100755 --- a/xwords4/relay/scripts/relay.py +++ b/xwords4/relay/scripts/relay.py @@ -171,13 +171,13 @@ def query(req, params): print('params', params) params = json.loads(params) ids = params['ids'] - timeoutSecs = 'timeoutSecs' in params and float(params['timeoutSecs']) or 2.0 + # timeoutSecs = 'timeoutSecs' in params and float(params['timeoutSecs']) or 2.0 idsLen = 0 for id in ids: idsLen += len(id) tcpSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - tcpSock.settimeout(timeoutSecs) + # tcpSock.settimeout(timeoutSecs) tcpSock.connect(('127.0.0.1', 10998)) lenShort = 2 + idsLen + len(ids) + 2 @@ -188,8 +188,9 @@ def query(req, params): for id in ids: tcpSock.send(id + '\n') - msgsLists = {} + result = {'ids':ids} try: + msgsLists = {} shortUnpacker = struct.Struct('!H') resLen, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) # not getting all bytes nameCount, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) @@ -212,10 +213,14 @@ def query(req, params): msgs.append(msg) perGame.append(msgs) msgsLists[ids[ii]] = perGame + + result['msgs'] = msgsLists except: + # Anything but a timeout should mean we abort/send nothing + result['err'] = 'hit exception' None - return json.dumps(msgsLists) + return json.dumps(result) def main(): result = None From dbf38f77596ca86543bd6fb6ed91282966fb1c7c Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 6 Dec 2017 19:11:32 -0800 Subject: [PATCH 118/138] don't use mlock in config; don't close socket Use of mutex logging recurses infinitely if config uses mlock, so remove that. And don't free sockets after handling their messages as they may be in use elsewhere. This likely introduces a leak of sockets. --- xwords4/relay/configs.cpp | 6 ++++-- xwords4/relay/udpqueue.cpp | 7 ++++++- xwords4/relay/xwrelay.cpp | 11 ++++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/xwords4/relay/configs.cpp b/xwords4/relay/configs.cpp index 90b222149..fc50fd84e 100644 --- a/xwords4/relay/configs.cpp +++ b/xwords4/relay/configs.cpp @@ -84,12 +84,13 @@ RelayConfigs::GetValueFor( const char* key, time_t* value ) bool RelayConfigs::GetValueFor( const char* key, char* buf, int len ) { - MutexLock ml( &m_values_mutex ); + pthread_mutex_lock( &m_values_mutex ); map::const_iterator iter = m_values.find(key); bool found = iter != m_values.end(); if ( found ) { snprintf( buf, len, "%s", iter->second ); } + pthread_mutex_unlock( &m_values_mutex ); return found; } @@ -125,7 +126,7 @@ RelayConfigs::GetValueFor( const char* key, vector& ints ) void RelayConfigs::SetValueFor( const char* key, const char* value ) { - MutexLock ml( &m_values_mutex ); + pthread_mutex_lock( &m_values_mutex ); /* Remove any entry already there */ map::iterator iter = m_values.find(key); @@ -136,6 +137,7 @@ RelayConfigs::SetValueFor( const char* key, const char* value ) pair::iterator,bool> result = m_values.insert( pair(strdup(key),strdup(value) ) ); assert( result.second ); + pthread_mutex_unlock( &m_values_mutex ); } ino_t diff --git a/xwords4/relay/udpqueue.cpp b/xwords4/relay/udpqueue.cpp index b59e237b0..6ffab296e 100644 --- a/xwords4/relay/udpqueue.cpp +++ b/xwords4/relay/udpqueue.cpp @@ -103,7 +103,11 @@ UdpQueue::get() return s_instance; } -// return false if socket should no longer be used +// If we're already assembling data from this socket, continue. Otherwise +// create a new parital packet and store data there. If we wind up with a +// complete packet, dispatch it and delete since the data's been delivered. +// +// Return false if socket should no longer be used. bool UdpQueue::handle( const AddrInfo* addr, QueueCallback cb ) { @@ -148,6 +152,7 @@ UdpQueue::handle( const AddrInfo* addr, QueueCallback cb ) } success = success && (NULL == packet || packet->stillGood()); + logf( XW_LOGVERBOSE0, "%s(sock=%d) => %d", __func__, sock, success ); return success; } diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index f241382e9..7b2363541 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -1030,7 +1030,7 @@ processDisconnect( const uint8_t* bufp, int bufLen, const AddrInfo* addr ) } /* processDisconnect */ static void -killSocket( const AddrInfo* addr ) +rmSocketRefs( const AddrInfo* addr ) { logf( XW_LOGINFO, "%s(addr.socket=%d)", __func__, addr->getSocket() ); CRefMgr::Get()->RemoveSocketRefs( addr ); @@ -1304,6 +1304,7 @@ handleMsgsMsg( const AddrInfo* addr, bool sendFull, { unsigned short nameCount; if ( getNetShort( &bufp, end, &nameCount ) ) { + assert( nameCount == 1 ); // Don't commit this!!! DBMgr* dbmgr = DBMgr::Get(); vector out(4); /* space for len and n_msgs */ assert( out.size() == 4 ); @@ -1330,6 +1331,7 @@ handleMsgsMsg( const AddrInfo* addr, bool sendFull, break; } + logf( XW_LOGVERBOSE0, "%s(): connName: %s", __func__, connName ); dbmgr->RecordAddress( connName, hid, addr ); /* For each relayID, write the number of messages and then @@ -1359,6 +1361,8 @@ handleMsgsMsg( const AddrInfo* addr, bool sendFull, // before. But for some reason if I make that change apps wind up // stalling. dbmgr->RemoveStoredMessages( msgIDs ); + } else { + assert(0); } } } // handleMsgsMsg @@ -1568,7 +1572,8 @@ proxy_thread_proc( PacketThreadClosure* ptc ) } } } - XWThreadPool::GetTPool()->CloseSocket( addr ); + // Should I remove this, or make it into more of an unref() call? + // XWThreadPool::GetTPool()->CloseSocket( addr ); } // proxy_thread_proc static size_t @@ -2342,7 +2347,7 @@ main( int argc, char** argv ) (void)sigaction( SIGINT, &act, NULL ); XWThreadPool* tPool = XWThreadPool::GetTPool(); - tPool->Setup( nWorkerThreads, killSocket ); + tPool->Setup( nWorkerThreads, rmSocketRefs ); /* set up select call */ fd_set rfds; From 811c8f535ef81cd1d47dd9bc21c6acdea085d9ff Mon Sep 17 00:00:00 2001 From: Eric House Date: Wed, 6 Dec 2017 19:24:52 -0800 Subject: [PATCH 119/138] add socket refcounting AddrInfo now has ref()/unref() and keeps a global socket->refcount map (since actual AddrInfo instances come and go.) When the count drops to 0, the existing CloseSocket() method is called. This seems to fix a bunch of race conditions that had a socket being closed and reused while old code was still expecting to write to the device attached to the socket the first time (along with lots of calls to close() already-closed sockets, attempts to write() to closed sockets, etc.) --- xwords4/relay/Makefile | 1 + xwords4/relay/addrinfo.cpp | 40 ++++++++++++++++++++++++++++++++++++++ xwords4/relay/addrinfo.h | 6 ++++++ xwords4/relay/tpool.cpp | 5 +++-- xwords4/relay/udpqueue.cpp | 6 ++++-- xwords4/relay/udpqueue.h | 8 ++++++-- xwords4/relay/xwrelay.cpp | 2 +- 7 files changed, 61 insertions(+), 7 deletions(-) diff --git a/xwords4/relay/Makefile b/xwords4/relay/Makefile index 06fc29fa8..82cfaeae5 100644 --- a/xwords4/relay/Makefile +++ b/xwords4/relay/Makefile @@ -67,6 +67,7 @@ endif # turn on semaphore debugging # CPPFLAGS += -DDEBUG_LOCKS +# CPPFLAGS += -DLOG_POLL memdebug all: xwrelay rq diff --git a/xwords4/relay/addrinfo.cpp b/xwords4/relay/addrinfo.cpp index ead4105ca..546b828e6 100644 --- a/xwords4/relay/addrinfo.cpp +++ b/xwords4/relay/addrinfo.cpp @@ -20,13 +20,16 @@ */ #include +#include #include #include +#include #include "addrinfo.h" #include "xwrelay_priv.h" #include "tpool.h" #include "udpager.h" +#include "mlock.h" // static uint32_t s_prevCreated = 0L; @@ -82,3 +85,40 @@ AddrInfo::equals( const AddrInfo& other ) const return equal; } +static pthread_mutex_t s_refMutex = PTHREAD_MUTEX_INITIALIZER; +static map s_socketRefs; + +void AddrInfo::ref() const +{ + // logf( XW_LOGVERBOSE0, "%s(socket=%d)", __func__, m_socket ); + MutexLock ml( &s_refMutex ); + ++s_socketRefs[m_socket]; + printRefMap(); +} + +void +AddrInfo::unref() const +{ + // logf( XW_LOGVERBOSE0, "%s(socket=%d)", __func__, m_socket ); + + MutexLock ml( &s_refMutex ); + assert( s_socketRefs[m_socket] > 0 ); + --s_socketRefs[m_socket]; + if ( s_socketRefs[m_socket] == 0 ) { + XWThreadPool::GetTPool()->CloseSocket( this ); + } + printRefMap(); +} + +/* private, and assumes have mutex */ +void +AddrInfo::printRefMap() const +{ + /* for ( map::const_iterator iter = s_socketRefs.begin(); */ + /* iter != s_socketRefs.end(); ++iter ) { */ + /* int count = iter->second; */ + /* if ( count > 0 ) { */ + /* logf( XW_LOGVERBOSE0, "socket: %d; count: %d", iter->first, count ); */ + /* } */ + /* } */ +} diff --git a/xwords4/relay/addrinfo.h b/xwords4/relay/addrinfo.h index d92e70a1b..94b04a816 100644 --- a/xwords4/relay/addrinfo.h +++ b/xwords4/relay/addrinfo.h @@ -81,12 +81,18 @@ class AddrInfo { bool equals( const AddrInfo& other ) const; + /* refcount the underlying socket (doesn't modify instance) */ + void ref() const; + void unref() const; + int getref() const; + private: void construct( int sock, const AddrUnion* saddr, bool isTCP ); void init( int sock, ClientToken clientToken, const AddrUnion* saddr ) { construct( sock, saddr, false ); m_clientToken = clientToken; } + void printRefMap() const; // AddrInfo& operator=(const AddrInfo&); // Prevent assignment int m_socket; diff --git a/xwords4/relay/tpool.cpp b/xwords4/relay/tpool.cpp index 804be3de5..3ea401022 100644 --- a/xwords4/relay/tpool.cpp +++ b/xwords4/relay/tpool.cpp @@ -119,6 +119,7 @@ XWThreadPool::Stop() void XWThreadPool::AddSocket( SockType stype, QueueCallback proc, const AddrInfo* from ) { + from->ref(); int sock = from->getSocket(); logf( XW_LOGVERBOSE0, "%s(sock=%d, isTCP=%d)", __func__, sock, from->isTCP() ); SockInfo si = { .m_type = stype, @@ -174,7 +175,6 @@ XWThreadPool::RemoveSocket( const AddrInfo* addr ) void XWThreadPool::CloseSocket( const AddrInfo* addr ) { - int sock = addr->getSocket(); if ( addr->isTCP() ) { if ( !RemoveSocket( addr ) ) { MutexLock ml( &m_queueMutex ); @@ -187,6 +187,7 @@ XWThreadPool::CloseSocket( const AddrInfo* addr ) ++iter; } } + int sock = addr->getSocket(); int err = close( sock ); if ( 0 != err ) { logf( XW_LOGERROR, "%s(): close(socket=%d) => %d/%s", __func__, @@ -284,7 +285,7 @@ XWThreadPool::real_tpool_main( ThreadInfo* tip ) case Q_KILL: logf( XW_LOGINFO, "worker thread got socket %d from queue (to close it)", sock ); (*m_kFunc)( &pr.m_info.m_addr ); - CloseSocket( &pr.m_info.m_addr ); + pr.m_info.m_addr.unref(); break; } } else { diff --git a/xwords4/relay/udpqueue.cpp b/xwords4/relay/udpqueue.cpp index 6ffab296e..bb59a0892 100644 --- a/xwords4/relay/udpqueue.cpp +++ b/xwords4/relay/udpqueue.cpp @@ -58,7 +58,7 @@ PartialPacket::readAtMost( int len ) logf( XW_LOGERROR, "%s(len=%d, socket=%d): recv failed: %d (%s)", __func__, len, m_sock, m_errno, strerror(m_errno) ); } - } else if ( 0 == nRead ) { // remote socket closed + } else if ( 0 == nRead ) { // remote socket half-closed logf( XW_LOGINFO, "%s(): remote closed (socket=%d)", __func__, m_sock ); m_errno = -1; // so stillGood will fail } else { @@ -160,11 +160,12 @@ void UdpQueue::handle( const AddrInfo* addr, const uint8_t* buf, int len, QueueCallback cb ) { + // addr->ref(); PacketThreadClosure* ptc = new PacketThreadClosure( addr, buf, len, cb ); MutexLock ml( &m_queueMutex ); int id = ++m_nextID; ptc->setID( id ); - logf( XW_LOGINFO, "%s: enqueuing packet %d (socket %d, len %d)", + logf( XW_LOGINFO, "%s(): enqueuing packet %d (socket %d, len %d)", __func__, id, addr->getSocket(), len ); m_queue.push_back( ptc ); @@ -223,6 +224,7 @@ UdpQueue::thread_main() logf( XW_LOGINFO, "%s: dropping packet %d; it's %d seconds old!", __func__, age ); } + // ptc->addr()->unref(); delete ptc; } return NULL; diff --git a/xwords4/relay/udpqueue.h b/xwords4/relay/udpqueue.h index 3926b3903..befa99893 100644 --- a/xwords4/relay/udpqueue.h +++ b/xwords4/relay/udpqueue.h @@ -44,10 +44,14 @@ public: , m_cb(cb) , m_created(time( NULL )) { - memcpy( m_buf, buf, len ); + memcpy( m_buf, buf, len ); + m_addr.ref(); } - ~PacketThreadClosure() { delete[] m_buf; } + ~PacketThreadClosure() { + m_addr.unref(); + delete[] m_buf; + } const uint8_t* buf() const { return m_buf; } int len() const { return m_len; } diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index 7b2363541..e94fd7666 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -1490,7 +1490,7 @@ game_thread_proc( PacketThreadClosure* ptc ) { logf( XW_LOGVERBOSE0, "%s()", __func__ ); if ( !processMessage( ptc->buf(), ptc->len(), ptc->addr(), 0 ) ) { - XWThreadPool::GetTPool()->CloseSocket( ptc->addr() ); + // XWThreadPool::GetTPool()->CloseSocket( ptc->addr() ); } } From 530e0181e5237a8710192fcc665fa9bc780cf919 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 9 Dec 2017 13:30:08 -0800 Subject: [PATCH 120/138] list open sockets --- xwords4/relay/scripts/showinplay.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xwords4/relay/scripts/showinplay.sh b/xwords4/relay/scripts/showinplay.sh index 8a593ea02..51e7c6ca3 100755 --- a/xwords4/relay/scripts/showinplay.sh +++ b/xwords4/relay/scripts/showinplay.sh @@ -51,7 +51,8 @@ fi echo -n "Device (pid) count: $(pidof xwords | wc | awk '{print $2}')" echo "; relay pid[s]: $(pidof xwrelay)" -echo "Row count:" $(psql -t xwgames -c "select count(*) FROM games $QUERY;") +echo -n "Row count:" $(psql -t xwgames -c "select count(*) FROM games $QUERY;") +echo "; Relay sockets: $(for PID in $(pidof xwrelay); do ls /proc/$PID/fd; done | sort -un | tr '\n' ' ')" # Games echo "SELECT dead as d,connname,cid,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ From afda9ce4460e4ef6014cf1de9f1ea72dcaa6cf4d Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 9 Dec 2017 13:30:25 -0800 Subject: [PATCH 121/138] add assertion --- xwords4/relay/tpool.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xwords4/relay/tpool.cpp b/xwords4/relay/tpool.cpp index 3ea401022..8abd16277 100644 --- a/xwords4/relay/tpool.cpp +++ b/xwords4/relay/tpool.cpp @@ -120,6 +120,7 @@ void XWThreadPool::AddSocket( SockType stype, QueueCallback proc, const AddrInfo* from ) { from->ref(); + int sock = from->getSocket(); logf( XW_LOGVERBOSE0, "%s(sock=%d, isTCP=%d)", __func__, sock, from->isTCP() ); SockInfo si = { .m_type = stype, @@ -128,6 +129,7 @@ XWThreadPool::AddSocket( SockType stype, QueueCallback proc, const AddrInfo* fro }; { RWWriteLock ml( &m_activeSocketsRWLock ); + assert( m_activeSockets.find( sock ) == m_activeSockets.end() ); m_activeSockets.insert( pair( sock, si ) ); } interrupt_poll(); From 8d10dca23e22dc581ef994dc853df6338d6dde8a Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 9 Dec 2017 20:39:16 -0800 Subject: [PATCH 122/138] pull in test scripts from dev branch --- xwords4/linux/scripts/discon_ok2.py | 131 +++++++++++++++++++++------- xwords4/relay/scripts/showinplay.sh | 3 +- 2 files changed, 101 insertions(+), 33 deletions(-) diff --git a/xwords4/linux/scripts/discon_ok2.py b/xwords4/linux/scripts/discon_ok2.py index 87c1d5c52..127eba615 100755 --- a/xwords4/linux/scripts/discon_ok2.py +++ b/xwords4/linux/scripts/discon_ok2.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import re, os, sys, getopt, shutil, threading, requests, json, glob -import argparse, datetime, random, subprocess, time +import argparse, datetime, random, signal, subprocess, time # LOGDIR=./$(basename $0)_logs # APP_NEW="" @@ -161,7 +161,8 @@ class Device(): sTilesLeftPat = re.compile('.*pool_removeTiles: (\d+) tiles left in pool') sRelayIDPat = re.compile('.*UPDATE games.*seed=(\d+),.*relayid=\'([^\']+)\'.*') - def __init__(self, args, indx, app, params, room, db, log, nInGame): + def __init__(self, args, game, indx, app, params, room, db, log, nInGame): + self.game = game self.indx = indx self.args = args self.pid = 0 @@ -178,7 +179,7 @@ class Device(): self.devID = '' self.launchCount = 0 self.allDone = False # when true, can be killed - self.nTilesLeft = -1 # negative means don't know + self.nTilesLeft = None self.relayID = None self.relaySeed = 0 @@ -257,6 +258,12 @@ class Device(): self.proc = None self.check_game() + def handleAllDone(self): + if self.allDone: + self.moveFiles() + self.send_dead() + return self.allDone + def moveFiles(self): assert not self.running() shutil.move(self.logPath, self.args.LOGDIR + '/done') @@ -268,10 +275,9 @@ class Device(): req = requests.get(url, params = {'params' : JSON}) def getTilesCount(self): - result = None - if self.nTilesLeft != -1: - result = '%.2d:%.2d' % (self.indx, self.nTilesLeft) - return result + return {'index': self.indx, 'nTilesLeft': self.nTilesLeft, + 'launchCount': self.launchCount, 'game': self.game, + } def update_ldevid(self): if not self.app in Device.sHasLDevIDMap: @@ -306,6 +312,7 @@ class Device(): if allDone: for dev in Device.sConnnameMap[self.connname]: + assert self.game == dev.game dev.allDone = True # print('Closing', self.connname, datetime.datetime.now()) @@ -343,7 +350,6 @@ def build_cmds(args): for GAME in range(1, args.NGAMES + 1): ROOM = 'ROOM_%.3d' % (GAME % args.NROOMS) - # check_room $ROOM NDEVS = pick_ndevs(args) LOCALS = figure_locals(args, NDEVS) # as array NPLAYERS = sum(LOCALS) @@ -355,12 +361,9 @@ def build_cmds(args): DEV = 0 for NLOCALS in LOCALS: DEV += 1 - FILE="%s/GAME_%d_%d.sql3" % (args.LOGDIR, GAME, DEV) - LOG='%s/%d_%d_LOG.txt' % (args.LOGDIR, GAME, DEV) - # os.system("rm -f $LOG") # clear the log + DB = '{}/{:02d}_{:02d}_DB.sql3'.format(args.LOGDIR, GAME, DEV) + LOG = '{}/{:02d}_{:02d}_LOG.txt'.format(args.LOGDIR, GAME, DEV) - # APPS[$COUNTER]="$APP_NEW" - # NEW_ARGS[$COUNTER]="$APP_NEW_PARAMS" BOARD_SIZE = ['--board-size', '15'] # if [ 0 -lt ${#APPS_OLD[@]} ]; then # # 50% chance of starting out with old app @@ -379,7 +382,7 @@ def build_cmds(args): PARAMS += ['--undo-pct', args.UNDO_PCT] PARAMS += [ '--game-dict', DICT, '--relay-port', args.PORT, '--host', args.HOST] PARAMS += ['--slow-robot', '1:3', '--skip-confirm'] - PARAMS += ['--db', FILE] + PARAMS += ['--db', DB] if random.randint(0,100) % 100 < g_UDP_PCT_START: PARAMS += ['--use-udp'] @@ -409,7 +412,7 @@ def build_cmds(args): # print('PARAMS:', PARAMS) - dev = Device(args, COUNTER, args.APP_NEW, PARAMS, ROOM, FILE, LOG, len(LOCALS)) + dev = Device(args, GAME, COUNTER, args.APP_NEW, PARAMS, ROOM, DB, LOG, len(LOCALS)) dev.update_ldevid() devs.append(dev) @@ -627,22 +630,75 @@ def build_cmds(args): # fi # } -def summarizeTileCounts(devs): - nDevs = len(devs) - strs = [dev.getTilesCount() for dev in devs] - strs = [s for s in strs if s] - nWithTiles = len(strs) - print('%s %d/%d %s' % (datetime.datetime.now().strftime("%H:%M:%S"), nDevs, nWithTiles, ' '.join(strs))) +def summarizeTileCounts(devs, endTime, state): + shouldGoOn = True + data = [dev.getTilesCount() for dev in devs] + nDevs = len(data) + totalTiles = 0 + colWidth = max(2, len(str(nDevs))) + headWidth = 0 + fmtData = [{'head' : 'dev', }, + {'head' : 'launches', }, + {'head' : 'tls left', }, + ] + for datum in fmtData: + headWidth = max(headWidth, len(datum['head'])) + datum['data'] = [] + + # Group devices by game + games = [] + prev = -1 + for datum in data: + gameNo = datum['game'] + if gameNo != prev: + games.append([]) + prev = gameNo + games[-1].append('{:0{width}d}'.format(datum['index'], width=colWidth)) + fmtData[0]['data'] = ['+'.join(game) for game in games] + + nLaunches = 0 + for datum in data: + launchCount = datum['launchCount'] + nLaunches += launchCount + fmtData[1]['data'].append('{:{width}d}'.format(launchCount, width=colWidth)) + + nTiles = datum['nTilesLeft'] + fmtData[2]['data'].append(nTiles is None and ('-' * colWidth) or '{:{width}d}'.format(nTiles, width=colWidth)) + if not nTiles is None: totalTiles += int(nTiles) + + + print('') + print('devs left: {}; tiles left: {}; total launches: {}; {}/{}' + .format(nDevs, totalTiles, nLaunches, datetime.datetime.now(), endTime )) + fmt = '{head:>%d} {data}' % headWidth + for datum in fmtData: datum['data'] = ' '.join(datum['data']) + for datum in fmtData: + print(fmt.format(**datum)) + + # Now let's see if things are stuck: if the tile string hasn't + # changed in two minutes bail. Note that the count of tiles left + # isn't enough because it's zero for a long time as devices are + # using up what's left in their trays and getting killed. + now = datetime.datetime.now() + tilesStr = fmtData[2]['data'] + if not 'tilesStr' in state or state['tilesStr'] != tilesStr: + state['lastChange'] = now + state['tilesStr'] = tilesStr + + return now - state['lastChange'] < datetime.timedelta(minutes = 1) def countCores(): return len(glob.glob1('/tmp',"core*")) +gDone = False + def run_cmds(args, devs): nCores = countCores() - endTime = datetime.datetime.now() + datetime.timedelta(seconds = args.TIMEOUT) + endTime = datetime.datetime.now() + datetime.timedelta(minutes = args.TIMEOUT_MINS) LOOPCOUNT = 0 + printState = {} - while len(devs) > 0: + while len(devs) > 0 and not gDone: if countCores() > nCores: print('core file count increased; exiting') break @@ -651,13 +707,14 @@ def run_cmds(args, devs): break LOOPCOUNT += 1 - if 0 == LOOPCOUNT % 20: summarizeTileCounts(devs) + if 0 == LOOPCOUNT % 20: + if not summarizeTileCounts(devs, endTime, printState): + print('no change in too long; exiting') + break dev = random.choice(devs) if not dev.running(): - if dev.allDone: - dev.moveFiles() - dev.send_dead() + if dev.handleAllDone(): devs.remove(dev) else: # if [ -n "$ONE_PER_ROOM" -a 0 -ne ${ROOM_PIDS[$ROOM]} ]; then @@ -674,9 +731,11 @@ def run_cmds(args, devs): # MINEND[$KEY]=$(($NOW + $MINRUN)) elif not dev.minTimeExpired(): # print('sleeping...') - time.sleep(2) + time.sleep(1.0) else: dev.kill() + if dev.handleAllDone(): + devs.remove(dev) # if g_DROP_N >= 0: dev.increment_drop() # update_ldevid $KEY @@ -739,8 +798,8 @@ def mkParser(): parser.add_argument('--num-games', dest = 'NGAMES', type = int, default = 1, help = 'number of games') parser.add_argument('--num-rooms', dest = 'NROOMS', type = int, default = 0, help = 'number of roooms (default to --num-games)') - parser.add_argument('--no-timeout', dest = 'TIMEOUT', default = False, action = 'store_true', - help = 'run forever (default proportional to number of games') + parser.add_argument('--timeout-mins', dest = 'TIMEOUT_MINS', default = 10000, type = int, + help = 'minutes after which to timeout') parser.add_argument('--log-root', dest='LOGROOT', default = '.', help = 'where logfiles go') parser.add_argument('--dup-packets', dest = 'DUP_PACKETS', default = False, help = 'send all packet twice') parser.add_argument('--use-gtk', dest = 'USE_GTK', default = False, action = 'store_true', @@ -768,7 +827,6 @@ def mkParser(): help = 'Port relay\'s on') parser.add_argument('--resign-pct', dest = 'RESIGN_PCT', default = 0, type = int, \ help = 'Odds of resigning [0..100]') - # # echo " [--no-timeout] # run until all games done \\" >&2 parser.add_argument('--seed', type = int, dest = 'SEED', default = random.randint(1, 1000000000)) # # echo " [--send-chat \\" >&2 @@ -900,7 +958,6 @@ def parseArgs(): def assignDefaults(args): if not args.NROOMS: args.NROOMS = args.NGAMES - args.TIMEOUT = not args.TIMEOUT and (args.NGAMES * 60 + 500) or 100000000000 if len(args.DICTS) == 0: args.DICTS.append('CollegeEng_2to8.xwd') args.LOGDIR = os.path.basename(sys.argv[0]) + '_logs' # Move an existing logdir aside @@ -975,10 +1032,20 @@ def assignDefaults(args): # SECONDS=$((SECONDS%60)) # echo "*********$0 finished: $(date) (took $HOURS:$MINUTES:$SECONDS)**************" +def termHandler(signum, frame): + global gDone + print('termHandler() called') + gDone = True + def main(): + startTime = datetime.datetime.now() + signal.signal(signal.SIGINT, termHandler) + args = parseArgs() devs = build_cmds(args) + nDevs = len(devs) run_cmds(args, devs) + print('{} finished; took {} for {} devices'.format(sys.argv[0], datetime.datetime.now() - startTime, nDevs)) ############################################################################## if __name__ == '__main__': diff --git a/xwords4/relay/scripts/showinplay.sh b/xwords4/relay/scripts/showinplay.sh index 8a593ea02..51e7c6ca3 100755 --- a/xwords4/relay/scripts/showinplay.sh +++ b/xwords4/relay/scripts/showinplay.sh @@ -51,7 +51,8 @@ fi echo -n "Device (pid) count: $(pidof xwords | wc | awk '{print $2}')" echo "; relay pid[s]: $(pidof xwrelay)" -echo "Row count:" $(psql -t xwgames -c "select count(*) FROM games $QUERY;") +echo -n "Row count:" $(psql -t xwgames -c "select count(*) FROM games $QUERY;") +echo "; Relay sockets: $(for PID in $(pidof xwrelay); do ls /proc/$PID/fd; done | sort -un | tr '\n' ' ')" # Games echo "SELECT dead as d,connname,cid,room,lang as lg,clntVers as cv ,ntotal as t,nperdevice as npd,nsents as snts, seeds,devids,tokens,ack, mtimes "\ From 2e71aedc024bf4ba504973afb0fbdd74e0ad1ab5 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 9 Dec 2017 21:32:05 -0800 Subject: [PATCH 123/138] fix --game-dict option --- xwords4/linux/scripts/discon_ok2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwords4/linux/scripts/discon_ok2.sh b/xwords4/linux/scripts/discon_ok2.sh index 19d914e7c..dc959547d 100755 --- a/xwords4/linux/scripts/discon_ok2.sh +++ b/xwords4/linux/scripts/discon_ok2.sh @@ -43,7 +43,7 @@ declare -A LOGS declare -A MINEND declare -A ROOM_PIDS declare -a APPS_OLD=() -declare -a DICTS= # wants to be =() too? +declare -a DICTS=() # wants to be =() too? declare -A CHECKED_ROOMS function cleanup() { From fc4e577f1b8f20a4e4fcad56ba901f15996930d6 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 10 Dec 2017 08:21:22 -0800 Subject: [PATCH 124/138] bring changes over from dev branch old tests pass without any socket leakage --- xwords4/relay/addrinfo.cpp | 42 ++++++++++++++++++++++- xwords4/relay/addrinfo.h | 6 ++++ xwords4/relay/configs.cpp | 6 ++-- xwords4/relay/tpool.cpp | 58 +++++++++++++++++++------------ xwords4/relay/udpqueue.cpp | 45 +++++++++++++++--------- xwords4/relay/udpqueue.h | 20 ++++++----- xwords4/relay/xwrelay.cpp | 70 ++++++++++++++++++++++---------------- 7 files changed, 170 insertions(+), 77 deletions(-) diff --git a/xwords4/relay/addrinfo.cpp b/xwords4/relay/addrinfo.cpp index 32839d1fd..546b828e6 100644 --- a/xwords4/relay/addrinfo.cpp +++ b/xwords4/relay/addrinfo.cpp @@ -20,13 +20,16 @@ */ #include +#include #include #include +#include #include "addrinfo.h" #include "xwrelay_priv.h" #include "tpool.h" #include "udpager.h" +#include "mlock.h" // static uint32_t s_prevCreated = 0L; @@ -68,7 +71,7 @@ AddrInfo::equals( const AddrInfo& other ) const if ( isTCP() ) { equal = m_socket == other.m_socket; if ( equal && created() != other.created() ) { - logf( XW_LOGINFO, "%s: rejecting on time mismatch (%lx vs %lx)", + logf( XW_LOGINFO, "%s(): rejecting on time mismatch (%lx vs %lx)", __func__, created(), other.created() ); equal = false; } @@ -82,3 +85,40 @@ AddrInfo::equals( const AddrInfo& other ) const return equal; } +static pthread_mutex_t s_refMutex = PTHREAD_MUTEX_INITIALIZER; +static map s_socketRefs; + +void AddrInfo::ref() const +{ + // logf( XW_LOGVERBOSE0, "%s(socket=%d)", __func__, m_socket ); + MutexLock ml( &s_refMutex ); + ++s_socketRefs[m_socket]; + printRefMap(); +} + +void +AddrInfo::unref() const +{ + // logf( XW_LOGVERBOSE0, "%s(socket=%d)", __func__, m_socket ); + + MutexLock ml( &s_refMutex ); + assert( s_socketRefs[m_socket] > 0 ); + --s_socketRefs[m_socket]; + if ( s_socketRefs[m_socket] == 0 ) { + XWThreadPool::GetTPool()->CloseSocket( this ); + } + printRefMap(); +} + +/* private, and assumes have mutex */ +void +AddrInfo::printRefMap() const +{ + /* for ( map::const_iterator iter = s_socketRefs.begin(); */ + /* iter != s_socketRefs.end(); ++iter ) { */ + /* int count = iter->second; */ + /* if ( count > 0 ) { */ + /* logf( XW_LOGVERBOSE0, "socket: %d; count: %d", iter->first, count ); */ + /* } */ + /* } */ +} diff --git a/xwords4/relay/addrinfo.h b/xwords4/relay/addrinfo.h index d92e70a1b..94b04a816 100644 --- a/xwords4/relay/addrinfo.h +++ b/xwords4/relay/addrinfo.h @@ -81,12 +81,18 @@ class AddrInfo { bool equals( const AddrInfo& other ) const; + /* refcount the underlying socket (doesn't modify instance) */ + void ref() const; + void unref() const; + int getref() const; + private: void construct( int sock, const AddrUnion* saddr, bool isTCP ); void init( int sock, ClientToken clientToken, const AddrUnion* saddr ) { construct( sock, saddr, false ); m_clientToken = clientToken; } + void printRefMap() const; // AddrInfo& operator=(const AddrInfo&); // Prevent assignment int m_socket; diff --git a/xwords4/relay/configs.cpp b/xwords4/relay/configs.cpp index 90b222149..fc50fd84e 100644 --- a/xwords4/relay/configs.cpp +++ b/xwords4/relay/configs.cpp @@ -84,12 +84,13 @@ RelayConfigs::GetValueFor( const char* key, time_t* value ) bool RelayConfigs::GetValueFor( const char* key, char* buf, int len ) { - MutexLock ml( &m_values_mutex ); + pthread_mutex_lock( &m_values_mutex ); map::const_iterator iter = m_values.find(key); bool found = iter != m_values.end(); if ( found ) { snprintf( buf, len, "%s", iter->second ); } + pthread_mutex_unlock( &m_values_mutex ); return found; } @@ -125,7 +126,7 @@ RelayConfigs::GetValueFor( const char* key, vector& ints ) void RelayConfigs::SetValueFor( const char* key, const char* value ) { - MutexLock ml( &m_values_mutex ); + pthread_mutex_lock( &m_values_mutex ); /* Remove any entry already there */ map::iterator iter = m_values.find(key); @@ -136,6 +137,7 @@ RelayConfigs::SetValueFor( const char* key, const char* value ) pair::iterator,bool> result = m_values.insert( pair(strdup(key),strdup(value) ) ); assert( result.second ); + pthread_mutex_unlock( &m_values_mutex ); } ino_t diff --git a/xwords4/relay/tpool.cpp b/xwords4/relay/tpool.cpp index 9e402c744..8abd16277 100644 --- a/xwords4/relay/tpool.cpp +++ b/xwords4/relay/tpool.cpp @@ -119,13 +119,17 @@ XWThreadPool::Stop() void XWThreadPool::AddSocket( SockType stype, QueueCallback proc, const AddrInfo* from ) { + from->ref(); + + int sock = from->getSocket(); + logf( XW_LOGVERBOSE0, "%s(sock=%d, isTCP=%d)", __func__, sock, from->isTCP() ); + SockInfo si = { .m_type = stype, + .m_proc = proc, + .m_addr = *from + }; { - int sock = from->getSocket(); RWWriteLock ml( &m_activeSocketsRWLock ); - SockInfo si; - si.m_type = stype; - si.m_proc = proc; - si.m_addr = *from; + assert( m_activeSockets.find( sock ) == m_activeSockets.end() ); m_activeSockets.insert( pair( sock, si ) ); } interrupt_poll(); @@ -158,13 +162,14 @@ XWThreadPool::RemoveSocket( const AddrInfo* addr ) size_t prevSize = m_activeSockets.size(); - map::iterator iter = m_activeSockets.find( addr->getSocket() ); + int sock = addr->getSocket(); + map::iterator iter = m_activeSockets.find( sock ); if ( m_activeSockets.end() != iter && iter->second.m_addr.equals( *addr ) ) { m_activeSockets.erase( iter ); found = true; } - logf( XW_LOGINFO, "%s: AFTER: %d sockets active (was %d)", __func__, - m_activeSockets.size(), prevSize ); + logf( XW_LOGINFO, "%s(): AFTER closing %d: %d sockets active (was %d)", __func__, + sock, m_activeSockets.size(), prevSize ); } return found; } /* RemoveSocket */ @@ -184,8 +189,14 @@ XWThreadPool::CloseSocket( const AddrInfo* addr ) ++iter; } } - logf( XW_LOGINFO, "CLOSING socket %d", addr->getSocket() ); - close( addr->getSocket() ); + int sock = addr->getSocket(); + int err = close( sock ); + if ( 0 != err ) { + logf( XW_LOGERROR, "%s(): close(socket=%d) => %d/%s", __func__, + sock, errno, strerror(errno) ); + } else { + logf( XW_LOGINFO, "%s(): close(socket=%d) succeeded", __func__, sock ); + } /* We always need to interrupt the poll because the socket we're closing will be in the list being listened to. That or we need to drop sockets @@ -198,7 +209,7 @@ XWThreadPool::CloseSocket( const AddrInfo* addr ) void XWThreadPool::EnqueueKill( const AddrInfo* addr, const char* const why ) { - logf( XW_LOGINFO, "%s(%d) reason: %s", __func__, addr->getSocket(), why ); + logf( XW_LOGINFO, "%s(socket = %d) reason: %s", __func__, addr->getSocket(), why ); if ( addr->isTCP() ) { SockInfo si; si.m_type = STYPE_UNKNOWN; @@ -265,7 +276,6 @@ XWThreadPool::real_tpool_main( ThreadInfo* tip ) if ( gotOne ) { sock = pr.m_info.m_addr.getSocket(); - logf( XW_LOGINFO, "worker thread got socket %d from queue", socket ); switch ( pr.m_act ) { case Q_READ: assert( 0 ); @@ -275,8 +285,9 @@ XWThreadPool::real_tpool_main( ThreadInfo* tip ) // } break; case Q_KILL: + logf( XW_LOGINFO, "worker thread got socket %d from queue (to close it)", sock ); (*m_kFunc)( &pr.m_info.m_addr ); - CloseSocket( &pr.m_info.m_addr ); + pr.m_info.m_addr.unref(); break; } } else { @@ -392,35 +403,40 @@ XWThreadPool::real_listener() curfd = 1; int ii; - for ( ii = 0; ii < nSockets && nEvents > 0; ++ii ) { + for ( ii = 0; ii < nSockets && nEvents > 0; ++ii, ++curfd ) { if ( fds[curfd].revents != 0 ) { // int socket = fds[curfd].fd; SockInfo* sinfo = &sinfos[curfd]; const AddrInfo* addr = &sinfo->m_addr; - assert( fds[curfd].fd == addr->getSocket() ); + int sock = addr->getSocket(); + assert( fds[curfd].fd == sock ); if ( !SocketFound( addr ) ) { + logf( XW_LOGINFO, "%s(): dropping socket %d: not found", + __func__, addr->getSocket() ); /* no further processing if it's been removed while - we've been sleeping in poll */ + we've been sleeping in poll. BUT: shouldn't curfd + be incremented?? */ --nEvents; continue; } if ( 0 != (fds[curfd].revents & (POLLIN | POLLPRI)) ) { if ( !UdpQueue::get()->handle( addr, sinfo->m_proc ) ) { + // This is likely wrong!!! return of 0 means + // remote closed, not error. RemoveSocket( addr ); - EnqueueKill( addr, "bad packet" ); + EnqueueKill( addr, "got EOF" ); } } else { - logf( XW_LOGERROR, "odd revents: %x", - fds[curfd].revents ); + logf( XW_LOGERROR, "%s(): odd revents: %x; bad socket %d", + __func__, fds[curfd].revents, sock ); RemoveSocket( addr ); - EnqueueKill( addr, "error/hup in poll()" ); + EnqueueKill( addr, "error/hup in poll()" ); } --nEvents; } - ++curfd; } assert( nEvents == 0 ); } diff --git a/xwords4/relay/udpqueue.cpp b/xwords4/relay/udpqueue.cpp index 675c28ec3..bb59a0892 100644 --- a/xwords4/relay/udpqueue.cpp +++ b/xwords4/relay/udpqueue.cpp @@ -28,7 +28,7 @@ static UdpQueue* s_instance = NULL; void -UdpThreadClosure::logStats() +PacketThreadClosure::logStats() { time_t now = time( NULL ); if ( 1 < now - m_created ) { @@ -48,6 +48,7 @@ PartialPacket::stillGood() const bool PartialPacket::readAtMost( int len ) { + assert( len > 0 ); bool success = false; uint8_t tmp[len]; ssize_t nRead = recv( m_sock, tmp, len, 0 ); @@ -57,10 +58,12 @@ PartialPacket::readAtMost( int len ) logf( XW_LOGERROR, "%s(len=%d, socket=%d): recv failed: %d (%s)", __func__, len, m_sock, m_errno, strerror(m_errno) ); } - } else if ( 0 == nRead ) { // remote socket closed - logf( XW_LOGINFO, "%s: remote closed (socket=%d)", __func__, m_sock ); + } else if ( 0 == nRead ) { // remote socket half-closed + logf( XW_LOGINFO, "%s(): remote closed (socket=%d)", __func__, m_sock ); m_errno = -1; // so stillGood will fail } else { + // logf( XW_LOGVERBOSE0, "%s(): read %d bytes on socket %d", __func__, + // nRead, m_sock ); m_errno = 0; success = len == nRead; int curSize = m_buf.size(); @@ -100,7 +103,11 @@ UdpQueue::get() return s_instance; } -// return false if socket should no longer be used +// If we're already assembling data from this socket, continue. Otherwise +// create a new parital packet and store data there. If we wind up with a +// complete packet, dispatch it and delete since the data's been delivered. +// +// Return false if socket should no longer be used. bool UdpQueue::handle( const AddrInfo* addr, QueueCallback cb ) { @@ -145,6 +152,7 @@ UdpQueue::handle( const AddrInfo* addr, QueueCallback cb ) } success = success && (NULL == packet || packet->stillGood()); + logf( XW_LOGVERBOSE0, "%s(sock=%d) => %d", __func__, sock, success ); return success; } @@ -152,17 +160,21 @@ void UdpQueue::handle( const AddrInfo* addr, const uint8_t* buf, int len, QueueCallback cb ) { - UdpThreadClosure* utc = new UdpThreadClosure( addr, buf, len, cb ); + // addr->ref(); + PacketThreadClosure* ptc = new PacketThreadClosure( addr, buf, len, cb ); MutexLock ml( &m_queueMutex ); int id = ++m_nextID; - utc->setID( id ); - logf( XW_LOGINFO, "%s: enqueuing packet %d (socket %d, len %d)", + ptc->setID( id ); + logf( XW_LOGINFO, "%s(): enqueuing packet %d (socket %d, len %d)", __func__, id, addr->getSocket(), len ); - m_queue.push_back( utc ); + m_queue.push_back( ptc ); pthread_cond_signal( &m_queueCondVar ); } +// Remove any PartialPacket record with the same socket/fd. This makes sense +// when the socket's being reused or when we have just dealt with a single +// packet and might be getting more. void UdpQueue::newSocket_locked( int sock ) { @@ -194,25 +206,26 @@ UdpQueue::thread_main() while ( m_queue.size() == 0 ) { pthread_cond_wait( &m_queueCondVar, &m_queueMutex ); } - UdpThreadClosure* utc = m_queue.front(); + PacketThreadClosure* ptc = m_queue.front(); m_queue.pop_front(); pthread_mutex_unlock( &m_queueMutex ); - utc->noteDequeued(); + ptc->noteDequeued(); - time_t age = utc->ageInSeconds(); + time_t age = ptc->ageInSeconds(); if ( 30 > age ) { logf( XW_LOGINFO, "%s: dispatching packet %d (socket %d); " - "%d seconds old", __func__, utc->getID(), - utc->addr()->getSocket(), age ); - (*utc->cb())( utc ); - utc->logStats(); + "%d seconds old", __func__, ptc->getID(), + ptc->addr()->getSocket(), age ); + (*ptc->cb())( ptc ); + ptc->logStats(); } else { logf( XW_LOGINFO, "%s: dropping packet %d; it's %d seconds old!", __func__, age ); } - delete utc; + // ptc->addr()->unref(); + delete ptc; } return NULL; } diff --git a/xwords4/relay/udpqueue.h b/xwords4/relay/udpqueue.h index cc467af75..befa99893 100644 --- a/xwords4/relay/udpqueue.h +++ b/xwords4/relay/udpqueue.h @@ -30,13 +30,13 @@ using namespace std; -class UdpThreadClosure; +class PacketThreadClosure; -typedef void (*QueueCallback)( UdpThreadClosure* closure ); +typedef void (*QueueCallback)( PacketThreadClosure* closure ); -class UdpThreadClosure { +class PacketThreadClosure { public: - UdpThreadClosure( const AddrInfo* addr, const uint8_t* buf, + PacketThreadClosure( const AddrInfo* addr, const uint8_t* buf, int len, QueueCallback cb ) : m_buf(new uint8_t[len]) , m_len(len) @@ -44,10 +44,14 @@ public: , m_cb(cb) , m_created(time( NULL )) { - memcpy( m_buf, buf, len ); + memcpy( m_buf, buf, len ); + m_addr.ref(); } - ~UdpThreadClosure() { delete[] m_buf; } + ~PacketThreadClosure() { + m_addr.unref(); + delete[] m_buf; + } const uint8_t* buf() const { return m_buf; } int len() const { return m_len; } @@ -109,8 +113,8 @@ class UdpQueue { pthread_mutex_t m_partialsMutex; pthread_mutex_t m_queueMutex; pthread_cond_t m_queueCondVar; - deque m_queue; - // map > m_bySocket; + deque m_queue; + // map > m_bySocket; int m_nextID; map m_partialPackets; }; diff --git a/xwords4/relay/xwrelay.cpp b/xwords4/relay/xwrelay.cpp index def2b44f1..e94fd7666 100644 --- a/xwords4/relay/xwrelay.cpp +++ b/xwords4/relay/xwrelay.cpp @@ -124,8 +124,6 @@ logf( XW_LogLevel level, const char* format, ... ) va_end(ap); #else FILE* where = NULL; - struct tm* timp; - struct timeval tv; bool useFile; char logFile[256]; @@ -143,13 +141,14 @@ logf( XW_LogLevel level, const char* format, ... ) if ( !!where ) { static int tm_yday = 0; + struct timeval tv; gettimeofday( &tv, NULL ); struct tm result; - timp = localtime_r( &tv.tv_sec, &result ); + struct tm* timp = localtime_r( &tv.tv_sec, &result ); char timeBuf[64]; - sprintf( timeBuf, "%.2d:%.2d:%.2d", timp->tm_hour, - timp->tm_min, timp->tm_sec ); + sprintf( timeBuf, "%.2d:%.2d:%.2d.%03ld", timp->tm_hour, + timp->tm_min, timp->tm_sec, tv.tv_usec / 1000 ); /* log the date once/day. This isn't threadsafe so may be repeated but that's harmless. */ @@ -1031,7 +1030,7 @@ processDisconnect( const uint8_t* bufp, int bufLen, const AddrInfo* addr ) } /* processDisconnect */ static void -killSocket( const AddrInfo* addr ) +rmSocketRefs( const AddrInfo* addr ) { logf( XW_LOGINFO, "%s(addr.socket=%d)", __func__, addr->getSocket() ); CRefMgr::Get()->RemoveSocketRefs( addr ); @@ -1304,14 +1303,17 @@ handleMsgsMsg( const AddrInfo* addr, bool sendFull, const uint8_t* bufp, const uint8_t* end ) { unsigned short nameCount; - int ii; if ( getNetShort( &bufp, end, &nameCount ) ) { + assert( nameCount == 1 ); // Don't commit this!!! DBMgr* dbmgr = DBMgr::Get(); vector out(4); /* space for len and n_msgs */ assert( out.size() == 4 ); vector msgIDs; - for ( ii = 0; ii < nameCount && bufp < end; ++ii ) { - + for ( int ii = 0; ii < nameCount; ++ii ) { + if ( bufp >= end ) { + logf( XW_LOGERROR, "%s(): ran off the end", __func__ ); + break; + } // See NetUtils.java for reply format // message-length: 2 // nameCount: 2 @@ -1329,6 +1331,7 @@ handleMsgsMsg( const AddrInfo* addr, bool sendFull, break; } + logf( XW_LOGVERBOSE0, "%s(): connName: %s", __func__, connName ); dbmgr->RecordAddress( connName, hid, addr ); /* For each relayID, write the number of messages and then @@ -1345,14 +1348,21 @@ handleMsgsMsg( const AddrInfo* addr, bool sendFull, memcpy( &out[0], &tmp, sizeof(tmp) ); tmp = htons( nameCount ); memcpy( &out[2], &tmp, sizeof(tmp) ); - ssize_t nwritten = write( addr->getSocket(), &out[0], out.size() ); - logf( XW_LOGVERBOSE0, "%s: wrote %d bytes", __func__, nwritten ); - if ( sendFull && nwritten >= 0 && (size_t)nwritten == out.size() ) { + int sock = addr->getSocket(); + ssize_t nWritten = write( sock, &out[0], out.size() ); + if ( nWritten < 0 ) { + logf( XW_LOGERROR, "%s(): write to socket %d failed: %d/%s", __func__, + sock, errno, strerror(errno) ); + } else if ( sendFull && (size_t)nWritten == out.size() ) { + logf( XW_LOGVERBOSE0, "%s(): wrote %d bytes to socket %d", __func__, + nWritten, sock ); dbmgr->RecordSent( &msgIDs[0], msgIDs.size() ); // This is wrong: should be removed when ACK returns and not // before. But for some reason if I make that change apps wind up // stalling. dbmgr->RemoveStoredMessages( msgIDs ); + } else { + assert(0); } } } // handleMsgsMsg @@ -1476,23 +1486,24 @@ handleProxyMsgs( int sock, const AddrInfo* addr, const uint8_t* bufp, } // handleProxyMsgs static void -game_thread_proc( UdpThreadClosure* utc ) +game_thread_proc( PacketThreadClosure* ptc ) { - if ( !processMessage( utc->buf(), utc->len(), utc->addr(), 0 ) ) { - XWThreadPool::GetTPool()->CloseSocket( utc->addr() ); + logf( XW_LOGVERBOSE0, "%s()", __func__ ); + if ( !processMessage( ptc->buf(), ptc->len(), ptc->addr(), 0 ) ) { + // XWThreadPool::GetTPool()->CloseSocket( ptc->addr() ); } } static void -proxy_thread_proc( UdpThreadClosure* utc ) +proxy_thread_proc( PacketThreadClosure* ptc ) { - const int len = utc->len(); - const AddrInfo* addr = utc->addr(); + const int len = ptc->len(); + const AddrInfo* addr = ptc->addr(); if ( len > 0 ) { assert( addr->isTCP() ); int sock = addr->getSocket(); - const uint8_t* bufp = utc->buf(); + const uint8_t* bufp = ptc->buf(); const uint8_t* end = bufp + len; if ( (0 == *bufp++) ) { /* protocol */ XWPRXYCMD cmd = (XWPRXYCMD)*bufp++; @@ -1561,7 +1572,8 @@ proxy_thread_proc( UdpThreadClosure* utc ) } } } - XWThreadPool::GetTPool()->CloseSocket( addr ); + // Should I remove this, or make it into more of an unref() call? + // XWThreadPool::GetTPool()->CloseSocket( addr ); } // proxy_thread_proc static size_t @@ -1726,10 +1738,10 @@ ackPacketIf( const UDPHeader* header, const AddrInfo* addr ) } static void -handle_udp_packet( UdpThreadClosure* utc ) +handle_udp_packet( PacketThreadClosure* ptc ) { - const uint8_t* ptr = utc->buf(); - const uint8_t* end = ptr + utc->len(); + const uint8_t* ptr = ptc->buf(); + const uint8_t* end = ptr + ptc->len(); UDPHeader header; if ( getHeader( &ptr, end, &header ) ) { @@ -1752,7 +1764,7 @@ handle_udp_packet( UdpThreadClosure* utc ) if ( 3 >= clientVers ) { checkAllAscii( model, "bad model" ); } - registerDevice( relayID, &devID, utc->addr(), + registerDevice( relayID, &devID, ptc->addr(), clientVers, devDesc, model, osVers ); } } @@ -1765,7 +1777,7 @@ handle_udp_packet( UdpThreadClosure* utc ) ptr += sizeof(clientToken); clientToken = ntohl( clientToken ); if ( AddrInfo::NULL_TOKEN != clientToken ) { - AddrInfo addr( g_udpsock, clientToken, utc->saddr() ); + AddrInfo addr( g_udpsock, clientToken, ptc->saddr() ); (void)processMessage( ptr, end - ptr, &addr, clientToken ); } else { logf( XW_LOGERROR, "%s: dropping packet with token of 0", @@ -1786,7 +1798,7 @@ handle_udp_packet( UdpThreadClosure* utc ) } SafeCref scr( connName, hid ); if ( scr.IsValid() ) { - AddrInfo addr( g_udpsock, clientToken, utc->saddr() ); + AddrInfo addr( g_udpsock, clientToken, ptc->saddr() ); handlePutMessage( scr, hid, &addr, end - ptr, &ptr, end ); assert( ptr == end ); // DON'T CHECK THIS IN!!! } else { @@ -1821,7 +1833,7 @@ handle_udp_packet( UdpThreadClosure* utc ) case XWPDEV_RQSTMSGS: { DevID devID( ID_TYPE_RELAY ); if ( getVLIString( &ptr, end, devID.m_devIDString ) ) { - const AddrInfo* addr = utc->addr(); + const AddrInfo* addr = ptc->addr(); DevMgr::Get()->rememberDevice( devID.asRelayID(), addr ); if ( XWPDEV_RQSTMSGS == header.cmd ) { @@ -1862,7 +1874,7 @@ handle_udp_packet( UdpThreadClosure* utc ) } // Do this after the device and address are registered - ackPacketIf( &header, utc->addr() ); + ackPacketIf( &header, ptc->addr() ); } } @@ -2335,7 +2347,7 @@ main( int argc, char** argv ) (void)sigaction( SIGINT, &act, NULL ); XWThreadPool* tPool = XWThreadPool::GetTPool(); - tPool->Setup( nWorkerThreads, killSocket ); + tPool->Setup( nWorkerThreads, rmSocketRefs ); /* set up select call */ fd_set rfds; From 701e6968de2b9ced69a0c0375f8b9560d8c4e714 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 10 Dec 2017 11:56:52 -0800 Subject: [PATCH 125/138] file to pull an apk app-id is hardcoded for now. --- xwords4/android/scripts/adb-pull-apk.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 xwords4/android/scripts/adb-pull-apk.sh diff --git a/xwords4/android/scripts/adb-pull-apk.sh b/xwords4/android/scripts/adb-pull-apk.sh new file mode 100755 index 000000000..106eb9ea3 --- /dev/null +++ b/xwords4/android/scripts/adb-pull-apk.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e -u + +APP_ID=org.eehouse.android.xw4 + +APK_PATH=$(adb shell pm path $APP_ID) +APK_PATH=${APK_PATH/package:/} +adb pull $APK_PATH From d1a5a9474062e68da895814c43fd4d2b92b1c99a Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 11 Dec 2017 06:42:35 -0800 Subject: [PATCH 126/138] fix crash processing multiple packets When grouping to allow multiple packets per outbound API call I forgot that some are there to mark the end-of-queue: can't be sent! Trying caused a NPE. Now if any EOQ is found in the queue that batch is dropped and the thread's exited. --- .../org/eehouse/android/xw4/RelayService.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index 29ffd94f2..f2a378026 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -618,13 +618,18 @@ public class RelayService extends XWService for ( ; ; ) { List dataList = null; try { - PacketData outData = m_queue.take(); - if ( null != outData ) { - dataList = new ArrayList<>(); + dataList = new ArrayList<>(); + for ( PacketData outData = m_queue.take(); // blocks + null != outData; + outData = m_queue.poll() ) { // doesn't block + if ( outData.isEOQ() ) { + dataList = null; + break; + } dataList.add(outData); - m_queue.drainTo(dataList); + Log.d( TAG, "got %d packets; %d more left", dataList.size(), + m_queue.size()); } - Log.d( TAG, "got %d packets; %d more left", dataList.size(), m_queue.size()); } catch ( InterruptedException ie ) { Log.w( TAG, "write thread killed" ); break; @@ -665,6 +670,7 @@ public class RelayService extends XWService try { JSONArray dataArray = new JSONArray(); for ( PacketData packet : packets ) { + Assert.assertFalse( packet.isEOQ() ); byte[] datum = packet.assemble(); dataArray.put( Utils.base64Encode(datum) ); sentLen += datum.length; @@ -1554,6 +1560,8 @@ public class RelayService extends XWService System.currentTimeMillis() - m_created ); } + public boolean isEOQ() { return 0 == getLength(); } + public int getLength() { int result = 0; From 4fa8079bf0bd7ffb8fbfdf4af6d68b3513c955bb Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 11 Dec 2017 06:55:25 -0800 Subject: [PATCH 127/138] log outgoing "post" params json --- .../app/src/main/java/org/eehouse/android/xw4/RelayService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index f2a378026..d6372f30b 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -680,7 +680,7 @@ public class RelayService extends XWService String result = NetUtils.runConn(conn, params); if ( null != result ) { - Log.d( TAG, "sendViaWeb(): POST => %s", result ); + Log.d( TAG, "sendViaWeb(): POST(%s) => %s", params, result ); JSONObject resultObj = new JSONObject( result ); JSONArray resData = resultObj.getJSONArray( "data" ); int nReplies = resData.length(); From 4746f153e33a398c43fc24fb95ec159cc45714e0 Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 11 Dec 2017 06:56:53 -0800 Subject: [PATCH 128/138] add assertion Need to revisit that join() call; this'll help. --- .../app/src/main/java/org/eehouse/android/xw4/RelayService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index d6372f30b..d9c49e57f 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -763,6 +763,8 @@ public class RelayService extends XWService private void stopUDPThreadsIf() { + DbgUtils.assertOnUIThread(); + if ( null != m_UDPWriteThread ) { // can't add null m_queue.add( new PacketData() ); From ddadee9bef4e72cb3aeaf48f5fc547bc63e1bfa1 Mon Sep 17 00:00:00 2001 From: Eric House Date: Mon, 11 Dec 2017 07:09:44 -0800 Subject: [PATCH 129/138] copy in changes from dev branch --- xwords4/relay/scripts/relay.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/xwords4/relay/scripts/relay.py b/xwords4/relay/scripts/relay.py index 7ca6c2fa6..5cf1c615c 100755 --- a/xwords4/relay/scripts/relay.py +++ b/xwords4/relay/scripts/relay.py @@ -171,13 +171,13 @@ def query(req, params): print('params', params) params = json.loads(params) ids = params['ids'] - timeoutSecs = 'timeoutSecs' in params and float(params['timeoutSecs']) or 2.0 + # timeoutSecs = 'timeoutSecs' in params and float(params['timeoutSecs']) or 2.0 idsLen = 0 for id in ids: idsLen += len(id) tcpSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - tcpSock.settimeout(timeoutSecs) + # tcpSock.settimeout(timeoutSecs) tcpSock.connect(('127.0.0.1', 10998)) lenShort = 2 + idsLen + len(ids) + 2 @@ -188,8 +188,9 @@ def query(req, params): for id in ids: tcpSock.send(id + '\n') - msgsLists = {} + result = {'ids':ids} try: + msgsLists = {} shortUnpacker = struct.Struct('!H') resLen, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) # not getting all bytes nameCount, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) @@ -212,10 +213,14 @@ def query(req, params): msgs.append(msg) perGame.append(msgs) msgsLists[ids[ii]] = perGame + + result['msgs'] = msgsLists except: + # Anything but a timeout should mean we abort/send nothing + result['err'] = 'hit exception' None - return json.dumps(msgsLists) + return json.dumps(result) def main(): result = None From e96a982f243797126b35e6439c93e1cf08b0a498 Mon Sep 17 00:00:00 2001 From: Eric House Date: Tue, 12 Dec 2017 06:59:31 -0800 Subject: [PATCH 130/138] get rid of moved file already --- xwords4/android/scripts/relay.py | 250 ------------------------------- 1 file changed, 250 deletions(-) delete mode 100755 xwords4/android/scripts/relay.py diff --git a/xwords4/android/scripts/relay.py b/xwords4/android/scripts/relay.py deleted file mode 100755 index 7ca6c2fa6..000000000 --- a/xwords4/android/scripts/relay.py +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/python - -import base64, json, mod_python, socket, struct, sys -import psycopg2, random - -PROTOCOL_VERSION = 0 -PRX_DEVICE_GONE = 3 -PRX_GET_MSGS = 4 - -# try: -# from mod_python import apache -# apacheAvailable = True -# except ImportError: -# apacheAvailable = False - -# Joining a game. Basic idea is you have stuff to match on (room, -# number in game, language) and when somebody wants to join you add to -# an existing matching game if there's space otherwise create a new -# one. Problems are the unreliablity of transport: if you give a space -# and the device doesn't get the message you can't hold it forever. So -# device provides a seed that holds the space. If it asks again for a -# space with the same seed it gets the same space. If it never asks -# again (app deleted, say), the space needs eventually to be given to -# somebody else. I think that's done by adding a timestamp array and -# treating the space as available if TIME has expired. Need to think -# about this: what if app fails to ACK for TIME, then returns with -# seed to find it given away. Let's do a 30 minute reservation for -# now? [Note: much of this is PENDING] - -def join(req, devID, room, seed, hid = 0, lang = 1, nInGame = 2, nHere = 1, inviteID = None): - assert hid <= 4 - seed = int(seed) - assert seed != 0 - nInGame = int(nInGame) - nHere = int(nHere) - assert nHere <= nInGame - assert nInGame <= 4 - - devID = int(devID, 16) - - connname = None - logs = [] # for debugging - # logs.append('vers: ' + platform.python_version()) - - con = psycopg2.connect(database='xwgames') - cur = con.cursor() - # cur.execute('LOCK TABLE games IN ACCESS EXCLUSIVE MODE') - - # First see if there's a game with a space for me. Must match on - # room, lang and size. Must have room OR must have already given a - # spot for a seed equal to mine, in which case I get it - # back. Assumption is I didn't ack in time. - - query = "SELECT connname, seeds, nperdevice FROM games " - query += "WHERE lang = %s AND nTotal = %s AND room = %s " - query += "AND (njoined + %s <= ntotal OR %s = ANY(seeds)) " - query += "LIMIT 1" - cur.execute( query, (lang, nInGame, room, nHere, seed)) - for row in cur: - (connname, seeds, nperdevice) = row - print('found', connname, seeds, nperdevice) - break # should be only one! - - # If we've found a match, we either need to UPDATE or, if the - # seeds match, remind the caller of where he belongs. If a hid's - # been specified, we honor it by updating if the slot's available; - # otherwise a new game has to be created. - if connname: - if seed in seeds and nHere == nperdevice[seeds.index(seed)]: - hid = seeds.index(seed) + 1 - print('resusing seed case; outta here!') - else: - if hid == 0: - # Any gaps? Assign it - if None in seeds: - hid = seeds.index(None) + 1 - else: - hid = len(seeds) + 1 - print('set hid to', hid, 'based on ', seeds) - else: - print('hid already', hid) - query = "UPDATE games SET njoined = njoined + %s, " - query += "devids[%d] = %%s, " % hid - query += "seeds[%d] = %%s, " % hid - query += "jtimes[%d] = 'now', " % hid - query += "nperdevice[%d] = %%s " % hid - query += "WHERE connname = %s " - print(query) - params = (nHere, devID, seed, nHere, connname) - cur.execute(query, params) - - # If nothing was found, add a new game and add me. Honor my hid - # preference if specified - if not connname: - # This requires python3, which likely requires mod_wsgi - # ts = datetime.datetime.utcnow().timestamp() - # connname = '%s:%d:1' % (xwconfig.k_HOSTNAME, int(ts * 1000)) - connname = '%s:%d:1' % (xwconfig.k_HOSTNAME, random.randint(0, 10000000000)) - useHid = hid == 0 and 1 or hid - print('not found case; inserting using hid:', useHid) - query = "INSERT INTO games (connname, room, lang, ntotal, njoined, " + \ - "devids[%d], seeds[%d], jtimes[%d], nperdevice[%d]) " % (4 * (useHid,)) - query += "VALUES (%s, %s, %s, %s, %s, %s, %s, 'now', %s) " - query += "RETURNING connname, array_length(seeds,1); " - cur.execute(query, (connname, room, lang, nInGame, nHere, devID, seed, nHere)) - for row in cur: - connname, gothid = row - break - if hid == 0: hid = gothid - - con.commit() - con.close() - - result = {'connname': connname, 'hid' : hid, 'log' : ':'.join(logs)} - - return json.dumps(result) - -def kill(req, params): - print(params) - params = json.loads(params) - count = len(params) - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('127.0.0.1', 10998)) - - header = struct.Struct('!BBh') - strLens = 0 - for ii in range(count): - strLens += len(params[ii]['relayID']) + 1 - size = header.size + (2*count) + strLens - sock.send(struct.Struct('!h').pack(size)) - sock.send(header.pack(PROTOCOL_VERSION, PRX_DEVICE_GONE, count)) - - for ii in range(count): - elem = params[ii] - asBytes = bytes(elem['relayID']) - sock.send(struct.Struct('!H%dsc' % (len(asBytes))).pack(elem['seed'], asBytes, '\n')) - sock.close() - - result = {'err': 0} - return json.dumps(result) - -# winds up in handle_udp_packet() in xwrelay.cpp -def post(req, params): - err = 'none' - params = json.loads(params) - data = params['data'] - timeoutSecs = 'timeoutSecs' in params and params['timeoutSecs'] or 1.0 - binData = [base64.b64decode(datum) for datum in data] - - udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - udpSock.settimeout(float(timeoutSecs)) # seconds - addr = ("127.0.0.1", 10997) - for binDatum in binData: - udpSock.sendto(binDatum, addr) - - responses = [] - while True: - try: - data, server = udpSock.recvfrom(1024) - responses.append(base64.b64encode(data)) - except socket.timeout: - #If data is not received back from server, print it has timed out - err = 'timeout' - break - - result = {'err' : err, 'data' : responses} - return json.dumps(result) - -def query(req, params): - print('params', params) - params = json.loads(params) - ids = params['ids'] - timeoutSecs = 'timeoutSecs' in params and float(params['timeoutSecs']) or 2.0 - - idsLen = 0 - for id in ids: idsLen += len(id) - - tcpSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - tcpSock.settimeout(timeoutSecs) - tcpSock.connect(('127.0.0.1', 10998)) - - lenShort = 2 + idsLen + len(ids) + 2 - print(lenShort, PROTOCOL_VERSION, PRX_GET_MSGS, len(ids)) - header = struct.Struct('!hBBh') - assert header.size == 6 - tcpSock.send(header.pack(lenShort, PROTOCOL_VERSION, PRX_GET_MSGS, len(ids))) - - for id in ids: tcpSock.send(id + '\n') - - msgsLists = {} - try: - shortUnpacker = struct.Struct('!H') - resLen, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) # not getting all bytes - nameCount, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) - resLen -= shortUnpacker.size - print('resLen:', resLen, 'nameCount:', nameCount) - if nameCount == len(ids) and resLen > 0: - print('nameCount', nameCount) - for ii in range(nameCount): - perGame = [] - countsThisGame, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) # problem - print('countsThisGame:', countsThisGame) - for jj in range(countsThisGame): - msgLen, = shortUnpacker.unpack(tcpSock.recv(shortUnpacker.size)) - print('msgLen:', msgLen) - msgs = [] - if msgLen > 0: - msg = tcpSock.recv(msgLen) - print('msg len:', len(msg)) - msg = base64.b64encode(msg) - msgs.append(msg) - perGame.append(msgs) - msgsLists[ids[ii]] = perGame - except: - None - - return json.dumps(msgsLists) - -def main(): - result = None - if len(sys.argv) > 1: - cmd = sys.argv[1] - args = sys.argv[2:] - if cmd == 'query' and len(args) > 0: - result = query(None, json.dumps({'ids':args})) - elif cmd == 'post': - # Params = { 'data' : 'V2VkIE9jdCAxOCAwNjowNDo0OCBQRFQgMjAxNwo=' } - # params = json.dumps(params) - # print(post(None, params)) - pass - elif cmd == 'join': - if len(args) == 6: - result = join(None, 1, args[0], int(args[1]), int(args[2]), int(args[3]), int(args[4]), int(args[5])) - elif cmd == 'kill': - result = kill( None, json.dumps([{'relayID': args[0], 'seed':int(args[1])}]) ) - - if result: - print '->', result - else: - print 'USAGE: query [connname/hid]*' - print ' join ' - print ' query [connname/hid]*' - # print ' post ' - print ' kill ' - print ' join ' - -############################################################################## -if __name__ == '__main__': - main() From bda545fc8e17d7d8b31a33b2411a16d01c5ac0fe Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 14 Dec 2017 06:39:40 -0800 Subject: [PATCH 131/138] comment out iface On Linode anyway the relay's unreachable from python scripts unless I do this, and it seems harmless in local tests too. --- xwords4/relay/xwrelay.conf_tmplate | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/xwords4/relay/xwrelay.conf_tmplate b/xwords4/relay/xwrelay.conf_tmplate index 2b6862ecc..fce4469fd 100644 --- a/xwords4/relay/xwrelay.conf_tmplate +++ b/xwords4/relay/xwrelay.conf_tmplate @@ -28,8 +28,12 @@ DEVICE_PORTS=10998 # Port for per-device UDP interface (experimental) UDP_PORT=10997 -# interface to listen on -- may get dup packets if not specified -UDP_IFACE=eth0 + +# interface to listen on -- may get dup packets if not specified. BUT: +# at least on Linode specifying this leads to an socket that can't be +# reached from localhost, e.g. by python scripts, and local tests pass +# fine without it. So the dup packets thing may no longer apply. +# UDP_IFACE=eth0 # How long after we've read from an address before we assume it's # recycled. Also sent to clients as a suggested ping interval From c7a635285cb88178397179b60d8fd016bd97db72 Mon Sep 17 00:00:00 2001 From: Eric House Date: Thu, 14 Dec 2017 06:47:03 -0800 Subject: [PATCH 132/138] change text and names around native-vs-web choice The plan's to use the native relay protocol first, then to fall back to the slower but more reliable (esp. on paranoid/block-everything wifi networks) webAPI. This is the name change without behavior change (except that the native kill() to report deleted games is gone.) --- .../org/eehouse/android/xw4/NetUtils.java | 62 +------------------ .../org/eehouse/android/xw4/RelayService.java | 12 ++-- .../java/org/eehouse/android/xw4/XWPrefs.java | 4 +- .../app/src/main/res/values/common_rsrc.xml | 2 +- .../app/src/main/res/values/strings.xml | 4 +- .../android/app/src/main/res/xml/xwprefs.xml | 6 +- 6 files changed, 15 insertions(+), 75 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java index fa8e4df5c..acf6ac4be 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/NetUtils.java @@ -90,16 +90,8 @@ public class NetUtils { m_obits = obits; } + @Override public void run() - { - if ( XWPrefs.getPreferWebAPI( m_context ) ) { - runViaWeb(); - } else { - runWithProxySocket(); - } - } - - private void runViaWeb() { try { JSONArray params = new JSONArray(); @@ -110,69 +102,19 @@ public class NetUtils { params.put( one ); } HttpURLConnection conn = makeHttpRelayConn( m_context, "kill" ); - Log.d( TAG, "runViaWeb(): kill params: %s", params.toString() ); String resStr = runConn( conn, params ); - Log.d( TAG, "runViaWeb(): kill => %s", resStr ); + Log.d( TAG, "runViaWeb(): kill(%s) => %s", params, resStr ); if ( null != resStr ) { JSONObject result = new JSONObject( resStr ); if ( 0 == result.optInt( "err", -1 ) ) { DBUtils.clearObits( m_context, m_obits ); } - } else { - Log.e( TAG, "runViaWeb(): KILL => null" ); } } catch ( JSONException ex ) { Assert.assertFalse( BuildConfig.DEBUG ); } } - - private void runWithProxySocket() - { - Socket socket = makeProxySocket( m_context, 10000 ); - if ( null != socket ) { - int strLens = 0; - int nObits = 0; - for ( int ii = 0; ii < m_obits.length; ++ii ) { - String relayID = m_obits[ii].m_relayID; - if ( null != relayID ) { - ++nObits; - strLens += relayID.length() + 1; // 1 for /n - } - } - - try { - DataOutputStream outStream = - new DataOutputStream( socket.getOutputStream() ); - outStream.writeShort( 2 + 2 + (2*nObits) + strLens ); - outStream.writeByte( NetUtils.PROTOCOL_VERSION ); - outStream.writeByte( NetUtils.PRX_DEVICE_GONE ); - outStream.writeShort( m_obits.length ); - - for ( int ii = 0; ii < m_obits.length; ++ii ) { - String relayID = m_obits[ii].m_relayID; - if ( null != relayID ) { - outStream.writeShort( m_obits[ii].m_seed ); - outStream.writeBytes( relayID ); - outStream.write( '\n' ); - } - } - - outStream.flush(); - - DataInputStream dis = - new DataInputStream( socket.getInputStream() ); - short resLen = dis.readShort(); - socket.close(); - - if ( resLen == 0 ) { - DBUtils.clearObits( m_context, m_obits ); - } - } catch ( java.io.IOException ioe ) { - Log.ex( TAG, ioe ); - } - } - } } public static void informOfDeaths( Context context ) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index d9c49e57f..f4b7ac714 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -640,7 +640,7 @@ public class RelayService extends XWService } int sentLen; - if ( XWPrefs.getPreferWebAPI( RelayService.this ) ) { + if ( XWPrefs.getSkipToWebAPI( RelayService.this ) ) { sentLen = sendViaWeb( dataList ); } else { sentLen = sendViaUDP( dataList ); @@ -715,9 +715,7 @@ public class RelayService extends XWService try { DatagramPacket udpPacket = new DatagramPacket( data, data.length ); m_UDPSocket.send( udpPacket ); - if ( BuildConfig.DEBUG ) { - Assert.assertFalse( XWPrefs.getPreferWebAPI( this ) ); - } + sentLen += udpPacket.getLength(); noteSent( packet ); getOut = false; @@ -746,7 +744,7 @@ public class RelayService extends XWService { int pid = packet.m_packetID; Log.d( TAG, "Sent [udp?] packet: cmd=%s, id=%d", - packet.m_cmd.toString(), pid); + packet.m_cmd.toString(), pid ); if ( packet.m_cmd != XWRelayReg.XWPDEV_ACK ) { synchronized( s_packetsSent ) { s_packetsSent.put( pid, packet ); @@ -1188,7 +1186,7 @@ public class RelayService extends XWService @Override protected Void doInBackground( Void... ignored ) { - Assert.assertFalse( XWPrefs.getPreferWebAPI( m_context ) ); + Assert.assertFalse( XWPrefs.getSkipToWebAPI( m_context ) ); // format: total msg lenth: 2 // number-of-relayIDs: 2 // for-each-relayid: relayid + '\n': varies @@ -1237,7 +1235,7 @@ public class RelayService extends XWService // Now open a real socket, write size and proto, and // copy in the formatted buffer - Assert.assertFalse( XWPrefs.getPreferWebAPI( m_context ) ); + Assert.assertFalse( XWPrefs.getSkipToWebAPI( m_context ) ); Socket socket = NetUtils.makeProxySocket( m_context, 8000 ); if ( null != socket ) { DataOutputStream outStream = diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWPrefs.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWPrefs.java index 813fcede8..6747f05a2 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWPrefs.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/XWPrefs.java @@ -120,9 +120,9 @@ public class XWPrefs { return getPrefsString( context, R.string.key_relay_url ); } - public static boolean getPreferWebAPI( Context context ) + public static boolean getSkipToWebAPI( Context context ) { - return getPrefsBoolean( context, R.string.key_relay_via_http, false ); + return getPrefsBoolean( context, R.string.key_relay_via_http_first, false ); } public static int getDefaultProxyPort( Context context ) diff --git a/xwords4/android/app/src/main/res/values/common_rsrc.xml b/xwords4/android/app/src/main/res/values/common_rsrc.xml index 89bda9697..50ab6a10b 100644 --- a/xwords4/android/app/src/main/res/values/common_rsrc.xml +++ b/xwords4/android/app/src/main/res/values/common_rsrc.xml @@ -36,7 +36,7 @@ key_relay_host key_relay_port2 - key_relay_via_http + key_relay_via_http_first key_update_url key_relay_url key_update_prerel diff --git a/xwords4/android/app/src/main/res/values/strings.xml b/xwords4/android/app/src/main/res/values/strings.xml index 48312a6ca..cee42c6cd 100644 --- a/xwords4/android/app/src/main/res/values/strings.xml +++ b/xwords4/android/app/src/main/res/values/strings.xml @@ -2486,8 +2486,8 @@ For debugging You should never need these... Relay host - Use Web APIs - (instead of custom protocol to ports below) + Use Web APIs first + (instead of as fallback for custom protocol) Wordlist download URL Enable logging (release builds only) diff --git a/xwords4/android/app/src/main/res/xml/xwprefs.xml b/xwords4/android/app/src/main/res/xml/xwprefs.xml index 1ede6a0b9..43dc96048 100644 --- a/xwords4/android/app/src/main/res/xml/xwprefs.xml +++ b/xwords4/android/app/src/main/res/xml/xwprefs.xml @@ -426,9 +426,9 @@ android:defaultValue="@string/default_host" /> - From 2db67ed3395a97f178028752fa20afd55b439fe3 Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 15 Dec 2017 07:12:14 -0800 Subject: [PATCH 133/138] try send via udp, then web Send each packet via UDP if that's thought to be working (always is, now) and start a 10-second timer. If it hasn't been ack'd by then, resend via Web API. Tested by configuring to use a UDP socket that the relay isn't listening on. Only problem is that the backoff timers are broken: never stops sending every few seconds. --- .../org/eehouse/android/xw4/RelayService.java | 180 +++++++++++------- 1 file changed, 113 insertions(+), 67 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index f4b7ac714..42d2f0fd7 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -99,7 +99,8 @@ public class RelayService extends XWService private static final String ROWID = "ROWID"; private static final String BINBUFFER = "BINBUFFER"; - private static Map s_packetsSent = new HashMap<>(); + private static Map s_packetsSentUDP = new HashMap<>(); + private static Map s_packetsSentWeb = new HashMap<>(); private static AtomicInteger s_nextPacketID = new AtomicInteger(); private static boolean s_gcmWorking = false; private static boolean s_registered = false; @@ -119,6 +120,8 @@ public class RelayService extends XWService private Runnable m_onInactivity; private int m_maxIntervalSeconds = 0; private long m_lastGamePacketReceived; + // m_nativeNotWorking: set to true if too many acks missed? + private boolean m_nativeNotWorking = false; private static DevIDType s_curType = DevIDType.ID_TYPE_NONE; private static long s_regStartTime = 0; @@ -411,7 +414,7 @@ public class RelayService extends XWService byte[][][] msgss = expandMsgsArray( intent ); for ( byte[][] msgs : msgss ) { for ( byte[] msg : msgs ) { - gotPacket( msg, true ); + gotPacket( msg, true, false ); } } break; @@ -609,6 +612,15 @@ public class RelayService extends XWService } } + private boolean skipNativeSend() + { + boolean skip = m_nativeNotWorking; + if ( ! skip ) { + skip = XWPrefs.getSkipToWebAPI( RelayService.this ); + } + return skip; + } + private void startWriteThread() { if ( null == m_UDPWriteThread ) { @@ -616,35 +628,35 @@ public class RelayService extends XWService public void run() { Log.i( TAG, "write thread starting" ); for ( ; ; ) { - List dataList = null; + boolean exitNow = false; + boolean useWeb = skipNativeSend(); + List dataListUDP = new ArrayList<>(); + List dataListWeb = new ArrayList<>(); try { - dataList = new ArrayList<>(); for ( PacketData outData = m_queue.take(); // blocks null != outData; outData = m_queue.poll() ) { // doesn't block if ( outData.isEOQ() ) { - dataList = null; + exitNow = true; break; } - dataList.add(outData); - Log.d( TAG, "got %d packets; %d more left", dataList.size(), - m_queue.size()); + if ( useWeb || outData.getForWeb() ) { + dataListWeb.add(outData); + } else { + dataListUDP.add(outData); + } } } catch ( InterruptedException ie ) { Log.w( TAG, "write thread killed" ); break; } - if ( null == dataList ) { + if ( exitNow ) { Log.i( TAG, "stopping write thread" ); break; } - int sentLen; - if ( XWPrefs.getSkipToWebAPI( RelayService.this ) ) { - sentLen = sendViaWeb( dataList ); - } else { - sentLen = sendViaUDP( dataList ); - } + sendViaWeb( dataListWeb ); + sendViaUDP( dataListUDP ); resetExitTimer(); ConnStatusHandler.showSuccessOut(); @@ -663,44 +675,43 @@ public class RelayService extends XWService { Log.d( TAG, "sendViaWeb(): sending %d at once", packets.size() ); int sentLen = 0; - HttpURLConnection conn = NetUtils.makeHttpRelayConn( this, "post" ); - if ( null == conn ) { - Log.e( TAG, "sendViaWeb(): null conn for POST" ); - } else { - try { - JSONArray dataArray = new JSONArray(); - for ( PacketData packet : packets ) { - Assert.assertFalse( packet.isEOQ() ); - byte[] datum = packet.assemble(); - dataArray.put( Utils.base64Encode(datum) ); - sentLen += datum.length; - } - JSONObject params = new JSONObject(); - params.put( "data", dataArray ); - - String result = NetUtils.runConn(conn, params); - if ( null != result ) { - Log.d( TAG, "sendViaWeb(): POST(%s) => %s", params, result ); - JSONObject resultObj = new JSONObject( result ); - JSONArray resData = resultObj.getJSONArray( "data" ); - int nReplies = resData.length(); - Log.d( TAG, "sendViaWeb(): got %d replies", nReplies ); - - noteSent( packets ); // before we process the acks below :-) - - if ( nReplies > 0 ) { - resetExitTimer(); + if ( packets.size() > 0 ) { + HttpURLConnection conn = NetUtils.makeHttpRelayConn( this, "post" ); + if ( null == conn ) { + Log.e( TAG, "sendViaWeb(): null conn for POST" ); + } else { + try { + JSONArray dataArray = new JSONArray(); + for ( PacketData packet : packets ) { + Assert.assertFalse( packet.isEOQ() ); + byte[] datum = packet.assemble(); + dataArray.put( Utils.base64Encode(datum) ); + sentLen += datum.length; } - for ( int ii = 0; ii < nReplies; ++ii ) { - byte[] datum = Utils.base64Decode( resData.getString( ii ) ); - // PENDING: skip ack or not - gotPacket( datum, false ); + JSONObject params = new JSONObject(); + params.put( "data", dataArray ); + + String result = NetUtils.runConn(conn, params); + if ( null != result ) { + Log.d( TAG, "sendViaWeb(): POST(%s) => %s", params, result ); + JSONObject resultObj = new JSONObject( result ); + JSONArray resData = resultObj.getJSONArray( "data" ); + int nReplies = resData.length(); + Log.d( TAG, "sendViaWeb(): got %d replies", nReplies ); + + noteSent( packets, s_packetsSentWeb ); // before we process the acks below :-) + + for ( int ii = 0; ii < nReplies; ++ii ) { + byte[] datum = Utils.base64Decode( resData.getString( ii ) ); + // PENDING: skip ack or not + gotPacket( datum, false, false ); + } + } else { + Log.e( TAG, "sendViaWeb(): failed result for POST" ); } - } else { - Log.e( TAG, "sendViaWeb(): failed result for POST" ); + } catch ( JSONException ex ) { + Assert.assertFalse( BuildConfig.DEBUG ); } - } catch ( JSONException ex ) { - Assert.assertFalse( BuildConfig.DEBUG ); } } return sentLen; @@ -717,7 +728,7 @@ public class RelayService extends XWService m_UDPSocket.send( udpPacket ); sentLen += udpPacket.getLength(); - noteSent( packet ); + noteSent( packet, s_packetsSentUDP ); getOut = false; } catch ( java.net.SocketException se ) { Log.ex( TAG, se ); @@ -737,25 +748,54 @@ public class RelayService extends XWService break; } } + + if ( sentLen > 0 ) { + startAckTimer( packets ); + } + return sentLen; } - private void noteSent( PacketData packet ) + private void startAckTimer( final List packets ) + { + Runnable ackTimer = new Runnable() { + @Override + public void run() { + List forResend = new ArrayList<>(); + Log.d( TAG, "ackTimer.run() called" ); + synchronized ( s_packetsSentUDP ) { + for ( PacketData packet : packets ) { + PacketData stillThere = s_packetsSentUDP.remove(packet.m_packetID); + if ( stillThere != null ) { + Log.d( TAG, "packed %d not yet acked; resending", + stillThere.m_packetID ); + stillThere.setForWeb(); + forResend.add( stillThere ); + } + } + } + m_queue.addAll( forResend ); + } + }; + m_handler.postDelayed( ackTimer, 10 * 1000 ); + } + + private void noteSent( PacketData packet, Map map ) { int pid = packet.m_packetID; Log.d( TAG, "Sent [udp?] packet: cmd=%s, id=%d", packet.m_cmd.toString(), pid ); if ( packet.m_cmd != XWRelayReg.XWPDEV_ACK ) { - synchronized( s_packetsSent ) { - s_packetsSent.put( pid, packet ); + synchronized( map ) { + map.put( pid, packet ); } } } - private void noteSent( List packets ) + private void noteSent( List packets, Map map ) { for ( PacketData packet : packets ) { - noteSent( packet ); + noteSent( packet, map ); } } @@ -789,7 +829,7 @@ public class RelayService extends XWService } // MIGHT BE Running on reader thread - private void gotPacket( byte[] data, boolean skipAck ) + private void gotPacket( byte[] data, boolean skipAck, boolean fromUDP ) { boolean resetBackoff = false; ByteArrayInputStream bis = new ByteArrayInputStream( data ); @@ -868,7 +908,7 @@ public class RelayService extends XWService startService( intent ); break; case XWPDEV_ACK: - noteAck( vli2un( dis ) ); + noteAck( vli2un( dis ), fromUDP ); break; // case XWPDEV_MSGFWDOTHERS: // Assert.assertTrue( 0 == dis.readByte() ); // protocol; means "invite", I guess. @@ -897,7 +937,7 @@ public class RelayService extends XWService byte[] data = new byte[packetLen]; System.arraycopy( packet.getData(), 0, data, 0, packetLen ); // DbgUtils.logf( "RelayService::gotPacket: %d bytes of data", packetLen ); - gotPacket( data, false ); + gotPacket( data, false, true ); } // gotPacket private boolean shouldRegister() @@ -1317,23 +1357,25 @@ public class RelayService extends XWService return nextPacketID; } - private static void noteAck( int packetID ) + private static void noteAck( int packetID, boolean fromUDP ) { PacketData packet; - synchronized( s_packetsSent ) { - packet = s_packetsSent.remove( packetID ); + Map map = fromUDP ? s_packetsSentUDP : s_packetsSentWeb; + synchronized( map ) { + packet = map.remove( packetID ); if ( packet != null ) { - Log.d( TAG, "noteAck(): removed for id %d: %s", packetID, packet ); + Log.d( TAG, "noteAck(fromUDP=%b): removed for id %d: %s", + fromUDP, packetID, packet ); } else { Log.w( TAG, "Weird: got ack %d but never sent", packetID ); } if ( BuildConfig.DEBUG ) { ArrayList pstrs = new ArrayList<>(); - for ( Integer pkid : s_packetsSent.keySet() ) { - pstrs.add( s_packetsSent.get(pkid).toString() ); + for ( Integer pkid : map.keySet() ) { + pstrs.add( map.get(pkid).toString() ); } - Log.d( TAG, "noteAck(): Got ack for %d; there are %d unacked packets: %s", - packetID, s_packetsSent.size(), TextUtils.join( ",", pstrs ) ); + Log.d( TAG, "noteAck(fromUDP=%b): Got ack for %d; there are %d unacked packets: %s", + fromUDP, packetID, map.size(), TextUtils.join( ",", pstrs ) ); } } } @@ -1540,6 +1582,7 @@ public class RelayService extends XWService public byte[] m_header; public int m_packetID; private long m_created; + private boolean m_useWeb; public PacketData() { m_bas = null; @@ -1560,6 +1603,9 @@ public class RelayService extends XWService System.currentTimeMillis() - m_created ); } + void setForWeb() { m_useWeb = true; } + boolean getForWeb() { return m_useWeb; } + public boolean isEOQ() { return 0 == getLength(); } public int getLength() From b0befa62e303b406cab320c61d3f620346a53b52 Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 15 Dec 2017 07:17:14 -0800 Subject: [PATCH 134/138] remove logging --- .../app/src/main/java/org/eehouse/android/xw4/RelayService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java index 89c24ebb4..d5d00c1e3 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/RelayService.java @@ -160,7 +160,7 @@ public class RelayService extends XWService { boolean enabled = ! XWPrefs .getPrefsBoolean( context, R.string.key_disable_relay, false ); - Log.d( TAG, "relayEnabled() => %b", enabled ); + // Log.d( TAG, "relayEnabled() => %b", enabled ); return enabled; } From 39dbe679299f33d39c68c14fb8cb589fa6f0c732 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sat, 16 Dec 2017 08:06:20 -0800 Subject: [PATCH 135/138] add archive menuitem As with Rematch, you should have the option once you've dismissed the special alert that comes up when finished games are opened. --- .../eehouse/android/xw4/BoardDelegate.java | 53 +++++++++++++------ .../app/src/main/res/menu/board_menu.xml | 4 ++ 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/BoardDelegate.java b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/BoardDelegate.java index b90febc40..7a498c8b1 100644 --- a/xwords4/android/app/src/main/java/org/eehouse/android/xw4/BoardDelegate.java +++ b/xwords4/android/app/src/main/java/org/eehouse/android/xw4/BoardDelegate.java @@ -206,23 +206,16 @@ public class BoardDelegate extends DelegateBase ab.setNegativeButton( R.string.button_rematch, lstnr ); // If we're not already in the "archive" group, offer to move - final String archiveName = LocUtils - .getString( m_activity, R.string.group_name_archive ); - final long archiveGroup = DBUtils.getGroup( m_activity, archiveName ); - long curGroup = DBUtils.getGroupForGame( m_activity, m_rowid ); - if ( curGroup != archiveGroup ) { + if ( !inArchiveGroup() ) { lstnr = new OnClickListener() { public void onClick( DialogInterface dlg, int whichButton ) { - makeNotAgainBuilder( R.string.not_again_archive, - R.string.key_na_archive, - Action.ARCHIVE_ACTION ) - .setParams( archiveName, archiveGroup ) - .show(); + showArchiveNA(); } }; ab.setNeutralButton( R.string.button_archive, lstnr ); } + } else if ( DlgID.DLG_CONNSTAT == dlgID && BuildConfig.DEBUG && null != m_connTypes && (m_connTypes.contains( CommsConnType.COMMS_CONN_RELAY ) @@ -847,6 +840,9 @@ public class BoardDelegate extends DelegateBase enable = m_gameOver && rematchSupported( false ); Utils.setItemVisible( menu, R.id.board_menu_rematch, enable ); + enable = m_gameOver && !inArchiveGroup(); + Utils.setItemVisible( menu, R.id.board_menu_archive, enable ); + boolean netGame = null != m_gi && DeviceRole.SERVER_STANDALONE != m_gi.serverRole; Utils.setItemVisible( menu, R.id.gamel_menu_checkmoves, netGame ); @@ -890,6 +886,10 @@ public class BoardDelegate extends DelegateBase doRematchIf(); break; + case R.id.board_menu_archive: + showArchiveNA(); + break; + case R.id.board_menu_trade_commit: cmd = JNICmd.CMD_COMMIT; break; @@ -1112,9 +1112,7 @@ public class BoardDelegate extends DelegateBase break; case ARCHIVE_ACTION: - String archiveName = (String)params[0]; - long archiveGroup = (Long)params[1]; - archiveAndClose( archiveName, archiveGroup ); + archiveAndClose(); break; case ENABLE_SMS_DO: @@ -2600,12 +2598,33 @@ public class BoardDelegate extends DelegateBase return wordsArray; } - private void archiveAndClose( String archiveName, long groupID ) + private boolean inArchiveGroup() { - if ( DBUtils.GROUPID_UNSPEC == groupID ) { - groupID = DBUtils.addGroup( m_activity, archiveName ); + String archiveName = LocUtils + .getString( m_activity, R.string.group_name_archive ); + long archiveGroup = DBUtils.getGroup( m_activity, archiveName ); + long curGroup = DBUtils.getGroupForGame( m_activity, m_rowid ); + return curGroup == archiveGroup; + } + + private void showArchiveNA() + { + makeNotAgainBuilder( R.string.not_again_archive, + R.string.key_na_archive, + Action.ARCHIVE_ACTION ) + .show(); + } + + private void archiveAndClose() + { + String archiveName = LocUtils + .getString( m_activity, R.string.group_name_archive ); + long archiveGroupID = DBUtils.getGroup( m_activity, archiveName ); + + if ( DBUtils.GROUPID_UNSPEC == archiveGroupID ) { + archiveGroupID = DBUtils.addGroup( m_activity, archiveName ); } - DBUtils.moveGame( m_activity, m_rowid, groupID ); + DBUtils.moveGame( m_activity, m_rowid, archiveGroupID ); waitCloseGame( false ); finish(); } diff --git a/xwords4/android/app/src/main/res/menu/board_menu.xml b/xwords4/android/app/src/main/res/menu/board_menu.xml index f49b85f19..e8a5a2207 100644 --- a/xwords4/android/app/src/main/res/menu/board_menu.xml +++ b/xwords4/android/app/src/main/res/menu/board_menu.xml @@ -5,6 +5,10 @@ + Date: Sat, 16 Dec 2017 08:17:15 -0800 Subject: [PATCH 136/138] catch up small-board menu --- .../src/main/res/menu-small/board_menu.xml | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/xwords4/android/app/src/main/res/menu-small/board_menu.xml b/xwords4/android/app/src/main/res/menu-small/board_menu.xml index 49835d6a7..b553c5711 100644 --- a/xwords4/android/app/src/main/res/menu-small/board_menu.xml +++ b/xwords4/android/app/src/main/res/menu-small/board_menu.xml @@ -6,6 +6,13 @@ android:title="@string/board_menu_invite" /> + + + - + - + - - - + + + From 771c9ba4a6f6e336b600ae6be38923e73db367ed Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 17 Dec 2017 11:07:43 -0800 Subject: [PATCH 137/138] add three navbar icons from The Noun Project with attribution of course. --- .../app/src/main/res/menu/board_menu.xml | 3 + .../app/src/main/res/values/strings.xml | 4 +- xwords4/android/img_src/archive.svg | 81 +++++++++++++++++++ xwords4/android/img_src/rematch.svg | 54 +++++++++++++ xwords4/android/img_src/trade.svg | 50 ++++++++++++ 5 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 xwords4/android/img_src/archive.svg create mode 100644 xwords4/android/img_src/rematch.svg create mode 100644 xwords4/android/img_src/trade.svg diff --git a/xwords4/android/app/src/main/res/menu/board_menu.xml b/xwords4/android/app/src/main/res/menu/board_menu.xml index e8a5a2207..a027b9aaf 100644 --- a/xwords4/android/app/src/main/res/menu/board_menu.xml +++ b/xwords4/android/app/src/main/res/menu/board_menu.xml @@ -8,10 +8,12 @@ diff --git a/xwords4/android/app/src/main/res/values/strings.xml b/xwords4/android/app/src/main/res/values/strings.xml index cee42c6cd..57aa7015b 100644 --- a/xwords4/android/app/src/main/res/values/strings.xml +++ b/xwords4/android/app/src/main/res/values/strings.xml @@ -1709,7 +1709,9 @@ - Toolbar icons by Sarah Chu. + Toolbar icons by Sarah Chu. Navbar + icons from the Noun Project: \"archive\" by Trendy; \"rematch\" by + Becris; and \"swap\" by iconomania. diff --git a/xwords4/android/img_src/archive.svg b/xwords4/android/img_src/archive.svg new file mode 100644 index 000000000..c12a564e8 --- /dev/null +++ b/xwords4/android/img_src/archive.svg @@ -0,0 +1,81 @@ + + + + + + image/svg+xml + + 01 + + + + + + + + 01 + + + + + diff --git a/xwords4/android/img_src/rematch.svg b/xwords4/android/img_src/rematch.svg new file mode 100644 index 000000000..01cd1dd04 --- /dev/null +++ b/xwords4/android/img_src/rematch.svg @@ -0,0 +1,54 @@ + +image/svg+xml \ No newline at end of file diff --git a/xwords4/android/img_src/trade.svg b/xwords4/android/img_src/trade.svg new file mode 100644 index 000000000..7d1752d97 --- /dev/null +++ b/xwords4/android/img_src/trade.svg @@ -0,0 +1,50 @@ + +image/svg+xml \ No newline at end of file From ec698c4c62b897bddfc77bc4317229569b9291e2 Mon Sep 17 00:00:00 2001 From: Eric House Date: Sun, 17 Dec 2017 11:17:23 -0800 Subject: [PATCH 138/138] remove empty strings (currently an error) --- xwords4/android/res_src/values-nb-rNO/strings.xml | 4 ---- xwords4/android/scripts/copy-strings.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/xwords4/android/res_src/values-nb-rNO/strings.xml b/xwords4/android/res_src/values-nb-rNO/strings.xml index 6f27c4bcc..3c6613f48 100644 --- a/xwords4/android/res_src/values-nb-rNO/strings.xml +++ b/xwords4/android/res_src/values-nb-rNO/strings.xml @@ -284,7 +284,6 @@ SMS (tekstmelding) E-post Blåtann - Invitasjon av spillere: Hvordan? "Meg: " "Ikke meg: " @@ -445,7 +444,6 @@ Nedlasting mislyktes Lagre ordlister internt - Nedlastingsmappe @@ -594,7 +592,6 @@ Skru på offentlige rom Rom andre kan se og ta del i - Skjul knapper @@ -612,7 +609,6 @@ Skru på feilrettingsfunksjoner Nettverksstatistikk Vis invitasjoner - %1$s/%2$s Skriv spill til SD-kort Last spill fra SD-kort diff --git a/xwords4/android/scripts/copy-strings.py b/xwords4/android/scripts/copy-strings.py index 2ba15fb79..8dcd73644 100755 --- a/xwords4/android/scripts/copy-strings.py +++ b/xwords4/android/scripts/copy-strings.py @@ -141,7 +141,7 @@ def writeDoc(doc, src, dest): def checkOrConvertString(engNames, elem, verbose): name = elem.get('name') if not elem.text: - print "elem", name, "is empty" + print "ERROR: elem", name, "is empty" sys.exit(1) elif not name in engNames or elem.text.startswith(s_prefix): ok = False