as first step in using mmap for dictionaries instead of passing byte

arrays into the jni, pass the full file paths in in addition to the
byte arrays.  This isn't possible with the built-in dicts, but does
work for the downloaded ones (which are usually larger).  This checkin
does the mmap and uses memcmp to verify that the bytes are the same as
passed in.  Next step is to not pass the bytes when the path will do
and to actually use the mmap'd ptr.

xwords4/android/XWords4/jni/anddict.c

@@ -21,6 +21,11 @@
 #include <jni.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/mman.h>
 
 #include "anddict.h"
 #include "xptypes.h"
@@ -424,7 +429,8 @@ and_dictionary_make_empty( MPFORMAL JNIEnv* env, JNIUtilCtxt* jniutil )
 void
 makeDicts( MPFORMAL JNIEnv *env, JNIUtilCtxt* jniutil, 
            DictionaryCtxt** dictp, PlayerDicts* dicts,
-           jobjectArray jdicts, jobjectArray jnames, jstring jlang )
+           jobjectArray jnames, jobjectArray jdicts, jobjectArray jpaths,
+           jstring jlang )
 {
     int ii;
     jsize len = (*env)->GetArrayLength( env, jdicts );
@@ -436,10 +442,16 @@ makeDicts( MPFORMAL JNIEnv *env, JNIUtilCtxt* jniutil,
             jobject jdict = (*env)->GetObjectArrayElement( env, jdicts, ii );
             if ( NULL != jdict ) { 
                 jstring jname = (*env)->GetObjectArrayElement( env, jnames, ii );
-                dict = makeDict( MPPARM(mpool) env, jniutil, jdict, jname, jlang );
+                jstring jpath = jpaths == NULL ? 
+                    NULL : (*env)->GetObjectArrayElement( env, jpaths, ii );
+                dict = makeDict( MPPARM(mpool) env, jniutil, jname, jdict, 
+                                 jpath, jlang );
                 XP_ASSERT( !!dict );
                 (*env)->DeleteLocalRef( env, jdict );
                 (*env)->DeleteLocalRef( env, jname );
+                if ( NULL != jpath) {
+                    (*env)->DeleteLocalRef( env, jpath );
+                }
             }
         }
         if ( 0 == ii ) {
@@ -452,8 +464,8 @@ makeDicts( MPFORMAL JNIEnv *env, JNIUtilCtxt* jniutil,
 }
 
 DictionaryCtxt* 
-makeDict( MPFORMAL JNIEnv *env, JNIUtilCtxt* jniutil, jbyteArray jbytes,
+makeDict( MPFORMAL JNIEnv *env, JNIUtilCtxt* jniutil, jstring jname, 
-          jstring jname, jstring jlangname )
+          jbyteArray jbytes, jstring jpath, jstring jlangname )
 {
     AndDictionaryCtxt* anddict = (AndDictionaryCtxt*)
         and_dictionary_make_empty( MPPARM(mpool) env, jniutil );
@@ -463,6 +475,30 @@ makeDict( MPFORMAL JNIEnv *env, JNIUtilCtxt* jniutil, jbyteArray jbytes,
     anddict->bytes = (*env)->GetByteArrayElements( env, anddict->byteArray, 
                                                    NULL );
 
+    if ( NULL != jpath ) {
+        const char* path = (*env)->GetStringUTFChars( env, jpath, NULL );
+
+        struct stat statbuf;
+        if ( 0 == stat( path, &statbuf ) ) {
+            int fd = open( path,  O_RDONLY );
+            if ( fd >= 0 ) {
+                void* ptr = mmap( NULL, statbuf.st_size,
+                                  PROT_READ, MAP_PRIVATE,
+                                  fd, 0 );
+                close( fd );
+                if ( MAP_FAILED != ptr ) {
+                    XP_ASSERT( 0 == memcmp( ptr, anddict->bytes, 
+                                            statbuf.st_size ) );
+                    (void)munmap( ptr, statbuf.st_size );
+                    XP_LOGF( "%s: all %lld bytes of %s checked out!!!!", 
+                             __func__, statbuf.st_size, path );
+                }
+            }
+        }
+
+        (*env)->ReleaseStringUTFChars( env, jpath, path );
+    }
+
     anddict->super.destructor = and_dictionary_destroy;
 
     parseDict( anddict, (XP_U8*)anddict->bytes, len );

xwords4/android/XWords4/jni/anddict.h

@@ -30,12 +30,12 @@ dict_splitFaces( DictionaryCtxt* dict, const XP_U8* bytes,
                  XP_U16 nBytes, XP_U16 nFaces );
 
 DictionaryCtxt* makeDict( MPFORMAL JNIEnv *env, JNIUtilCtxt* jniutil, 
-                          jbyteArray bytes, jstring jname, jstring jlang );
+                          jstring jname, jbyteArray bytes, jstring path,
+                          jstring jlang );
 
 void makeDicts( MPFORMAL JNIEnv *env, JNIUtilCtxt* jniutil, 
-                DictionaryCtxt** dict,
+                DictionaryCtxt** dict, PlayerDicts* dicts, jobjectArray jnames, 
-                PlayerDicts* dicts, jobjectArray jdicts, jobjectArray jnames, 
+                jobjectArray jdicts, jobjectArray jpaths, jstring jlang );
-                jstring jlang );
 
 void destroyDicts( PlayerDicts* dicts );
 

xwords4/android/XWords4/jni/xwjni.c

@@ -267,14 +267,15 @@ Java_org_eehouse_android_xw4_jni_XwJNI_comms_1getInitialAddr
 
 JNIEXPORT void JNICALL
 Java_org_eehouse_android_xw4_jni_XwJNI_dict_1getInfo
-(JNIEnv* env, jclass C, jbyteArray jDictBytes, jobject jniu, jobject jinfo )
+( JNIEnv* env, jclass C, jbyteArray jDictBytes, jstring jpath, 
+  jobject jniu, jobject jinfo )
 {
 #ifdef MEM_DEBUG
     MemPoolCtx* mpool = mpool_make();
 #endif
     JNIUtilCtxt* jniutil = makeJNIUtil( MPPARM(mpool) &env, jniu );
-    DictionaryCtxt* dict = makeDict( MPPARM(mpool) env, jniutil,
+    DictionaryCtxt* dict = makeDict( MPPARM(mpool) env, jniutil, NULL,
-                                     jDictBytes, NULL, NULL );
+                                     jDictBytes, jpath, NULL );
     jint code = dict_getLangCode( dict );
     jint nWords = dict_getWordCount( dict );
     dict_destroy( dict );
@@ -384,7 +385,8 @@ JNIEXPORT void JNICALL
 Java_org_eehouse_android_xw4_jni_XwJNI_game_1makeNewGame
 ( JNIEnv* env, jclass C, jint gamePtr, jobject j_gi, jobject j_util, 
   jobject jniu, jobject j_draw, jobject j_cp, jobject j_procs, 
-  jobjectArray j_dicts, jobjectArray j_names, jstring j_lang )
+  jobjectArray j_names, jobjectArray j_dicts, jobjectArray j_paths,
+  jstring j_lang )
 {
     XWJNI_START_GLOBALS();
     CurGameInfo* gi = makeGI( MPPARM(mpool) env, j_gi );
@@ -404,8 +406,8 @@ Java_org_eehouse_android_xw4_jni_XwJNI_game_1makeNewGame
 
     DictionaryCtxt* dict;
     PlayerDicts dicts;
-    makeDicts( MPPARM(mpool) env, globals->jniutil, &dict, &dicts, j_dicts, 
+    makeDicts( MPPARM(mpool) env, globals->jniutil, &dict, &dicts, j_names, 
-               j_names, j_lang );
+               j_dicts, j_paths, j_lang );
 #ifdef STUBBED_DICT
     if ( !dict ) {
         XP_LOGF( "falling back to stubbed dict" );
@@ -446,8 +448,9 @@ JNIEXPORT void JNICALL Java_org_eehouse_android_xw4_jni_XwJNI_game_1dispose
 JNIEXPORT jboolean JNICALL
 Java_org_eehouse_android_xw4_jni_XwJNI_game_1makeFromStream
 ( JNIEnv* env, jclass C, jint gamePtr, jbyteArray jstream, jobject /*out*/jgi,
-  jobjectArray jdicts, jobjectArray jdictNames, jstring jlang, 
+  jobjectArray jdictNames, jobjectArray jdicts, jobjectArray jpaths,
-  jobject jutil, jobject jniu, jobject jdraw, jobject jcp, jobject jprocs )
+  jstring jlang, jobject jutil, jobject jniu, jobject jdraw, jobject jcp,
+  jobject jprocs )
 {
     jboolean result;
     DictionaryCtxt* dict;
@@ -458,8 +461,8 @@ Java_org_eehouse_android_xw4_jni_XwJNI_game_1makeFromStream
     globals->util = makeUtil( MPPARM(mpool) &state->env, 
                               jutil, globals->gi, globals );
     globals->jniutil = makeJNIUtil( MPPARM(mpool) &state->env, jniu );
-    makeDicts( MPPARM(mpool) env, globals->jniutil, &dict, &dicts, jdicts, 
+    makeDicts( MPPARM(mpool) env, globals->jniutil, &dict, &dicts, jdictNames,
-               jdictNames, jlang );
+               jdicts, jpaths,  jlang );
     globals->dctx = makeDraw( MPPARM(mpool) &state->env, jdraw );
     globals->xportProcs = makeXportProcs( MPPARM(mpool) &state->env, jprocs );
 

xwords4/android/XWords4/src/org/eehouse/android/xw4/BoardActivity.java

@@ -1051,7 +1051,7 @@ public class BoardActivity extends XWActivity
             XwJNI.gi_from_stream( m_gi, stream );
 
             String[] dictNames = m_gi.dictNames();
-            byte[][] dictBytes = GameUtils.openDicts( this, dictNames );
+            GameUtils.DictPairs pairs = GameUtils.openDicts( this, dictNames );
             String langName = m_gi.langName();
             m_jniGamePtr = XwJNI.initJNI();
 
@@ -1063,12 +1063,13 @@ public class BoardActivity extends XWActivity
             CommonPrefs cp = CommonPrefs.get( this );
             if ( null == stream ||
                  ! XwJNI.game_makeFromStream( m_jniGamePtr, stream, 
-                                              m_gi, dictBytes, dictNames, 
+                                              m_gi, dictNames, pairs.m_bytes, 
-                                              langName, m_utils, m_jniu, 
+                                              pairs.m_paths, langName, m_utils,
-                                              m_view, cp, m_xport ) ) {
+                                              m_jniu, m_view, cp, m_xport ) ) {
                 XwJNI.game_makeNewGame( m_jniGamePtr, m_gi, m_utils, m_jniu, 
-                                        m_view, cp, m_xport, 
+                                        m_view, cp, m_xport, dictNames, 
-                                        dictBytes, dictNames, langName );
+                                        pairs.m_bytes, pairs.m_paths,
+                                        langName );
             }
 
             m_jniThread = new 

xwords4/android/XWords4/src/org/eehouse/android/xw4/DictLangCache.java

@@ -316,8 +316,9 @@ public class DictLangCache {
             info = s_nameToLang.get( name );
         } else {
             byte[] dict = GameUtils.openDict( context, name );
+            String path = GameUtils.getDictPath( context, name );
             info = new DictInfo();
-            XwJNI.dict_getInfo( dict, JNIUtilsImpl.get(), info );
+            XwJNI.dict_getInfo( dict, path, JNIUtilsImpl.get(), info );
             info.name = name;
             s_nameToLang.put( name, info );
         }

xwords4/android/XWords4/src/org/eehouse/android/xw4/GameUtils.java

@@ -152,6 +152,14 @@ public class GameUtils {
         }
     }
 
+    public static class DictPairs {
+        public byte[][] m_bytes;
+        public String[] m_paths;
+        public DictPairs( byte[][] bytes, String[] paths ) {
+            m_bytes = bytes; m_paths = paths;
+        }
+    }
+
     private static Object s_syncObj = new Object();
 
     public static byte[] savedGame( Context context, long rowid )
@@ -182,7 +190,7 @@ public class GameUtils {
         // as DeviceRole.SERVER_STANDALONE != gi.serverRole
         loadMakeGame( context, gamePtr, gi, lockSrc );
         String[] dictNames = gi.dictNames();
-        byte[][] dictBytes = openDicts( context, dictNames );
+        DictPairs pairs = openDicts( context, dictNames );
         
         if ( XwJNI.game_hasComms( gamePtr ) ) {
             addr = new CommsAddrRec( context );
@@ -197,8 +205,8 @@ public class GameUtils {
 
         gamePtr = XwJNI.initJNI();
         XwJNI.game_makeNewGame( gamePtr, gi, JNIUtilsImpl.get(), 
-                                CommonPrefs.get( context ), dictBytes, 
+                                CommonPrefs.get( context ), dictNames,
-                                dictNames, gi.langName() );
+                                pairs.m_bytes,  pairs.m_paths, gi.langName() );
                                 
         if ( null != addr ) {
             XwJNI.comms_setAddr( gamePtr, addr );
@@ -310,18 +318,19 @@ public class GameUtils {
         byte[] stream = savedGame( context, lock );
         XwJNI.gi_from_stream( gi, stream );
         String[] dictNames = gi.dictNames();
-        byte[][] dictBytes = openDicts( context, dictNames );
+        DictPairs pairs = openDicts( context, dictNames );
         String langName = gi.langName();
 
         boolean madeGame = XwJNI.game_makeFromStream( gamePtr, stream, 
                                                       JNIUtilsImpl.get(), gi, 
-                                                      dictBytes, dictNames,
+                                                      dictNames, pairs.m_bytes, 
-                                                      langName, util, 
+                                                      pairs.m_paths, langName,
+                                                      util, 
                                                       CommonPrefs.get(context));
         if ( !madeGame ) {
             XwJNI.game_makeNewGame( gamePtr, gi, JNIUtilsImpl.get(), 
-                                    CommonPrefs.get(context), dictBytes, 
+                                    CommonPrefs.get(context), dictNames,
-                                    dictNames, langName );
+                                    pairs.m_bytes, pairs.m_paths, langName );
         }
     }
 
@@ -702,12 +711,30 @@ public class GameUtils {
         return bytes;
     }
 
-    public static byte[][] openDicts( Context context, String[] names )
+    public static String getDictPath( Context context, String name )
     {
-        byte[][] result = new byte[names.length][];
+        name = addDictExtn( name );
+
+        File file = context.getFileStreamPath( name );
+        if ( !file.exists() ) {
+            file = getSDPathFor( context, name );
+            if ( !file.exists() ) {
+                file = null;
+            }
+        }
+        String path = null == file? null : file.getPath();
+        return path;
+    }
+
+    public static DictPairs openDicts( Context context, String[] names )
+    {
+        byte[][] dictBytes = new byte[names.length][];
+        String[] dictPaths = new String[names.length];
+
         HashMap<String,byte[]> seen = new HashMap<String,byte[]>();
         for ( int ii = 0; ii < names.length; ++ii ) {
             byte[] bytes = null;
+            String path = null;
             String name = names[ii];
             if ( null != name ) {
                 bytes = seen.get( name );
@@ -715,10 +742,12 @@ public class GameUtils {
                     bytes = openDict( context, name );
                     seen.put( name, bytes );
                 }
+                path = getDictPath( context, name );
             }
-            result[ii] = bytes;
+            dictBytes[ii] = bytes;
+            dictPaths[ii] = path;
         }
-        return result;
+        return new DictPairs( dictBytes, dictPaths );
     }
 
     public static boolean saveDict( Context context, InputStream in,
@@ -877,12 +906,12 @@ public class GameUtils {
         gi.replaceDicts( newDict );
 
         String[] dictNames = gi.dictNames();
-        byte[][] dictBytes = openDicts( context, dictNames );
+        DictPairs pairs = openDicts( context, dictNames );
         
         int gamePtr = XwJNI.initJNI();
         XwJNI.game_makeFromStream( gamePtr, stream, JNIUtilsImpl.get(), gi,
-                                   dictBytes, dictNames, gi.langName(), 
+                                   dictNames, pairs.m_bytes, pairs.m_paths,
-                                   CommonPrefs.get( context ) );
+                                   gi.langName(), CommonPrefs.get( context ) );
         // second time required as game_makeFromStream can overwrite
         gi.replaceDicts( newDict );
 
@@ -902,7 +931,7 @@ public class GameUtils {
         // that don't reset the game, e.g. player name for standalone
         // games?
         String[] dictNames = gi.dictNames();
-        byte[][] dictBytes = openDicts( context, dictNames );
+        DictPairs pairs = openDicts( context, dictNames );
         String langName = gi.langName();
         int gamePtr = XwJNI.initJNI();
         boolean madeGame = false;
@@ -916,13 +945,15 @@ public class GameUtils {
             madeGame = XwJNI.game_makeFromStream( gamePtr, stream, 
                                                   JNIUtilsImpl.get(),
                                                   new CurGameInfo(context), 
-                                                  dictBytes, dictNames, 
+                                                  dictNames, pairs.m_bytes,
+                                                  pairs.m_paths,
                                                   langName, cp );
         }
 
         if ( forceNew || !madeGame ) {
             XwJNI.game_makeNewGame( gamePtr, gi, JNIUtilsImpl.get(), 
-                                    cp, dictBytes, dictNames, langName );
+                                    cp, dictNames, pairs.m_bytes, 
+                                    pairs.m_paths, langName );
         }
 
         if ( null != car ) {

xwords4/android/XWords4/src/org/eehouse/android/xw4/jni/XwJNI.java

@@ -59,15 +59,17 @@ public class XwJNI {
                                                 JNIUtils jniu,
                                                 DrawCtx draw, CommonPrefs cp, 
                                                 TransportProcs procs, 
-                                                byte[][] dicts, 
                                                 String[] dictNames,
+                                                byte[][] dictBytes, 
+                                                String[] dictPaths, 
                                                 String langName );
 
     public static native boolean game_makeFromStream( int gamePtr,
                                                       byte[] stream, 
                                                       CurGameInfo gi, 
-                                                      byte[][] dicts, 
                                                       String[] dictNames,
+                                                      byte[][] dictBytes, 
+                                                      String[] dictPaths, 
                                                       String langName,
                                                       UtilCtxt util, 
                                                       JNIUtils jniu,
@@ -79,23 +81,24 @@ public class XwJNI {
     // played
     public static void game_makeNewGame( int gamePtr, CurGameInfo gi,
                                          JNIUtils jniu, CommonPrefs cp, 
-                                         byte[][] dicts, String[] dictNames,
+                                         String[] dictNames, byte[][] dictBytes, 
-                                         String langName ) {
+                                         String[] dictPaths, String langName ) {
         game_makeNewGame( gamePtr, gi, (UtilCtxt)null, jniu,
                           (DrawCtx)null, cp, (TransportProcs)null, 
-                          dicts, dictNames, langName );
+                          dictNames, dictBytes, dictPaths, langName );
     }
 
     public static boolean game_makeFromStream( int gamePtr,
                                                byte[] stream, 
                                                JNIUtils jniu,
                                                CurGameInfo gi, 
-                                               byte[][] dicts, 
                                                String[] dictNames,
+                                               byte[][] dictBytes, 
+                                               String[] dictPaths,
                                                String langName,
                                                CommonPrefs cp ) {
-        return game_makeFromStream( gamePtr, stream, gi, dicts, dictNames,
+        return game_makeFromStream( gamePtr, stream, gi, dictNames, dictBytes,
-                                    langName, (UtilCtxt)null, jniu,
+                                    dictPaths, langName, (UtilCtxt)null, jniu,
                                     (DrawCtx)null, cp, (TransportProcs)null );
     }
 
@@ -103,14 +106,15 @@ public class XwJNI {
                                                byte[] stream, 
                                                JNIUtils jniu,
                                                CurGameInfo gi, 
-                                               byte[][] dicts, 
                                                String[] dictNames,
+                                               byte[][] dictBytes, 
+                                               String[] dictPaths,
                                                String langName,
                                                UtilCtxt util,
                                                CommonPrefs cp ) {
-        return game_makeFromStream( gamePtr, stream, gi, dicts, dictNames,
+        return game_makeFromStream( gamePtr, stream, gi, dictNames, dictBytes,
-                                    langName, util, jniu, (DrawCtx)null, cp, 
+                                    dictPaths, langName, util, jniu, 
-                                    (TransportProcs)null );
+                                    (DrawCtx)null, cp, (TransportProcs)null );
     }
 
     public static native boolean game_receiveMessage( int gamePtr, 
@@ -226,7 +230,7 @@ public class XwJNI {
     // Dicts
     public static native boolean dict_tilesAreSame( int dictPtr1, int dictPtr2 );
     public static native String[] dict_getChars( int dictPtr );
-    public static native void dict_getInfo( byte[] dict, JNIUtils jniu, 
+    public static native void dict_getInfo( byte[] dict, String path, 
-                                            DictInfo info );
+                                            JNIUtils jniu, DictInfo info );
     public static native int dict_getTileValue( int dictPtr, int tile );
 }