From 07f4774a7cbbaab02bedb0a5ce363744365a09a4 Mon Sep 17 00:00:00 2001 From: Eric House Date: Fri, 1 May 2020 09:18:27 -0700 Subject: [PATCH] refactor so common dictnry does more work Linux and Android duplicated all the code to parse a wordlist file -- and shared a bug that needed fixing. So now most of that is in a common/ function both call, and the bug -- failing to mask out flag bits I don't care about -- is fixed. --- xwords4/android/jni/anddict.c | 314 ++------------------------------- xwords4/android/jni/andutils.c | 5 +- xwords4/common/dictnry.c | 263 +++++++++++++++++++++++++++ xwords4/common/dictnry.h | 15 +- xwords4/linux/Makefile | 1 + xwords4/linux/linuxdict.c | 254 ++------------------------ 6 files changed, 307 insertions(+), 545 deletions(-) diff --git a/xwords4/android/jni/anddict.c b/xwords4/android/jni/anddict.c index 521aadc6c..7a43e680b 100644 --- a/xwords4/android/jni/anddict.c +++ b/xwords4/android/jni/anddict.c @@ -49,12 +49,6 @@ typedef struct _AndDictionaryCtxt { #endif } AndDictionaryCtxt; -#define CHECK_PTR(p,c,e) \ - if ( ((p)+(c)) > (e) ) { \ - XP_LOGF( "%s (line %d); out of bytes", __func__, __LINE__ ); \ - goto error; \ - } - static void splitFaces_via_java( JNIEnv* env, AndDictionaryCtxt* ctxt, const XP_U8* ptr, int nFaceBytes, int nFaces, XP_Bool isUTF8 ); @@ -80,152 +74,6 @@ n_ptr_tohl( XP_U8 const** inp ) return XP_NTOHL(t); } /* n_ptr_tohl */ -static XP_U16 -n_ptr_tohs( XP_U8 const ** inp ) -{ - XP_U16 t; - XP_MEMCPY( &t, *inp, sizeof(t) ); - - *inp += sizeof(t); - - return XP_NTOHS(t); -} /* n_ptr_tohs */ - -static XP_U16 -andCountSpecials( AndDictionaryCtxt* ctxt ) -{ - XP_U16 result = 0; - - for ( int ii = 0; ii < ctxt->super.nFaces; ++ii ) { - if ( IS_SPECIAL( ctxt->super.facePtrs[ii][0] ) ) { - ++result; - } - } - - return result; -} /* andCountSpecials */ - -static XP_Bool -andMakeBitmap( AndDictionaryCtxt* ctxt, XP_U8 const** ptrp, - const XP_U8* end, XP_Bitmap* result ) -{ - XP_Bool success = XP_TRUE; - XP_U8 const* ptr = *ptrp; - jobject bitmap = NULL; - CHECK_PTR( ptr, 1, end ); - XP_U8 nCols = *ptr++; - if ( nCols > 0 ) { - CHECK_PTR( ptr, 1, end ); - XP_U8 nRows = *ptr++; - CHECK_PTR( ptr, ((nRows*nCols)+7) / 8, end ); -#ifdef DROP_BITMAPS - ptr += ((nRows*nCols)+7) / 8; -#else - XP_U8 srcByte = 0; - XP_U8 nBits; - - jboolean* colors = (jboolean*)XP_CALLOC( ctxt->super.mpool, - nCols * nRows * sizeof(*colors) ); - jboolean* next = colors; - - nBits = nRows * nCols; - for ( int ii = 0; ii < nBits; ++ii ) { - XP_U8 srcBitIndex = ii % 8; - XP_U8 srcMask; - - if ( srcBitIndex == 0 ) { - srcByte = *ptr++; - } - - srcMask = 1 << (7 - srcBitIndex); - XP_ASSERT( next < (colors + (nRows * nCols)) ); - *next++ = ((srcByte & srcMask) == 0) ? JNI_FALSE : JNI_TRUE; - } - - JNIEnv* env = ctxt->env; - jobject tmp = and_util_makeJBitmap( ctxt->jniutil, nCols, nRows, colors ); - bitmap = (*env)->NewGlobalRef( env, tmp ); - deleteLocalRef( env, tmp ); - XP_FREE( ctxt->super.mpool, colors ); -#endif - } - goto done; - error: - success = XP_FALSE; - done: - *ptrp = ptr; - *result = bitmap; - return success; -} /* andMakeBitmap */ - -static XP_Bool -andLoadSpecialData( AndDictionaryCtxt* ctxt, XP_U8 const** ptrp, - const XP_U8* end ) -{ - XP_Bool success = XP_TRUE; - XP_U16 nSpecials = andCountSpecials( ctxt ); - XP_U8 const* ptr = *ptrp; - XP_UCHAR** texts; - XP_UCHAR** textEnds; - SpecialBitmaps* bitmaps; - - texts = (XP_UCHAR**)XP_MALLOC( ctxt->super.mpool, - nSpecials * sizeof(*texts) ); - textEnds = (XP_UCHAR**)XP_MALLOC( ctxt->super.mpool, - nSpecials * sizeof(*textEnds) ); - - bitmaps = (SpecialBitmaps*) - XP_CALLOC( ctxt->super.mpool, nSpecials * sizeof(*bitmaps) ); - - for ( Tile ii = 0; ii < ctxt->super.nFaces; ++ii ) { - - const XP_UCHAR* facep = ctxt->super.facePtrs[(short)ii]; - if ( IS_SPECIAL(*facep) ) { - /* get the string */ - CHECK_PTR( ptr, 1, end ); - XP_U8 txtlen = *ptr++; - CHECK_PTR( ptr, txtlen, end ); - XP_UCHAR* text = (XP_UCHAR*)XP_MALLOC(ctxt->super.mpool, txtlen+1); - texts[(int)*facep] = text; - textEnds[(int)*facep] = text + txtlen + 1; - XP_MEMCPY( text, ptr, txtlen ); - ptr += txtlen; - text[txtlen] = '\0'; - XP_ASSERT( *facep < nSpecials ); /* firing */ - - /* This little hack is safe because all bytes but the first in a - multi-byte utf-8 char have the high bit set. SYNONYM_DELIM - does not have its high bit set */ - XP_ASSERT( 0 == (SYNONYM_DELIM & 0x80) ); - for ( ; '\0' != *text; ++text ) { - if ( *text == SYNONYM_DELIM ) { - *text = '\0'; - } - } - - if ( !andMakeBitmap( ctxt, &ptr, end, - &bitmaps[(int)*facep].largeBM ) ) { - goto error; - } - if ( !andMakeBitmap( ctxt, &ptr, end, - &bitmaps[(int)*facep].smallBM ) ) { - goto error; - } - } - } - - goto done; - error: - success = XP_FALSE; - done: - ctxt->super.chars = texts; - ctxt->super.charEnds = textEnds; - ctxt->super.bitmaps = bitmaps; - - *ptrp = ptr; - return success; -} /* andLoadSpecialData */ - /** Android doesn't include iconv for C code to use, so we'll have java do it. * Cons up a string with all the tile faces (skipping the specials to make * things easier) and have java return an array of strings. Then load one at @@ -290,16 +138,18 @@ splitFaces_via_java( JNIEnv* env, AndDictionaryCtxt* ctxt, const XP_U8* ptr, ctxt->super.facePtrs = ptrs; } /* splitFaces_via_java */ -static XP_UCHAR* -getNullTermParam( AndDictionaryCtxt* dctx, const XP_U8** ptr, - XP_U16* headerLen ) +void +computeChecksum( DictionaryCtxt* dctx, XWEnv xwe, const XP_U8* ptr, + XP_U32 len, XP_UCHAR* out ) { - XP_U16 len = 1 + XP_STRLEN( (XP_UCHAR*)*ptr ); - XP_UCHAR* result = XP_MALLOC( dctx->super.mpool, len ); - XP_MEMCPY( result, *ptr, len ); - *ptr += len; - *headerLen -= len; - return result; + AndDictionaryCtxt* ctxt = (AndDictionaryCtxt*)dctx; + JNIEnv* env = xwe; + jstring jsum = and_util_getMD5SumForDict( ctxt->jniutil, env, + ctxt->super.name, ptr, len ); + const char* sum = (*env)->GetStringUTFChars( env, jsum, NULL ); + XP_MEMCPY( out, sum, 1 + XP_STRLEN(sum) ); + (*env)->ReleaseStringUTFChars( env, jsum, sum ); + deleteLocalRef( env, jsum ); } static XP_Bool @@ -311,149 +161,15 @@ parseDict( AndDictionaryCtxt* ctxt, XWEnv xwe, XP_U8 const* ptr, ASSERT_ENV( ctxt->ti, xwe ); const XP_U8* end = ptr + dictLength; XP_U32 offset; - XP_U16 nFaces, numFaceBytes = 0; - XP_U16 flags; void* mappedBase = (void*)ptr; - XP_U8 nodeSize; - XP_Bool isUTF8 = XP_FALSE; - CHECK_PTR( ptr, sizeof(flags), end ); - flags = n_ptr_tohs( &ptr ); - if ( 0 != (DICT_HEADER_MASK & flags) ) { - XP_U16 headerLen; - flags &= ~DICT_HEADER_MASK; - CHECK_PTR( ptr, sizeof(headerLen), end ); - headerLen = n_ptr_tohs( &ptr ); - if ( 4 <= headerLen ) { /* have word count? */ - CHECK_PTR( ptr, sizeof(ctxt->super.nWords), end ); - ctxt->super.nWords = n_ptr_tohl( &ptr ); - headerLen -= 4; /* don't skip it */ - } - - if ( 1 <= headerLen ) { /* have description? */ - ctxt->super.desc = getNullTermParam( ctxt, &ptr, &headerLen ); - } - if ( 1 <= headerLen ) { /* have md5sum? */ - ctxt->super.md5Sum = getNullTermParam( ctxt, &ptr, &headerLen ); - } - - CHECK_PTR( ptr, headerLen, end ); - ptr += headerLen; - } - - flags &= ~DICT_SYNONYMS_MASK; - if ( flags == 0x0002 ) { - nodeSize = 3; - } else if ( flags == 0x0003 ) { - nodeSize = 4; - } else if ( flags == 0x0004 ) { - isUTF8 = XP_TRUE; - nodeSize = 3; - } else if ( flags == 0x0005 ) { - isUTF8 = XP_TRUE; - nodeSize = 4; - } else { - goto error; - } - - if ( isUTF8 ) { - CHECK_PTR( ptr, 1, end ); - numFaceBytes = (XP_U16)(*ptr++); - } - CHECK_PTR( ptr, 1, end ); - nFaces = (XP_U16)(*ptr++); - if ( nFaces > 64 ) { - goto error; - } - - JNIEnv* env = xwe; - if ( NULL == ctxt->super.md5Sum -#ifdef DEBUG - || XP_TRUE -#endif - ) { - jstring jsum = and_util_getMD5SumForDict( ctxt->jniutil, env, - ctxt->super.name, NULL, 0 ); - XP_UCHAR* md5Sum = NULL; - /* If we have a cached sum, check that it's correct. */ - if ( NULL != jsum && NULL != ctxt->super.md5Sum ) { - md5Sum = getStringCopy( MPPARM(ctxt->super.mpool) env, jsum ); - if ( 0 != XP_STRCMP( ctxt->super.md5Sum, md5Sum ) ) { - deleteLocalRef( env, jsum ); - jsum = NULL; - XP_FREE( ctxt->super.mpool, md5Sum ); - md5Sum = NULL; - } - } - - if ( NULL == jsum ) { - jsum = and_util_getMD5SumForDict( ctxt->jniutil, env, - ctxt->super.name, ptr, end - ptr ); - } - if ( NULL == md5Sum ) { - md5Sum = getStringCopy( MPPARM(ctxt->super.mpool) env, jsum ); - } - deleteLocalRef( env, jsum ); - - if ( NULL == ctxt->super.md5Sum ) { - ctxt->super.md5Sum = md5Sum; - } else { - XP_FREE( ctxt->super.mpool, md5Sum ); - } - } - - ctxt->super.nodeSize = nodeSize; - - if ( !isUTF8 ) { - numFaceBytes = nFaces * 2; - } - - ctxt->super.nFaces = (XP_U8)nFaces; - ctxt->super.isUTF8 = isUTF8; - - if ( isUTF8 ) { - CHECK_PTR( ptr, numFaceBytes, end ); - splitFaces_via_java( env, ctxt, ptr, numFaceBytes, nFaces, - XP_TRUE ); - ptr += numFaceBytes; - } else { - XP_U8 tmp[nFaces*4]; /* should be enough... */ - XP_U16 nBytes = 0; - /* Need to translate from iso-8859-n to utf8 */ - CHECK_PTR( ptr, 2 * nFaces, end ); - for ( int ii = 0; ii < nFaces; ++ii ) { - XP_UCHAR ch = ptr[1]; - - ptr += 2; - - tmp[nBytes] = ch; - nBytes += 1; - } - XP_ASSERT( nFaces == nBytes ); - splitFaces_via_java( env, ctxt, tmp, nBytes, nFaces, XP_FALSE ); - } - - ctxt->super.is_4_byte = (ctxt->super.nodeSize == 4); - - ctxt->super.countsAndValues = - (XP_U8*)XP_MALLOC(ctxt->super.mpool, nFaces*2); - - CHECK_PTR( ptr, 2, end ); - ctxt->super.langCode = ptr[0] & 0x7F; - ptr += 2; /* skip xloc header */ - CHECK_PTR( ptr, 2 * nFaces, end ); - for ( int ii = 0; ii < nFaces*2; ii += 2 ) { - ctxt->super.countsAndValues[ii] = *ptr++; - ctxt->super.countsAndValues[ii+1] = *ptr++; - } - - if ( !andLoadSpecialData( ctxt, &ptr, end ) ) { + if ( !parseCommon( &ctxt->super, xwe, &ptr, end ) ) { goto error; } dictLength -= ptr - (XP_U8*)mappedBase; if ( dictLength >= sizeof(offset) ) { - CHECK_PTR( ptr, sizeof(offset), end ); + CHECK_PTR( ptr, sizeof(offset), end, error ); offset = n_ptr_tohl( &ptr ); dictLength -= sizeof(offset); XP_ASSERT( dictLength % ctxt->super.nodeSize == 0 ); @@ -470,6 +186,8 @@ parseDict( AndDictionaryCtxt* ctxt, XWEnv xwe, XP_U8 const* ptr, ctxt->super.topEdge = ctxt->super.base + (offset * ctxt->super.nodeSize); } else { + XP_ASSERT( !ctxt->super.topEdge ); + XP_ASSERT( !ctxt->super.base ); ctxt->super.topEdge = (array_edge*)NULL; ctxt->super.base = (array_edge*)NULL; } @@ -489,7 +207,7 @@ and_dictionary_destroy( DictionaryCtxt* dict, XWEnv xwe ) AndDictionaryCtxt* ctxt = (AndDictionaryCtxt*)dict; ASSERT_ENV( ctxt->ti, xwe ); XP_LOGF( "%s(dict=%p); code=%x", __func__, ctxt, ctxt->dbgid ); - XP_U16 nSpecials = andCountSpecials( ctxt ); + XP_U16 nSpecials = countSpecials( &ctxt->super ); JNIEnv* env = xwe; if ( !!ctxt->super.chars ) { diff --git a/xwords4/android/jni/andutils.c b/xwords4/android/jni/andutils.c index 272787f9d..c04e3a0bf 100644 --- a/xwords4/android/jni/andutils.c +++ b/xwords4/android/jni/andutils.c @@ -26,6 +26,7 @@ #include "comtypes.h" #include "xwstream.h" +#include "strutils.h" void and_assert( const char* test, int line, const char* file, const char* func ) @@ -251,10 +252,8 @@ getStringCopy( MPFORMAL JNIEnv* env, jstring jstr ) { XP_UCHAR* result = NULL; if ( NULL != jstr ) { - jsize len = 1 + (*env)->GetStringUTFLength( env, jstr ); const char* chars = (*env)->GetStringUTFChars( env, jstr, NULL ); - result = XP_MALLOC( mpool, len ); - XP_MEMCPY( result, chars, len ); + result = copyString( mpool, chars ); (*env)->ReleaseStringUTFChars( env, jstr, chars ); } return result; diff --git a/xwords4/common/dictnry.c b/xwords4/common/dictnry.c index dd41ff19d..516dd2173 100644 --- a/xwords4/common/dictnry.c +++ b/xwords4/common/dictnry.c @@ -30,6 +30,7 @@ #include "strutils.h" #include "dictiter.h" #include "game.h" +#include "dbgutil.h" #ifdef CPLUS extern "C" { @@ -39,6 +40,8 @@ extern "C" { * ****************************************************************************/ +static XP_Bool makeBitmap( XP_U8 const** ptrp, const XP_U8* end ); + DictionaryCtxt* p_dict_ref( DictionaryCtxt* dict, XWEnv XP_UNUSED(xwe) #ifdef DEBUG_REF @@ -90,6 +93,266 @@ dict_unref_all( PlayerDicts* pd, XWEnv xwe ) } } +static XP_UCHAR* +getNullTermParam( DictionaryCtxt* XP_UNUSED_DBG(dctx), const XP_U8** ptr, + XP_U16* headerLen ) +{ + XP_U16 len = 1 + XP_STRLEN( (XP_UCHAR*)*ptr ); + XP_UCHAR* result = XP_MALLOC( dctx->mpool, len ); + XP_MEMCPY( result, *ptr, len ); + *ptr += len; + *headerLen -= len; + return result; +} + +static XP_Bool +loadSpecialData( DictionaryCtxt* ctxt, XP_U8 const** ptrp, + const XP_U8* end ) +{ + LOG_FUNC(); + XP_Bool success = XP_TRUE; + XP_U16 nSpecials = countSpecials( ctxt ); + XP_U8 const* ptr = *ptrp; + XP_UCHAR** texts; + XP_UCHAR** textEnds; + SpecialBitmaps* bitmaps; + + texts = (XP_UCHAR**)XP_MALLOC( ctxt->mpool, + nSpecials * sizeof(*texts) ); + textEnds = (XP_UCHAR**)XP_MALLOC( ctxt->mpool, + nSpecials * sizeof(*textEnds) ); + + bitmaps = (SpecialBitmaps*) + XP_CALLOC( ctxt->mpool, nSpecials * sizeof(*bitmaps) ); + + for ( Tile ii = 0; ii < ctxt->nFaces; ++ii ) { + const XP_UCHAR* facep = ctxt->facePtrs[(short)ii]; + if ( IS_SPECIAL(*facep) ) { + /* get the string */ + CHECK_PTR( ptr, 1, end, error ); + XP_U8 txtlen = *ptr++; + CHECK_PTR( ptr, txtlen, end, error ); + XP_UCHAR* text = (XP_UCHAR*)XP_MALLOC(ctxt->mpool, txtlen+1); + texts[(int)*facep] = text; + textEnds[(int)*facep] = text + txtlen + 1; + XP_MEMCPY( text, ptr, txtlen ); + ptr += txtlen; + text[txtlen] = '\0'; + XP_ASSERT( *facep < nSpecials ); /* firing */ + + /* This little hack is safe because all bytes but the first in a + multi-byte utf-8 char have the high bit set. SYNONYM_DELIM + does not have its high bit set */ + XP_ASSERT( 0 == (SYNONYM_DELIM & 0x80) ); + for ( ; '\0' != *text; ++text ) { + if ( *text == SYNONYM_DELIM ) { + *text = '\0'; + } + } + + if ( !makeBitmap( &ptr, end ) ) { + goto error; + } + if ( !makeBitmap( &ptr, end ) ) { + goto error; + } + } + } + + goto done; + error: + success = XP_FALSE; + done: + ctxt->chars = texts; + ctxt->charEnds = textEnds; + ctxt->bitmaps = bitmaps; + + *ptrp = ptr; + return success; +} /* loadSpecialData */ + +XP_Bool +parseCommon( DictionaryCtxt* dctx, XWEnv xwe, const XP_U8** ptrp, const XP_U8* end ) +{ + const XP_U8* ptr = *ptrp; + XP_Bool hasHeader = XP_FALSE; + XP_Bool isUTF8 = XP_FALSE; + XP_U16 charSize; + + XP_U16 flags; + XP_Bool formatOk = sizeof(flags) <= end - ptr; + if ( formatOk ) { + XP_MEMCPY( &flags, ptr, sizeof(flags) ); + ptr += sizeof( flags ); + flags = XP_NTOHS(flags); + + XP_LOGFF( "flags=0X%X", flags ); + hasHeader = 0 != (DICT_HEADER_MASK & flags); + /* if ( hasHeader ) { */ + /* flags &= ~DICT_HEADER_MASK; */ + /* } */ + + XP_U8 nodeSize = 4; + switch ( flags & 0x0007 ) { + case 0x0001: + nodeSize = 3; + charSize = 1; + dctx->is_4_byte = XP_FALSE; + break; + case 0x0002: + nodeSize = 3; + charSize = 2; + dctx->is_4_byte = XP_FALSE; + break; + case 0x0003: + charSize = 2; + dctx->is_4_byte = XP_TRUE; + break; + case 0x0004: + nodeSize = 3; + isUTF8 = XP_TRUE; + dctx->is_4_byte = XP_FALSE; + break; + case 0x0005: + isUTF8 = XP_TRUE; + dctx->is_4_byte = XP_TRUE; + break; + default: + formatOk = XP_FALSE; + break; + } + dctx->isUTF8 = isUTF8; + dctx->nodeSize = nodeSize; + } + + if ( formatOk ) { + XP_U8 numFaceBytes, numFaces; + + if ( hasHeader ) { + XP_U16 headerLen; + XP_U32 wordCount; + + memcpy( &headerLen, ptr, sizeof(headerLen) ); + ptr += sizeof(headerLen); + headerLen = XP_NTOHS( headerLen ); + + memcpy( &wordCount, ptr, sizeof(wordCount) ); + ptr += sizeof(wordCount); + headerLen -= sizeof(wordCount); + dctx->nWords = XP_NTOHL( wordCount ); + XP_DEBUGF( "dict contains %d words", dctx->nWords ); + + if ( 0 < headerLen ) { + dctx->desc = getNullTermParam( dctx, &ptr, &headerLen ); + } else { + XP_LOGF( "%s: no note", __func__ ); + } + if ( 0 < headerLen ) { + dctx->md5Sum = getNullTermParam( dctx, &ptr, &headerLen ); + } else { + XP_LOGF( "%s: no md5Sum", __func__ ); + } + ptr += headerLen; + } + + if ( isUTF8 ) { + numFaceBytes = *ptr++; + } + numFaces = *ptr++; + if ( !isUTF8 ) { + numFaceBytes = numFaces * charSize; + } + + if ( NULL == dctx->md5Sum +#ifdef DEBUG + || XP_TRUE +#endif + ) { + XP_UCHAR checksum[256]; + // XP_LOGFF( "figuring checksum with len: %uz", end - ptr ); + computeChecksum( dctx, xwe, ptr, end - ptr, checksum ); + if ( NULL == dctx->md5Sum ) { + dctx->md5Sum = copyString( dctx->mpool, checksum ); + } else { + XP_ASSERT( 0 == XP_STRCMP( dctx->md5Sum, checksum ) ); + } + } + + dctx->nFaces = numFaces; + + dctx->countsAndValues = XP_MALLOC( dctx->mpool, numFaces * 2 ); + XP_U16 facesSize = numFaceBytes; + if ( !isUTF8 ) { + facesSize /= 2; + } + + XP_U8 tmp[numFaceBytes]; + XP_MEMCPY( tmp, ptr, numFaceBytes ); + ptr += numFaceBytes; + + dict_splitFaces( dctx, xwe, tmp, numFaceBytes, numFaces ); + + unsigned short xloc; + XP_MEMCPY( &xloc, ptr, sizeof(xloc) ); + ptr += sizeof(xloc); + XP_MEMCPY( dctx->countsAndValues, ptr, numFaces*2 ); + ptr += numFaces*2; + + dctx->langCode = xloc & 0x7F; + } + + if ( formatOk ) { + formatOk = loadSpecialData( dctx, &ptr, end ); + } + + if ( formatOk ) { + XP_ASSERT( ptr < end ); + *ptrp = ptr; + } + + LOG_RETURNF( "%s", boolToStr(formatOk) ); + return formatOk; +} + +static XP_Bool +makeBitmap( XP_U8 const** ptrp, const XP_U8* end ) +{ + XP_Bool success = XP_TRUE; + XP_U8 const* ptr = *ptrp; + CHECK_PTR( ptr, 1, end, error ); + XP_U8 nCols = *ptr++; + if ( nCols > 0 ) { + CHECK_PTR( ptr, 1, end, error ); + XP_U8 nRows = *ptr++; + CHECK_PTR( ptr, ((nRows*nCols)+7) / 8, end, error ); +#ifdef DROP_BITMAPS + ptr += ((nRows*nCols)+7) / 8; +#else + do not compile +#endif + } + goto done; + error: + success = XP_FALSE; + done: + *ptrp = ptr; + return success; +} + +XP_U16 +countSpecials( DictionaryCtxt* ctxt ) +{ + XP_U16 result = 0; + + for ( int ii = 0; ii < ctxt->nFaces; ++ii ) { + if ( IS_SPECIAL( ctxt->facePtrs[ii][0] ) ) { + ++result; + } + } + + return result; +} /* countSpecials */ + void setBlankTile( DictionaryCtxt* dict ) { diff --git a/xwords4/common/dictnry.h b/xwords4/common/dictnry.h index 87a088a89..4aa2a33f8 100644 --- a/xwords4/common/dictnry.h +++ b/xwords4/common/dictnry.h @@ -215,14 +215,25 @@ XP_U16 dict_getMaxWidth( const DictionaryCtxt* dict ); #ifdef STUBBED_DICT DictionaryCtxt* make_stubbed_dict( MPFORMAL_NOCOMMA ); #endif +XP_U16 countSpecials( DictionaryCtxt* ctxt ); +XP_Bool parseCommon( DictionaryCtxt* dict, XWEnv xwe, const XP_U8** ptrp, + const XP_U8* end ); +XP_Bool checkSanity( DictionaryCtxt* dict, XP_U32 numEdges ); /* To be called only by subclasses!!! */ void dict_super_init( DictionaryCtxt* ctxt ); -/* Must be implemented by subclass */ +/* Must be implemented by subclasses */ void dict_splitFaces( DictionaryCtxt* dict, XWEnv xwe, const XP_U8* bytes, XP_U16 nBytes, XP_U16 nFaceos ); +void computeChecksum( DictionaryCtxt* dctxt, XWEnv xwe, const XP_U8* ptr, + XP_U32 len, XP_UCHAR* out ); -XP_Bool checkSanity( DictionaryCtxt* dict, XP_U32 numEdges ); +/* Utility used only by dict-parsing code for now */ +#define CHECK_PTR(p,c,e,lab) \ + if ( ((p)+(c)) > (e) ) { \ + XP_LOGF( "%s (line %d); out of bytes", __func__, __LINE__ ); \ + goto lab; \ + } #ifdef CPLUS } diff --git a/xwords4/linux/Makefile b/xwords4/linux/Makefile index 9330bc0a8..31e9b4ee7 100644 --- a/xwords4/linux/Makefile +++ b/xwords4/linux/Makefile @@ -260,6 +260,7 @@ CFLAGS += `pkg-config --cflags glib-2.0` LIBS += `pkg-config --libs glib-2.0` CFLAGS += $(POINTER_SUPPORT) +CFLAGS += -DDROP_BITMAPS ifneq (,$(findstring DPLATFORM_NCURSES,$(DEFINES))) LIBS += $(OE_LIBDIR) -lncursesw diff --git a/xwords4/linux/linuxdict.c b/xwords4/linux/linuxdict.c index e477cf0da..fcf477140 100644 --- a/xwords4/linux/linuxdict.c +++ b/xwords4/linux/linuxdict.c @@ -98,122 +98,15 @@ linux_dictionary_make( MPFORMAL XWEnv xwe, const LaunchParams* params, return &result->super; } /* gtk_dictionary_make */ -static XP_UCHAR* -getNullTermParam( LinuxDictionaryCtxt* XP_UNUSED_DBG(dctx), const XP_U8** ptr, - XP_U16* headerLen ) +void +computeChecksum( DictionaryCtxt* XP_UNUSED(dctx), XWEnv XP_UNUSED(xwe), + const XP_U8* ptr, XP_U32 len, XP_UCHAR* out ) { - XP_U16 len = 1 + XP_STRLEN( (XP_UCHAR*)*ptr ); - XP_UCHAR* result = XP_MALLOC( dctx->super.mpool, len ); - XP_MEMCPY( result, *ptr, len ); - *ptr += len; - *headerLen -= len; - return result; + gchar* checksum = g_compute_checksum_for_data( G_CHECKSUM_MD5, ptr, len ); + XP_MEMCPY( out, checksum, XP_STRLEN(checksum) + 1 ); + g_free( checksum ); } -static XP_U16 -countSpecials( LinuxDictionaryCtxt* ctxt ) -{ - XP_U16 result = 0; - XP_U16 ii; - - for ( ii = 0; ii < ctxt->super.nFaces; ++ii ) { - if ( IS_SPECIAL(ctxt->super.facePtrs[ii][0]) ) { - ++result; - } - } - - return result; -} /* countSpecials */ - -static XP_Bitmap -skipBitmap( LinuxDictionaryCtxt* XP_UNUSED_DBG(ctxt), const XP_U8** ptrp ) -{ - XP_U8 nCols, nRows, nBytes; - LinuxBMStruct* lbs = NULL; - const XP_U8* ptr = *ptrp; - - nCols = *ptr++; - if ( nCols > 0 ) { - nRows = *ptr++; - - nBytes = ((nRows * nCols) + 7) / 8; - - lbs = XP_MALLOC( ctxt->super.mpool, sizeof(*lbs) + nBytes ); - lbs->nRows = nRows; - lbs->nCols = nCols; - lbs->nBytes = nBytes; - - memcpy( lbs + 1, ptr, nBytes ); - ptr += nBytes; - } - - *ptrp = ptr; - return lbs; -} /* skipBitmap */ - -static void -skipBitmaps( LinuxDictionaryCtxt* ctxt, const XP_U8** ptrp ) -{ - XP_U16 nSpecials; - XP_UCHAR* text; - XP_UCHAR** texts; - XP_UCHAR** textEnds; - SpecialBitmaps* bitmaps; - Tile tile; - const XP_U8* ptr = *ptrp; - - nSpecials = countSpecials( ctxt ); - - texts = (XP_UCHAR**)XP_MALLOC( ctxt->super.mpool, - nSpecials * sizeof(*texts) ); - textEnds = (XP_UCHAR**)XP_MALLOC( ctxt->super.mpool, - nSpecials * sizeof(*textEnds) ); - bitmaps = (SpecialBitmaps*)XP_MALLOC( ctxt->super.mpool, - nSpecials * sizeof(*bitmaps) ); - XP_MEMSET( bitmaps, 0, nSpecials * sizeof(*bitmaps) ); - - for ( tile = 0; tile < ctxt->super.nFaces; ++tile ) { - - const XP_UCHAR* facep = ctxt->super.facePtrs[(short)tile]; - if ( IS_SPECIAL(*facep) ) { - XP_U16 asIndex = (XP_U16)*facep; - XP_U8 txtlen; - XP_ASSERT( *facep < nSpecials ); - - /* get the string */ - txtlen = *ptr++; - text = (XP_UCHAR*)XP_MALLOC(ctxt->super.mpool, txtlen+1); - memcpy( text, ptr, txtlen ); - ptr += txtlen; - - text[txtlen] = '\0'; - texts[(XP_U16)*facep] = text; - textEnds[(XP_U16)*facep] = text + txtlen + 1; - - /* Now replace the delimiter char with \0. It must be one byte in - length and of course equal to the delimiter */ - XP_ASSERT( 0 == (SYNONYM_DELIM & 0x80) ); - while ( '\0' != *text ) { - XP_UCHAR* cp = g_utf8_offset_to_pointer( text, 1 ); - if ( 1 == (cp - text) && *text == SYNONYM_DELIM ) { - *text = '\0'; - } - text = cp; - } - - XP_DEBUGF( "skipping bitmaps for " XP_S, texts[asIndex] ); - - bitmaps[asIndex].largeBM = skipBitmap( ctxt, &ptr ); - bitmaps[asIndex].smallBM = skipBitmap( ctxt, &ptr ); - } - } - *ptrp = ptr; - - ctxt->super.chars = texts; - ctxt->super.charEnds = textEnds; - ctxt->super.bitmaps = bitmaps; -} /* skipBitmaps */ - void dict_splitFaces( DictionaryCtxt* dict, XWEnv XP_UNUSED(xwe), const XP_U8* utf8, XP_U16 nBytes, XP_U16 nFaces ) @@ -263,13 +156,6 @@ initFromDictFile( LinuxDictionaryCtxt* dctx, const LaunchParams* params, XP_Bool formatOk = XP_TRUE; size_t dictLength; XP_U32 topOffset; - unsigned short xloc; - XP_U16 flags; - XP_U16 facesSize; - XP_U16 charSize; - XP_Bool isUTF8 = XP_FALSE; - XP_Bool hasHeader = XP_FALSE; - const XP_U8* ptr; char path[256]; if ( !getDictPath( params, fileName, path, VSIZE(path) ) ) { @@ -298,130 +184,12 @@ initFromDictFile( LinuxDictionaryCtxt* dctx, const LaunchParams* params, fclose( dictF ); } - ptr = dctx->dictBase; - - memcpy( &flags, ptr, sizeof(flags) ); - ptr += sizeof( flags ); - flags = ntohs(flags); - - XP_DEBUGF( "flags=0X%X", flags ); - hasHeader = 0 != (DICT_HEADER_MASK & flags); - if ( hasHeader ) { - flags &= ~DICT_HEADER_MASK; - } - - flags &= ~DICT_SYNONYMS_MASK; - - if ( flags == 0x0001 ) { - dctx->super.nodeSize = 3; - charSize = 1; - dctx->super.is_4_byte = XP_FALSE; - } else if ( flags == 0x0002 ) { - dctx->super.nodeSize = 3; - charSize = 2; - dctx->super.is_4_byte = XP_FALSE; - } else if ( flags == 0x0003 ) { - dctx->super.nodeSize = 4; - charSize = 2; - dctx->super.is_4_byte = XP_TRUE; - } else if ( flags == 0x0004 ) { - dctx->super.nodeSize = 3; - dctx->super.isUTF8 = XP_TRUE; - isUTF8 = XP_TRUE; - dctx->super.is_4_byte = XP_FALSE; - } else if ( flags == 0x0005 ) { - dctx->super.nodeSize = 4; - dctx->super.isUTF8 = XP_TRUE; - isUTF8 = XP_TRUE; - dctx->super.is_4_byte = XP_TRUE; - } else { - /* case I don't know how to deal with */ - formatOk = XP_FALSE; - XP_ASSERT(0); - } + const XP_U8* ptr = dctx->dictBase; + const XP_U8* end = ptr + dctx->dictLength; + formatOk = parseCommon( &dctx->super, NULL_XWE, &ptr, end ); + /* && loadSpecialData( &dctx->super, &ptr, end ); */ if ( formatOk ) { - XP_U8 numFaceBytes, numFaces; - - if ( hasHeader ) { - XP_U16 headerLen; - XP_U32 wordCount; - - memcpy( &headerLen, ptr, sizeof(headerLen) ); - ptr += sizeof(headerLen); - headerLen = ntohs( headerLen ); - - memcpy( &wordCount, ptr, sizeof(wordCount) ); - ptr += sizeof(wordCount); - headerLen -= sizeof(wordCount); - dctx->super.nWords = ntohl( wordCount ); - XP_DEBUGF( "dict contains %d words", dctx->super.nWords ); - - if ( 0 < headerLen ) { - dctx->super.desc = getNullTermParam( dctx, &ptr, &headerLen ); - } else { - XP_LOGF( "%s: no note", __func__ ); - } - if ( 0 < headerLen ) { - dctx->super.md5Sum = getNullTermParam( dctx, &ptr, &headerLen ); - } else { - XP_LOGF( "%s: no md5Sum", __func__ ); - } - ptr += headerLen; - } - - if ( isUTF8 ) { - numFaceBytes = *ptr++; - } - numFaces = *ptr++; - if ( !isUTF8 ) { - numFaceBytes = numFaces * charSize; - } - - if ( NULL == dctx->super.md5Sum -#ifdef DEBUG - || XP_TRUE -#endif - ) { - size_t curPos = ptr - dctx->dictBase; - gssize dictLength = dctx->dictLength - curPos; - - gchar* checksum = g_compute_checksum_for_data( G_CHECKSUM_MD5, ptr, dictLength ); - if ( NULL == dctx->super.md5Sum ) { - dctx->super.md5Sum = copyString( dctx->super.mpool, checksum ); - } else { - XP_ASSERT( 0 == XP_STRCMP( dctx->super.md5Sum, checksum ) ); - } - g_free( checksum ); - } - - dctx->super.nFaces = numFaces; - - dctx->super.countsAndValues = XP_MALLOC( dctx->super.mpool, - numFaces*2 ); - facesSize = numFaceBytes; - if ( !isUTF8 ) { - facesSize /= 2; - } - - XP_U8 tmp[numFaceBytes]; - memcpy( tmp, ptr, numFaceBytes ); - ptr += numFaceBytes; - - dict_splitFaces( &dctx->super, NULL, tmp, numFaceBytes, numFaces ); - - memcpy( &xloc, ptr, sizeof(xloc) ); - ptr += sizeof(xloc); - memcpy( dctx->super.countsAndValues, ptr, numFaces*2 ); - ptr += numFaces*2; - } - - dctx->super.langCode = xloc & 0x7F; - - if ( formatOk ) { - XP_U32 numEdges; - skipBitmaps( dctx, &ptr ); - size_t curPos = ptr - dctx->dictBase; dictLength = dctx->dictLength - curPos; @@ -433,6 +201,7 @@ initFromDictFile( LinuxDictionaryCtxt* dctx, const LaunchParams* params, ptr += sizeof(topOffset); } + XP_U32 numEdges; if ( dictLength > 0 ) { numEdges = dictLength / dctx->super.nodeSize; #ifdef DEBUG @@ -509,6 +278,7 @@ linux_dictionary_destroy( DictionaryCtxt* dict, XWEnv XP_UNUSED(xwe) ) } } + /* super's destructor should do this!!!! */ XP_FREEP( dict->mpool, &ctxt->super.desc ); XP_FREEP( dict->mpool, &ctxt->super.md5Sum ); XP_FREEP( dict->mpool, &ctxt->super.countsAndValues );