move hashing into memstream impl, and reintroduce the old/broken

treatment of the final 8 bits. I'll need to release a version that
still produces the incorrect hash for compatibility with existing
clients that expect it, but that also looks for the correct hash. When
that's out there I can do a new release that sends the correct hash.

xwords4/common/memstream.c

@@ -25,6 +25,7 @@
 #include "comtypes.h"
 #include "memstream.h"
 #include "vtabmgr.h"
+#include "strutils.h"
 
 #ifdef CPLUS
 extern "C" {
@@ -184,7 +185,7 @@ mem_stream_getBits( XWStreamCtxt* p_sctx, XP_U16 nBits )
     return result;
 } /* stream_getBits */
 
-#if defined HASH_STREAM || defined DEBUG
+#if defined DEBUG
 static void
 mem_stream_copyBits( const XWStreamCtxt* p_sctx, XWStreamPos endPos,
                      XP_U8* buf, XP_U16* lenp )
@@ -371,6 +372,36 @@ mem_stream_getSize( const XWStreamCtxt* p_sctx )
     return size;
 } /* mem_stream_getSize */
 
+static XP_U32
+mem_stream_getHash( const XWStreamCtxt* p_sctx, XWStreamPos pos,
+                    XP_Bool correct )
+{
+    XP_U32 hash = 0;
+    const MemStreamCtxt* stream = (const MemStreamCtxt*)p_sctx;
+    const XP_U8* ptr = stream->buf; 
+    XP_U16 len = BYTE_PART(pos);
+    XP_U16 bits = BIT_PART(pos);
+    if ( 0 != bits ) {
+        XP_ASSERT( 0 < len );
+        --len;
+    }
+
+    hash = augmentHash( 0, ptr, len );
+    if ( 0 != bits ) {
+        XP_U8 byt = ptr[len];
+        if ( correct ) {
+            byt &= ~(0xFF << bits);
+        } else {
+            byt &= 1 << bits;
+        }
+        hash = augmentHash( hash, &byt, 1 );
+    }
+    hash = finishHash( hash );
+
+    LOG_RETURNF( "%X(%d:%d)", hash, len, bits );
+    return hash;
+} /* mem_stream_getHash */
+
 static const XP_U8*
 mem_stream_getPtr( const XWStreamCtxt* p_sctx )
 {
@@ -474,7 +505,7 @@ make_vtable( MemStreamCtxt* stream )
     SET_VTABLE_ENTRY( vtable, stream_getU16, mem );
     SET_VTABLE_ENTRY( vtable, stream_getU32, mem );
     SET_VTABLE_ENTRY( vtable, stream_getBits, mem );
-#if defined HASH_STREAM || defined DEBUG
+#if defined DEBUG
     SET_VTABLE_ENTRY( vtable, stream_copyBits, mem );
 #endif
 
@@ -495,6 +526,7 @@ make_vtable( MemStreamCtxt* stream )
     SET_VTABLE_ENTRY( vtable, stream_close, mem );
 
     SET_VTABLE_ENTRY( vtable, stream_getSize, mem );
+    SET_VTABLE_ENTRY( vtable, stream_getHash, mem );
     SET_VTABLE_ENTRY( vtable, stream_getPtr, mem );
     SET_VTABLE_ENTRY( vtable, stream_getAddress, mem );
     SET_VTABLE_ENTRY( vtable, stream_setAddress, mem );

xwords4/common/model.c

@@ -312,33 +312,22 @@ model_destroy( ModelCtxt* model )
 } /* model_destroy */
 
 XP_U32
-model_getHash( const ModelCtxt* model, XP_U16 version )
+model_getHash( const ModelCtxt* model )
 {
 #ifndef STREAM_VERS_HASHSTREAM
     XP_USE(version);
 #endif
     StackCtxt* stack = model->vol.stack;
     XP_ASSERT( !!stack );
-    XP_U32 hash =
+    return stack_getHash( stack, XP_FALSE );
-#ifdef STREAM_VERS_HASHSTREAM
-    STREAM_VERS_HASHSTREAM <= version ?
-        stack_getHash( stack ) : 
-#endif
-        stack_getHashOld( stack );
-    /* XP_LOGF( "%s(version=%x)=>%.8X", __func__, version,  */
-    /*          (unsigned int)hash ); */
-    return hash;
 }
 
 XP_Bool
 model_hashMatches( const ModelCtxt* model, const XP_U32 hash )
 {
     StackCtxt* stack = model->vol.stack;
-    XP_Bool matches = 
+    XP_Bool matches = hash == stack_getHash( stack, XP_TRUE )
-#ifdef STREAM_VERS_HASHSTREAM
+        || hash == stack_getHash( stack, XP_FALSE );
-        (hash == stack_getHash( stack )) ||
-#endif
-        (hash == stack_getHashOld( stack ));
     return matches;
 }
 
@@ -349,18 +338,11 @@ model_popToHash( ModelCtxt* model, const XP_U32 hash, PoolContext* pool )
     StackCtxt* stack = model->vol.stack;
     const XP_U16 nEntries = stack_getNEntries( stack );
     StackEntry entries[nEntries];
-#ifdef DEBUG
-    XP_U32 hashes[nEntries];
-#endif
     XP_S16 foundAt = -1;
 
     for ( XP_U16 ii = 0; ii < nEntries; ++ii ) {
-        XP_U32 thisHash =
+        if ( hash == stack_getHash( stack, XP_TRUE )
-#ifdef DEBUG
+             || hash == stack_getHash( stack, XP_FALSE ) ) {
-        hashes[ii] =
-#endif
-            stack_getHash( stack );
-        if ( hash == thisHash ) {
             foundAt = ii;
             break;
         }
@@ -372,8 +354,6 @@ model_popToHash( ModelCtxt* model, const XP_U32 hash, PoolContext* pool )
 
     for ( XP_S16 ii = nPopped - 1; ii >= 0; --ii ) {
         stack_redo( stack, &entries[ii] );
-        /* Assert not needed for long */
-        XP_ASSERT( hashes[ii] = stack_getHash( stack ) );
     }
 
     XP_Bool found = -1 != foundAt;
@@ -386,7 +366,8 @@ model_popToHash( ModelCtxt* model, const XP_U32 hash, PoolContext* pool )
             model_undoLatestMoves( model, pool, foundAt, NULL, NULL );
         XP_ASSERT( success );
         /* Assert not needed for long */
-        XP_ASSERT( hash == stack_getHash( model->vol.stack ) );
+        XP_ASSERT( hash == stack_getHash( model->vol.stack, XP_TRUE )
+                   || hash == stack_getHash( model->vol.stack, XP_FALSE ) );
     }
 
     return found;

xwords4/common/model.h

@@ -122,7 +122,7 @@ void model_writeToTextStream( const ModelCtxt* model, XWStreamCtxt* stream );
 
 void model_setSize( ModelCtxt* model, XP_U16 boardSize );
 void model_destroy( ModelCtxt* model );
-XP_U32 model_getHash( const ModelCtxt* model, XP_U16 version );
+XP_U32 model_getHash( const ModelCtxt* model );
 XP_Bool model_hashMatches( const ModelCtxt* model, XP_U32 hash );
 XP_Bool model_popToHash( ModelCtxt* model, const XP_U32 hash,
                          PoolContext* pool );

xwords4/common/movestak.c

@@ -67,101 +67,13 @@ stack_init( StackCtxt* stack )
        shrunk to fit as soon as we serialize/deserialize anyway. */
 } /* stack_init */
 
-static XP_U32
-augmentHash( XP_U32 hash, const XP_U8* ptr, XP_U16 len )
-{
-    XP_ASSERT( 0 < len );
-    // see http://en.wikipedia.org/wiki/Jenkins_hash_function
-    XP_U16 ii;
-    for ( ii = 0; ii < len; ++ii ) {
-        hash += *ptr++;
-        hash += (hash << 10);
-        hash ^= (hash >> 6);
-    }
-#ifdef DEBUG_HASHING
-    XP_LOGF( "%s: hashed %d bytes -> %X", __func__, len, (unsigned int)hash );
-#endif
-    return hash;
-}
-
-static XP_U32
-finishHash( XP_U32 hash )
-{
-    hash += (hash << 3);
-    hash ^= (hash >> 11);
-    hash += (hash << 15);
-    return hash;
-}
-
-static XP_U32
-augmentFor( XP_U32 hash, const StackEntry* entry )
-{
-    switch( entry->moveType ) {
-    case ASSIGN_TYPE: {
-        TrayTileSet tiles;
-        sortTiles( &tiles, &entry->u.assign.tiles, 0 );
-        hash = augmentHash( hash, (XP_U8*)&tiles, sizeof(tiles) );
-    }
-        break;
-    case MOVE_TYPE:
-        hash = augmentHash( hash, (XP_U8*)&entry->u.move, 
-                            sizeof(entry->u.move) );
-        break;
-    case TRADE_TYPE:
-        hash = augmentHash( hash, (XP_U8*)&entry->u.trade, 
-                            sizeof(entry->u.trade) );
-        break;
-    case PHONY_TYPE:
-        hash = augmentHash( hash, (XP_U8*)&entry->u.phony, 
-                            sizeof(entry->u.phony) );
-        break;
-    }
-    return hash;
-}
-
-XP_U32
-stack_getHashOld( StackCtxt* stack )
-{
-    XP_U16 nn, nEntries = stack->nEntries;
-    XP_U32 hash = 0L;
-    for ( nn = 0; nn < nEntries; ++nn ) {
-        StackEntry entry;
-        XP_MEMSET( &entry, 0, sizeof(entry) );
-        if ( !stack_getNthEntry( stack, nn, &entry ) ) {
-            XP_ASSERT( 0 );
-        }
-        hash = augmentFor( hash, &entry );
-#ifdef DEBUG_HASHING
-        XP_LOGF( "%s: hash after %d: %.8X", __func__, nn, (unsigned int)hash );
-#endif
-    }
-    XP_ASSERT( 0 != hash );
-    hash = finishHash( hash );
-#ifdef DEBUG_HASHING
-    LOG_RETURNF( "%.8X", (unsigned int)hash );
-#endif
-    return hash;
-} /* stack_getHashOld */
-
 #ifdef STREAM_VERS_HASHSTREAM
 XP_U32
-stack_getHash( const StackCtxt* stack )
+stack_getHash( const StackCtxt* stack, XP_Bool correct )
 {
     XP_U32 hash = 0;
     if ( !!stack->data ) {
-        XP_U16 len = 0;
+        hash = stream_getHash( stack->data, stack->top, correct );
-        stream_copyBits( stack->data, stack->top, NULL, &len );
-        if ( 0 < len ) {
-            XP_U8 buf[len];
-            stream_copyBits( stack->data, stack->top, buf, &len );
-#ifdef DEBUG_HASHING
-            LOG_HEX( buf, len, __func__ );
-#endif
-            hash = finishHash( augmentHash( 0L, buf, len ) );
-#ifdef DEBUG_HASHING
-            LOG_RETURNF( "%.8X", (unsigned int)hash );
-#endif
-        }
     }
     return hash;
 } /* stack_getHash */
@@ -332,7 +244,7 @@ static void
 pushEntry( StackCtxt* stack, const StackEntry* entry )
 {
 #ifdef DEBUG_HASHING
-    XP_U32 origHash = stack_getHash( stack );
+    XP_U32 origHash = stack_getHash( stack, XP_TRUE );
 #endif
 
     pushEntryImpl( stack, entry );
@@ -341,7 +253,7 @@ pushEntry( StackCtxt* stack, const StackEntry* entry )
     XP_U32 newHash = stack_getHash( stack );
     StackEntry lastEntry;
     if ( stack_popEntry( stack, &lastEntry ) ) {
-        XP_ASSERT( origHash == stack_getHash( stack ) );
+        XP_ASSERT( origHash == stack_getHash( stack, XP_TRUE ) );
         pushEntryImpl( stack, &lastEntry );
         XP_ASSERT( newHash == stack_getHash( stack ) );
         XP_LOGF( "%s: all ok", __func__ );

xwords4/common/movestak.h

@@ -69,8 +69,7 @@ StackCtxt* stack_make( MPFORMAL VTableMgr* vtmgr );
 void stack_destroy( StackCtxt* stack );
 
 void stack_init( StackCtxt* stack );
-XP_U32 stack_getHashOld( StackCtxt* stack );
+XP_U32 stack_getHash( const StackCtxt* stack, XP_Bool correct );
-XP_U32 stack_getHash( const StackCtxt* stack );
 void stack_setBitsPerTile( StackCtxt* stack, XP_U16 bitsPerTile );
 
 void stack_loadFromStream( StackCtxt* stack, XWStreamCtxt* stream );

xwords4/common/server.c

@@ -2032,7 +2032,7 @@ sendMoveTo( ServerCtxt* server, XP_U16 devIndex, XP_U16 turn,
     XP_U16 version = stream_getVersion( stream );
     if ( STREAM_VERS_BIGBOARD <= version ) {
         XP_ASSERT( version == server->nv.streamVersion );
-        XP_U32 hash = model_getHash( server->vol.model, version );
+        XP_U32 hash = model_getHash( server->vol.model );
 #ifdef DEBUG_HASHING
         XP_LOGF( "%s: adding hash %x", __func__, (unsigned int)hash );
 #endif

xwords4/common/strutils.c

@@ -228,6 +228,31 @@ p_replaceStringIfDifferent( MPFORMAL XP_UCHAR** curLoc, const XP_UCHAR* newStr
     *curLoc = curStr;
 } /* replaceStringIfDifferent */
 
+XP_U32
+augmentHash( XP_U32 hash, const XP_U8* ptr, XP_U16 len )
+{
+    XP_ASSERT( 0 < len );
+    // see http://en.wikipedia.org/wiki/Jenkins_hash_function
+    for ( XP_U16 ii = 0; ii < len; ++ii ) {
+        hash += *ptr++;
+        hash += (hash << 10);
+        hash ^= (hash >> 6);
+    }
+#ifdef DEBUG_HASHING
+    XP_LOGF( "%s: hashed %d bytes -> %X", __func__, len, (unsigned int)hash );
+#endif
+    return hash;
+}
+
+XP_U32
+finishHash( XP_U32 hash )
+{
+    hash += (hash << 3);
+    hash ^= (hash >> 11);
+    hash += (hash << 15);
+    return hash;
+}
+
 /* 
  * A wrapper for printing etc. potentially null strings.
  */

xwords4/common/strutils.h

@@ -68,6 +68,8 @@ XP_UCHAR* p_copyString( MPFORMAL const XP_UCHAR* instr
 # define copyString( p, in ) p_copyString( in )
 #endif
 
+XP_U32 augmentHash( XP_U32 hash, const XP_U8* ptr, XP_U16 len );
+XP_U32 finishHash( XP_U32 hash );
 
 void p_replaceStringIfDifferent( MPFORMAL XP_UCHAR** curLoc, 
                                  const XP_UCHAR* newStr

xwords4/common/xwstream.h

@@ -1,6 +1,7 @@
 /* -*-mode: C; fill-column: 78; c-basic-offset: 4; -*- */
 /* 
- * Copyright 1997 - 2000 by Eric House (xwords@eehouse.org).  All rights reserved.
+ * Copyright 1997 - 2015 by Eric House (xwords@eehouse.org).  All rights
+ * reserved.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -48,7 +49,7 @@ typedef struct StreamCtxVTable {
     XP_U16 (*m_stream_getU16)( XWStreamCtxt* dctx );
     XP_U32 (*m_stream_getU32)( XWStreamCtxt* dctx );
     XP_U32 (*m_stream_getBits)( XWStreamCtxt* dctx, XP_U16 nBits );
-#if defined HASH_STREAM || defined DEBUG
+#if defined DEBUG
     void (*m_stream_copyBits)( const XWStreamCtxt* dctx, XWStreamPos endPos,
                                XP_U8* buf, XP_U16* len );
 #endif
@@ -73,6 +74,8 @@ typedef struct StreamCtxVTable {
     void (*m_stream_close)( XWStreamCtxt* dctx );
 
     XP_U16 (*m_stream_getSize)( const XWStreamCtxt* dctx );
+    XP_U32 (*m_stream_getHash)( const XWStreamCtxt* dctx, XWStreamPos pos,
+                                XP_Bool correct );
     
     const XP_U8* (*m_stream_getPtr)( const XWStreamCtxt* dctx );
 
@@ -113,7 +116,7 @@ struct XWStreamCtxt {
 #define stream_getBits(sc, n) \
          (sc)->vtable->m_stream_getBits((sc), (n))
 
-#if defined HASH_STREAM || defined DEBUG
+#if defined DEBUG
 #define stream_copyBits(sc, e, b, l) \
          (sc)->vtable->m_stream_copyBits((sc), (e), (b), (l))
 #endif
@@ -154,6 +157,9 @@ struct XWStreamCtxt {
 #define stream_getSize(sc) \
          (sc)->vtable->m_stream_getSize((sc))
 
+#define stream_getHash(sc, p, c)                        \
+        (sc)->vtable->m_stream_getHash((sc), (p), (c))
+
 #define stream_getPtr(sc) \
          (sc)->vtable->m_stream_getPtr((sc))