Miroirs/xwords
1
0
Fork 0
mirror of git://xwords.git.sourceforge.net/gitroot/xwords/xwords synced 2025-01-06 05:24:46 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix md5sum calc for non-utf8 wordlists

Browse source 
And use apache logging
This commit is contained in:
Eric House 2019-01-05 18:46:58 -08:00
parent e77b830157
commit 3cf8d7571b
2 changed files with 48 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
xwords4/android/scripts/info.py
View file

@ -1,7 +1,7 @@
#!/usr/bin/python #!/usr/bin/python
# Script meant to be installed on eehouse.org. # Script meant to be installed on eehouse.org.
import logging, shelve, hashlib, sys, re, json, subprocess, glob, os import shelve, hashlib, sys, re, json, subprocess, glob, os
import struct, random, string, psycopg2, zipfile import struct, random, string, psycopg2, zipfile
import mk_for_download, mygit import mk_for_download, mygit
import xwconfig import xwconfig
@ -87,15 +87,6 @@ g_langs = {'English' : 'en',
'Italian' : 'it', 'Italian' : 'it',
} }
logging.basicConfig(level=logging.DEBUG
,format='%(asctime)s [[%(levelname)s]] %(message)s'
,datefmt='%d %b %y %H:%M'
,filename='/tmp/info_py.log')
# ,filemode='w')
# This seems to be required to prime the pump somehow.
# logging.debug( "loaded...." )
def languageCodeFor( lang ): def languageCodeFor( lang ):
result = '' result = ''
if lang in g_langs: result = g_langs[lang] if lang in g_langs: result = g_langs[lang]
@ -110,7 +101,9 @@ def getInternalSum( filePath ):
'-dict', filePath ], '-dict', filePath ],
stdout = subprocess.PIPE, stdout = subprocess.PIPE,
stderr = subprocess.PIPE) stderr = subprocess.PIPE)
return proc.communicate()[0].strip() results = proc.communicate()
# apache.log_error(filePath + ': ' + results[1].strip())
return results[0].strip()
def md5Checksums( sums, filePath ): def md5Checksums( sums, filePath ):
if not filePath.endswith(k_suffix): filePath += k_suffix if not filePath.endswith(k_suffix): filePath += k_suffix
@ -128,11 +121,11 @@ def md5Checksums( sums, filePath ):
sums[filePath] = [ md5.hexdigest(), sums[filePath] = [ md5.hexdigest(),
getInternalSum( filePath ) ] getInternalSum( filePath ) ]
logging.debug( "figured sum for %s: %s" % (filePath, apache.log_error( "figured sum for %s: %s" % (filePath,
sums[filePath] ) ) sums[filePath] ) )
result = sums[filePath] result = sums[filePath]
except: except:
# logging.debug( "Unexpected error: " + sys.exc_info()[0] ) # apache.log_error( "Unexpected error: " + sys.exc_info()[0] )
result = None result = None
return result return result
@ -148,7 +141,7 @@ def openShelf():
if not k_SUMS in s_shelf: s_shelf[k_SUMS] = {} if not k_SUMS in s_shelf: s_shelf[k_SUMS] = {}
if not k_COUNT in s_shelf: s_shelf[k_COUNT] = 0 if not k_COUNT in s_shelf: s_shelf[k_COUNT] = 0
s_shelf[k_COUNT] += 1 s_shelf[k_COUNT] += 1
# logging.debug( "Count now %d" % s_shelf[k_COUNT] ) # apache.log_error( "Count now %d" % s_shelf[k_COUNT] )
def closeShelf(): def closeShelf():
global s_shelf global s_shelf
@ -246,7 +239,7 @@ def getVariantDir( name ):
splits = string.split( name, '.' ) splits = string.split( name, '.' )
last = splits[-1] last = splits[-1]
if not last == 'xw4': result = last + '/' if not last == 'xw4': result = last + '/'
# logging.debug( 'getVariantDir(' + name + ") => " + result ) # apache.log_error( 'getVariantDir(' + name + ") => " + result )
return result return result
# public, but deprecated # public, but deprecated
@ -254,20 +247,20 @@ def curVersion( req, name, avers = 41, gvers = None, installer = None ):
global k_versions global k_versions
result = { k_SUCCESS : True } result = { k_SUCCESS : True }
if apacheAvailable: if apacheAvailable:
logging.debug( 'IP address of requester is %s' apache.log_error( 'IP address of requester is %s'
% req.get_remote_host(apache.REMOTE_NAME) ) % req.get_remote_host(apache.REMOTE_NAME) )
logging.debug( "name: %s; avers: %s; installer: %s; gvers: %s" apache.log_error( "name: %s; avers: %s; installer: %s; gvers: %s"
% (name, avers, installer, gvers) ) % (name, avers, installer, gvers) )
if name in k_versions: if name in k_versions:
versions = k_versions[name] versions = k_versions[name]
if versions[k_AVERS] > int(avers): if versions[k_AVERS] > int(avers):
logging.debug( avers + " is old" ) apache.log_error( avers + " is old" )
result[k_URL] = k_urlbase + '/' + versions[k_URL] result[k_URL] = k_urlbase + '/' + versions[k_URL]
else: else:
logging.debug(name + " is up-to-date") apache.log_error(name + " is up-to-date")
else: else:
logging.debug( 'Error: bad name ' + name ) apache.log_error( 'Error: bad name ' + name )
return json.dumps( result ) return json.dumps( result )
# public, but deprecated # public, but deprecated
@ -285,7 +278,7 @@ def dictVersion( req, name, lang, md5sum ):
if not md5sum in dictSums[path]: if not md5sum in dictSums[path]:
result[k_URL] = k_urlbase + "/and_wordlists/" + path result[k_URL] = k_urlbase + "/and_wordlists/" + path
else: else:
logging.debug( path + " not known" ) apache.log_error( path + " not known" )
closeShelf() closeShelf()
return json.dumps( result ) return json.dumps( result )
@ -303,10 +296,10 @@ def getApp( params, name = None, debug = False):
apk = apks[0] apk = apks[0]
curApk = params[k_GVERS] + '.apk' curApk = params[k_GVERS] + '.apk'
if curApk in apk: if curApk in apk:
logging.debug( "already have " + curApk ) apache.log_error( "already have " + curApk )
else: else:
url = k_urlbase + '/' + k_apkDir + variantDir + apk[len(dir):] url = k_urlbase + '/' + k_apkDir + variantDir + apk[len(dir):]
logging.debug("url: " + url) apache.log_error("url: " + url)
result = {k_URL: url} result = {k_URL: url}
elif k_DEVOK in params and params[k_DEVOK]: elif k_DEVOK in params and params[k_DEVOK]:
apks = getOrderedApks( k_filebase + k_apkDir, name, False ) apks = getOrderedApks( k_filebase + k_apkDir, name, False )
@ -315,18 +308,18 @@ def getApp( params, name = None, debug = False):
# Does path NOT contain name of installed file # Does path NOT contain name of installed file
curApk = params[k_GVERS] + '.apk' curApk = params[k_GVERS] + '.apk'
if curApk in apk: if curApk in apk:
logging.debug( "already have " + curApk ) apache.log_error( "already have " + curApk )
else: else:
url = k_urlbase + '/' + apk[len(k_filebase):] url = k_urlbase + '/' + apk[len(k_filebase):]
result = {k_URL: url} result = {k_URL: url}
logging.debug( result ) apache.log_error( result )
elif k_AVERS in params: elif k_AVERS in params:
vers = params[k_AVERS] vers = params[k_AVERS]
if k_INSTALLER in params: installer = params[k_INSTALLER] if k_INSTALLER in params: installer = params[k_INSTALLER]
else: installer = '' else: installer = ''
logging.debug( "name: %s; installer: %s; gvers: %s" apache.log_error( "name: %s; installer: %s; gvers: %s"
% (name, installer, vers) ) % (name, installer, vers) )
print "name: %s; installer: %s; vers: %s" % (name, installer, vers) print "name: %s; installer: %s; vers: %s" % (name, installer, vers)
dir = k_filebase + k_apkDir + 'rel/' dir = k_filebase + k_apkDir + 'rel/'
@ -335,11 +328,11 @@ def getApp( params, name = None, debug = False):
apk = apk[len(k_filebase):] # strip fs path apk = apk[len(k_filebase):] # strip fs path
result = {k_URL: k_urlbase + '/' + apk} result = {k_URL: k_urlbase + '/' + apk}
else: else:
logging.debug(name + " is up-to-date") apache.log_error(name + " is up-to-date")
else: else:
logging.debug( 'Error: bad name ' + name ) apache.log_error( 'Error: bad name ' + name )
else: else:
logging.debug( 'missing param' ) apache.log_error( 'missing param' )
return result return result
def getStats( path ): def getStats( path ):
@ -434,7 +427,7 @@ def getDicts( params ):
k_INDEX : index, k_ISUM: dictSums[path][1] } k_INDEX : index, k_ISUM: dictSums[path][1] }
result.append( cur ) result.append( cur )
else: else:
logging.debug( path + " not known" ) apache.log_error( path + " not known" )
closeShelf() closeShelf()
if 0 == len(result): result = None if 0 == len(result): result = None
@ -442,16 +435,16 @@ def getDicts( params ):
def variantFor( name ): def variantFor( name ):
if name == 'xw4': result = 'XWords4' if name == 'xw4': result = 'XWords4'
logging.debug( 'variantFor(%s)=>%s' % (name, result)) apache.log_error( 'variantFor(%s)=>%s' % (name, result))
return result return result
def getXlate( params, name, stringsHash ): def getXlate( params, name, stringsHash ):
result = [] result = []
path = xwconfig.k_REPOPATH path = xwconfig.k_REPOPATH
logging.debug('creating repo with path ' + path) apache.log_error('creating repo with path ' + path)
repo = mygit.GitRepo( path ) repo = mygit.GitRepo( path )
logging.debug( "getXlate: %s, hash=%s" % (json.dumps(params), stringsHash) ) apache.log_error( "getXlate: %s, hash=%s" % (json.dumps(params), stringsHash) )
# logging.debug( 'status: ' + repo.status() ) # apache.log_error( 'status: ' + repo.status() )
# reduce org.eehouse.anroid.xxx to xxx, then turn it into a # reduce org.eehouse.anroid.xxx to xxx, then turn it into a
# variant and get the contents of the R.java file # variant and get the contents of the R.java file
@ -466,7 +459,7 @@ def getXlate( params, name, stringsHash ):
# the revision BEFORE the revision that changed R.java # the revision BEFORE the revision that changed R.java
head = repo.getHeadRev() head = repo.getHeadRev()
logging.debug('head = %s' % head) apache.log_error('head = %s' % head)
rjavarevs = repo.getRevsBetween(head, stringsHash, rPath) rjavarevs = repo.getRevsBetween(head, stringsHash, rPath)
if rjavarevs: if rjavarevs:
assert( 1 >= len(rjavarevs) ) assert( 1 >= len(rjavarevs) )
@ -477,7 +470,7 @@ def getXlate( params, name, stringsHash ):
firstPossible = rjavarevs[-2] + '^' firstPossible = rjavarevs[-2] + '^'
# get actual number for rev^ # get actual number for rev^
firstPossible = repo.getRevsBetween( firstPossible, firstPossible )[0] firstPossible = repo.getRevsBetween( firstPossible, firstPossible )[0]
logging.debug('firstPossible: %s' % firstPossible) apache.log_error('firstPossible: %s' % firstPossible)
for entry in params: for entry in params:
curVers = entry[k_XLATEVERS] curVers = entry[k_XLATEVERS]
@ -493,7 +486,7 @@ def getXlate( params, name, stringsHash ):
} ) } )
if 0 == len(result): result = None if 0 == len(result): result = None
logging.debug( "getXlate=>%s" % (json.dumps(result)) ) apache.log_error( "getXlate=>%s" % (json.dumps(result)) )
return result return result
def init(): def init():
@ -549,7 +542,7 @@ def opponentIDsFor( req, params ):
def getUpdates( req, params ): def getUpdates( req, params ):
result = { k_SUCCESS : True } result = { k_SUCCESS : True }
appResult = None appResult = None
logging.debug( "getUpdates: got params: %s" % params ) apache.log_error( "getUpdates: got params: %s" % params )
asJson = json.loads( params ) asJson = json.loads( params )
if k_APP in asJson: if k_APP in asJson:
name = None name = None
@ -564,15 +557,15 @@ def getUpdates( req, params ):
# Let's not upgrade strings at the same time as we're upgrading the app # Let's not upgrade strings at the same time as we're upgrading the app
# if appResult: # if appResult:
# logging.debug( 'skipping xlation upgrade because app being updated' ) # apache.log_error( 'skipping xlation upgrade because app being updated' )
# elif k_XLATEINFO in asJson and k_NAME in asJson and k_STRINGSHASH in asJson: # elif k_XLATEINFO in asJson and k_NAME in asJson and k_STRINGSHASH in asJson:
# xlateResult = getXlate( asJson[k_XLATEINFO], asJson[k_NAME], asJson[k_STRINGSHASH] ) # xlateResult = getXlate( asJson[k_XLATEINFO], asJson[k_NAME], asJson[k_STRINGSHASH] )
# if xlateResult: # if xlateResult:
# logging.debug( xlateResult ) # apache.log_error( xlateResult )
# result[k_XLATEINFO] = xlateResult; # result[k_XLATEINFO] = xlateResult;
result = json.dumps( result ) result = json.dumps( result )
# logging.debug( result ) # apache.log_error( result )
return result return result
def clearShelf(): def clearShelf():

20
xwords4/dawg/dawg2dict.pl
View file

@ -35,6 +35,7 @@ my $gDoRaw = 0;
my $gDoJSON = 0; my $gDoJSON = 0;
my $gFileType; my $gFileType;
my $gNodeSize; my $gNodeSize;
my $gIsUTF;
use Fcntl 'SEEK_CUR'; use Fcntl 'SEEK_CUR';
@ -98,9 +99,12 @@ sub readXWDFaces($$$) {
$nRead = sysread( $fh, $buf, 1 ); $nRead = sysread( $fh, $buf, 1 );
$nBytes = unpack( 'c', $buf ); $nBytes = unpack( 'c', $buf );
printf STDERR "nBytes of faces: %d\n", $nBytes; printf STDERR "nBytes of faces: %d\n", $nBytes;
$nRead = sysread( $fh, $buf, 1 );
$nChars = unpack( 'c', $buf ); if ( $gIsUTF ) {
printf STDERR "nChars of faces: %d\n", $nChars; $nRead = sysread( $fh, $buf, 1 );
$nChars = unpack( 'c', $buf );
printf STDERR "nChars of faces: %d\n", $nChars;
}
# At this point $fh is pointing at the start of data # At this point $fh is pointing at the start of data
if ( $gSumOnly ) { if ( $gSumOnly ) {
@ -224,13 +228,17 @@ sub nodeSizeFromFlags($$) {
$flags = $flags & ~DICT_SYNONYMS_MASK; $flags = $flags & ~DICT_SYNONYMS_MASK;
my $nodeSize;
if ( $flags == 2 || $flags == 4 ) { if ( $flags == 2 || $flags == 4 ) {
return 3; $nodeSize = 3;
} elsif ( $flags == 3 || $flags == 5 ) { } elsif ( $flags == 3 || $flags == 5 ) {
return 4; $nodeSize = 4;
} else { } else {
die "invalid dict flags $flags"; die "invalid dict flags $flags";
} }
my $isUTF = $flags == 4 || $flags == 5;
return ( $nodeSize, $isUTF );
} # nodeSizeFromFlags } # nodeSizeFromFlags
sub mergeSpecials($$) { sub mergeSpecials($$) {
@ -268,7 +276,7 @@ sub prepXWD($$$$) {
my $nRead = sysread( $fh, $buf, 2 ); my $nRead = sysread( $fh, $buf, 2 );
my $flags = unpack( "n", $buf ); my $flags = unpack( "n", $buf );
$gNodeSize = nodeSizeFromFlags( $fh, $flags ); ($gNodeSize, $gIsUTF) = nodeSizeFromFlags( $fh, $flags );
my $nSpecials; my $nSpecials;
my $faceCount = readXWDFaces( $fh, $facRef, \$nSpecials ); # does checksum my $faceCount = readXWDFaces( $fh, $facRef, \$nSpecials ); # does checksum