Miroirs/xwords
1
0
Fork 0
mirror of git://xwords.git.sourceforge.net/gitroot/xwords/xwords synced 2025-02-11 08:48:06 +01:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'android_branch' into android_dictdb

Browse source
This commit is contained in:
Eric House 2012-09-15 10:51:58 -07:00
commit 1e12464934
2 changed files with 110 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
xwords4/android/scripts/info.py
View file

@ -1,6 +1,7 @@
#!/usr/bin/python
# Script meant to be installed on eehouse.org. # Script meant to be installed on eehouse.org.
import logging, shelve, hashlib, sys, json import logging, shelve, hashlib, sys, json, subprocess
try: try:
from mod_python import apache from mod_python import apache
apacheAvailable = True apacheAvailable = True
@ -18,6 +19,7 @@ k_DICTS = 'dicts'
k_LANG = 'lang' k_LANG = 'lang'
k_MD5SUM = 'md5sum' k_MD5SUM = 'md5sum'
k_INDEX = 'index' k_INDEX = 'index'
k_ISUM = 'isum'
k_SUCCESS = 'success' k_SUCCESS = 'success'
k_URL = 'url' k_URL = 'url'
@ -26,7 +28,7 @@ k_COUNT = 'count'
k_suffix = '.xwd' k_suffix = '.xwd'
k_filebase = "/var/www/" k_filebase = "/var/www/"
k_shelfFile = k_filebase + 'xw4/info_shelf' k_shelfFile = k_filebase + 'xw4/info_shelf_2'
k_urlbase = "http://eehouse.org/" k_urlbase = "http://eehouse.org/"
k_versions = { 'org.eehouse.android.xw4': { k_versions = { 'org.eehouse.android.xw4': {
'version' : 43, 'version' : 43,
@ -49,8 +51,8 @@ k_versions_dbg = { 'org.eehouse.android.xw4': {
'org.eehouse.android.xw4sms' : { 'org.eehouse.android.xw4sms' : {
'version' : 43, 'version' : 43,
k_AVERS : 43, k_AVERS : 43,
k_GVERS : 'android_beta_51', k_GVERS : 'android_beta_51-50-g0ccc233',
k_URL : 'xw4/android/sms/XWords4-release_android_beta_51.apk', k_URL : 'xw4/android/sms/XWords4-release_android_beta_51-50-g0ccc233.apk',
}, },
} }
s_shelf = None s_shelf = None
@ -63,24 +65,43 @@ logging.basicConfig(level=logging.DEBUG
# ,filemode='w') # ,filemode='w')
# This seems to be required to prime the pump somehow. # This seems to be required to prime the pump somehow.
logging.debug( "loaded...." ) # logging.debug( "loaded...." )
def md5Checksum(sums, filePath): def getInternalSum( filePath ):
filePath = k_filebase + "and_wordlists/" + filePath
proc = subprocess.Popen(['/usr/bin/perl',
'--',
'/var/www/xw4/dawg2dict.pl',
'-get-sum',
'-dict', filePath ],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
return proc.communicate()[0].strip()
def md5Checksums( sums, filePath ):
if not filePath.endswith(k_suffix): filePath += k_suffix if not filePath.endswith(k_suffix): filePath += k_suffix
if not filePath in sums: if not filePath in sums:
logging.debug( "opening %s" % (k_filebase + "and_wordlists/" + filePath))
file = open( k_filebase + "and_wordlists/" + filePath, 'rb' ) file = open( k_filebase + "and_wordlists/" + filePath, 'rb' )
md5 = hashlib.md5() md5 = hashlib.md5()
while True: while True:
buffer = file.read(128) buffer = file.read(128)
if not buffer: break if not buffer: break
md5.update( buffer ) md5.update( buffer )
sums[filePath] = md5.hexdigest()
sums[filePath] = [ md5.hexdigest(), getInternalSum( filePath ) ]
logging.debug( "figured sum for %s" % filePath ) logging.debug( "figured sum for %s" % filePath )
return sums[filePath] return sums[filePath]
def getDictSums(): def getDictSums():
global s_shelf global s_shelf
s_shelf = shelve.open(k_shelfFile) # shelve will fail if permissions are wrong. That's ok for some
# testing: just make a fake shelf and test before saving it later.
try:
s_shelf = shelve.open(k_shelfFile)
except:
s_shelf = {}
if not k_SUMS in s_shelf: s_shelf[k_SUMS] = {} if not k_SUMS in s_shelf: s_shelf[k_SUMS] = {}
if not k_COUNT in s_shelf: s_shelf[k_COUNT] = 0 if not k_COUNT in s_shelf: s_shelf[k_COUNT] = 0
s_shelf[k_COUNT] += 1 s_shelf[k_COUNT] += 1
@ -115,16 +136,16 @@ def dictVersion( req, name, lang, md5sum ):
dictSums = getDictSums() dictSums = getDictSums()
path = lang + "/" + name path = lang + "/" + name
if not path in dictSums: if not path in dictSums:
sum = md5Checksum( dictSums, path ) sums = md5Checksums( dictSums, path )
if sum: if sums:
dictSums[path] = sum dictSums[path] = sums
s_shelf[k_SUMS] = dictSums s_shelf[k_SUMS] = dictSums
if path in dictSums: if path in dictSums:
if dictSums[path] != md5sum: if not md5sum in dictSums[path]:
result[k_URL] = k_urlbase + "and_wordlists/" + path result[k_URL] = k_urlbase + "and_wordlists/" + path
else: else:
logging.debug( path + " not known" ) logging.debug( path + " not known" )
s_shelf.close() if 'close' in s_shelf: s_shelf.close()
return json.dumps( result ) return json.dumps( result )
def getApp( params ): def getApp( params ):
@ -137,6 +158,7 @@ def getApp( params ):
gvers = params[k_GVERS] gvers = params[k_GVERS]
if k_INSTALLER in params: installer = params[k_INSTALLER] if k_INSTALLER in params: installer = params[k_INSTALLER]
else: installer = '' else: installer = ''
logging.debug( "name: %s; avers: %s; installer: %s; gvers: %s" logging.debug( "name: %s; avers: %s; installer: %s; gvers: %s"
% (name, avers, installer, gvers) ) % (name, avers, installer, gvers) )
if k_DEVOK in params and params[k_DEVOK]: versions = k_versions_dbg if k_DEVOK in params and params[k_DEVOK]: versions = k_versions_dbg
@ -166,18 +188,19 @@ def getDicts( params ):
if not name.endswith(k_suffix): name += k_suffix if not name.endswith(k_suffix): name += k_suffix
path = lang + "/" + name path = lang + "/" + name
if not path in dictSums: if not path in dictSums:
sum = md5Checksum( dictSums, path ) sums = md5Checksums( dictSums, path )
if sum: if sums:
dictSums[path] = sum dictSums[path] = sums
s_shelf[k_SUMS] = dictSums s_shelf[k_SUMS] = dictSums
if path in dictSums: if path in dictSums:
if dictSums[path] != md5sum: if not md5sum in dictSums[path]:
cur = { k_URL : k_urlbase + "and_wordlists/" + path, cur = { k_URL : k_urlbase + "and_wordlists/" + path,
k_INDEX : index } k_INDEX : index, k_ISUM: dictSums[path][1] }
result.append( cur ) result.append( cur )
else: else:
logging.debug( path + " not known" ) logging.debug( path + " not known" )
if 'close' in s_shelf: s_shelf.close()
if 0 == len(result): result = None if 0 == len(result): result = None
return result return result
@ -201,7 +224,8 @@ def clearShelf():
def usage(): def usage():
print "usage:", sys.argv[0], '--get-sums [lang/dict]*' print "usage:", sys.argv[0], '--get-sums [lang/dict]*'
print " | --test-get-app app <org.eehouse.app.name> avers gvers" print ' | --test-get-app app <org.eehouse.app.name> avers gvers'
print ' | --test-get-dicts name lang curSum'
print ' | --clear-shelf' print ' | --clear-shelf'
sys.exit(-1) sys.exit(-1)
@ -213,9 +237,9 @@ def main():
elif arg == '--get-sums': elif arg == '--get-sums':
dictSums = getDictSums() dictSums = getDictSums()
for arg in sys.argv[2:]: for arg in sys.argv[2:]:
print arg, md5Checksum(dictSums, arg) print arg, md5Checksums(dictSums, arg)
s_shelf[k_SUMS] = dictSums s_shelf[k_SUMS] = dictSums
s_shelf.close() if 'close' in s_shelf: s_shelf.close()
elif arg == '--test-get-app': elif arg == '--test-get-app':
if not 5 == len(sys.argv): usage() if not 5 == len(sys.argv): usage()
params = { k_NAME: sys.argv[2], params = { k_NAME: sys.argv[2],
@ -223,6 +247,14 @@ def main():
k_GVERS: sys.argv[4], k_GVERS: sys.argv[4],
} }
print getApp( params ) print getApp( params )
elif arg == '--test-get-dicts':
if not 5 == len(sys.argv): usage()
params = { k_NAME: sys.argv[2],
k_LANG : sys.argv[3],
k_MD5SUM : sys.argv[4],
k_INDEX : 0,
}
print getDicts( [params] )
else: else:
usage() usage()

84
xwords4/dawg/dawg2dict.pl
View file

@ -1,4 +1,4 @@
#!/usr/bin/perl -CS #!/usr/bin/perl -C
# #
# Copyright 2004 - 2009 by Eric House (xwords@eehouse.org) # Copyright 2004 - 2009 by Eric House (xwords@eehouse.org)
# #
@ -24,10 +24,11 @@ use strict;
use Fcntl; use Fcntl;
use Encode 'from_to'; use Encode 'from_to';
use Encode; use Encode;
use Digest::MD5;
my $gInFile; my $gInFile;
my $gSumOnly = 0; my $gSumOnly = 0;
my $gSum; my $gSum = '';
my $gDescOnly = 0; my $gDescOnly = 0;
my $gDesc; my $gDesc;
my $gDoRaw = 0; my $gDoRaw = 0;
@ -95,24 +96,37 @@ sub readXWDFaces($$$) {
$nChars = unpack( 'c', $buf ); $nChars = unpack( 'c', $buf );
printf STDERR "nChars of faces: %d\n", $nChars; printf STDERR "nChars of faces: %d\n", $nChars;
binmode( $fh, ":encoding(utf8)" ) or die "binmode(:utf-8) failed\n"; # At this point $fh is pointing at the start of data
sysread( $fh, $buf, $nChars ); if ( $gSumOnly ) {
length($buf) == $nChars or die "didn't read expected number of bytes\n"; my $sum = sumRestOfFile( $fh );
binmode( $fh ) or die "binmode failed\n"; if ( $sum eq $gSum ) {
print STDERR "got: $sum, $gSum twice!!!\n";
} elsif ( !$gSum ) {
$gSum = $sum;
} else {
print STDERR "disagreement: $sum vs $gSum\n";
exit( -1 );
}
} else {
binmode( $fh, ":encoding(utf8)" ) or die "binmode(:utf-8) failed\n";
sysread( $fh, $buf, $nChars );
length($buf) == $nChars or die "didn't read expected number of bytes\n";
binmode( $fh ) or die "binmode failed\n";
print STDERR "string now: $buf\n"; print STDERR "string now: $buf\n";
my @faces; my @faces;
for ( my $ii = 0; $ii < $nChars; ++$ii ) { for ( my $ii = 0; $ii < $nChars; ++$ii ) {
my $chr = substr( $buf, $ii, 1 ); my $chr = substr( $buf, $ii, 1 );
print STDERR "pushing $chr \n"; print STDERR "pushing $chr \n";
push( @faces, $chr ); push( @faces, $chr );
}
printf STDERR "at 0x%x after reading faces\n", systell($fh);
${$nSpecials} = countSpecials( \@faces );
@{$facRef} = @faces;
printf STDERR "readXWDFaces=>%d\n", $nChars;
} }
printf STDERR "at 0x%x after reading faces\n", systell($fh);
${$nSpecials} = countSpecials( \@faces );
@{$facRef} = @faces;
printf STDERR "readXWDFaces=>%d\n", $nChars;
return $nChars; return $nChars;
} # readXWDFaces } # readXWDFaces
@ -170,7 +184,12 @@ sub printHeader($$) {
my ( $buf, $len ) = @_; my ( $buf, $len ) = @_;
printf STDERR "skipped %d bytes of header:\n", $len + 2; printf STDERR "skipped %d bytes of header:\n", $len + 2;
my $count; my $count;
($count, $gDesc, $gSum) = unpack( 'N Z* Z*', $buf ); if ( $len == 4 ) {
($count) = unpack( 'N', $buf );
} else {
print STDERR 'unpacking...\n';
($count, $gDesc, $gSum) = unpack( 'N Z* Z*', $buf );
}
printf STDERR "has %d words\n", $count; printf STDERR "has %d words\n", $count;
} }
@ -190,7 +209,7 @@ sub nodeSizeFromFlags($$) {
if ( $flags == 2 || $ flags == 4 ) { if ( $flags == 2 || $ flags == 4 ) {
return 3; return 3;
} elsif ( $flags == 5 ) { } elsif ( $flags == 3 || $flags == 5 ) {
return 4; return 4;
} else { } else {
die "invalid dict flags $flags"; die "invalid dict flags $flags";
@ -208,6 +227,21 @@ sub mergeSpecials($$) {
} }
} }
sub sumRestOfFile($) {
my ( $fh ) = @_;
my $buf;
my $count = 0;
my $md5 = Digest::MD5->new;
for ( ; ; ) {
my $nRead = sysread( $fh, $buf, 128 );
0 == $nRead && last;
$count += $nRead;
$md5->add( $buf );
}
# print STDERR "read $count bytes\n";
return $md5->hexdigest();
}
sub prepXWD($$$$) { sub prepXWD($$$$) {
my ( $fh, $facRef, $nodesRef, $startRef ) = @_; my ( $fh, $facRef, $nodesRef, $startRef ) = @_;
my $done = 1; my $done = 1;
@ -219,18 +253,14 @@ sub prepXWD($$$$) {
$gNodeSize = nodeSizeFromFlags( $fh, $flags ); $gNodeSize = nodeSizeFromFlags( $fh, $flags );
my $nSpecials;
my $faceCount = readXWDFaces( $fh, $facRef, \$nSpecials ); # does checksum
if ( $gSumOnly ) { if ( $gSumOnly ) {
print STDOUT $gSum, "\n"; print STDOUT $gSum, "\n";
} elsif( $gDescOnly ) { } elsif( $gDescOnly ) {
print STDOUT $gDesc, "\n"; print STDOUT $gDesc, "\n";
} else { } else {
$done = 0;
}
if ( !$done ) {
my $nSpecials;
my $faceCount = readXWDFaces( $fh, $facRef, \$nSpecials );
printf STDERR "at 0x%x before header read\n", systell($fh); printf STDERR "at 0x%x before header read\n", systell($fh);
# skip xloc header # skip xloc header
$nRead = sysread( $fh, $buf, 2 ); $nRead = sysread( $fh, $buf, 2 );