Merge branch 'android_branch' into android_dictdb

This commit is contained in:
Eric House 2012-09-15 10:51:58 -07:00
commit 1e12464934
2 changed files with 110 additions and 48 deletions

View file

@ -1,6 +1,7 @@
#!/usr/bin/python
# Script meant to be installed on eehouse.org.
import logging, shelve, hashlib, sys, json
import logging, shelve, hashlib, sys, json, subprocess
try:
from mod_python import apache
apacheAvailable = True
@ -18,6 +19,7 @@ k_DICTS = 'dicts'
k_LANG = 'lang'
k_MD5SUM = 'md5sum'
k_INDEX = 'index'
k_ISUM = 'isum'
k_SUCCESS = 'success'
k_URL = 'url'
@ -26,7 +28,7 @@ k_COUNT = 'count'
k_suffix = '.xwd'
k_filebase = "/var/www/"
k_shelfFile = k_filebase + 'xw4/info_shelf'
k_shelfFile = k_filebase + 'xw4/info_shelf_2'
k_urlbase = "http://eehouse.org/"
k_versions = { 'org.eehouse.android.xw4': {
'version' : 43,
@ -49,8 +51,8 @@ k_versions_dbg = { 'org.eehouse.android.xw4': {
'org.eehouse.android.xw4sms' : {
'version' : 43,
k_AVERS : 43,
k_GVERS : 'android_beta_51',
k_URL : 'xw4/android/sms/XWords4-release_android_beta_51.apk',
k_GVERS : 'android_beta_51-50-g0ccc233',
k_URL : 'xw4/android/sms/XWords4-release_android_beta_51-50-g0ccc233.apk',
},
}
s_shelf = None
@ -63,24 +65,43 @@ logging.basicConfig(level=logging.DEBUG
# ,filemode='w')
# This seems to be required to prime the pump somehow.
logging.debug( "loaded...." )
# logging.debug( "loaded...." )
def md5Checksum(sums, filePath):
def getInternalSum( filePath ):
filePath = k_filebase + "and_wordlists/" + filePath
proc = subprocess.Popen(['/usr/bin/perl',
'--',
'/var/www/xw4/dawg2dict.pl',
'-get-sum',
'-dict', filePath ],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
return proc.communicate()[0].strip()
def md5Checksums( sums, filePath ):
if not filePath.endswith(k_suffix): filePath += k_suffix
if not filePath in sums:
logging.debug( "opening %s" % (k_filebase + "and_wordlists/" + filePath))
file = open( k_filebase + "and_wordlists/" + filePath, 'rb' )
md5 = hashlib.md5()
while True:
buffer = file.read(128)
if not buffer: break
md5.update( buffer )
sums[filePath] = md5.hexdigest()
sums[filePath] = [ md5.hexdigest(), getInternalSum( filePath ) ]
logging.debug( "figured sum for %s" % filePath )
return sums[filePath]
def getDictSums():
global s_shelf
s_shelf = shelve.open(k_shelfFile)
# shelve will fail if permissions are wrong. That's ok for some
# testing: just make a fake shelf and test before saving it later.
try:
s_shelf = shelve.open(k_shelfFile)
except:
s_shelf = {}
if not k_SUMS in s_shelf: s_shelf[k_SUMS] = {}
if not k_COUNT in s_shelf: s_shelf[k_COUNT] = 0
s_shelf[k_COUNT] += 1
@ -115,16 +136,16 @@ def dictVersion( req, name, lang, md5sum ):
dictSums = getDictSums()
path = lang + "/" + name
if not path in dictSums:
sum = md5Checksum( dictSums, path )
if sum:
dictSums[path] = sum
sums = md5Checksums( dictSums, path )
if sums:
dictSums[path] = sums
s_shelf[k_SUMS] = dictSums
if path in dictSums:
if dictSums[path] != md5sum:
if not md5sum in dictSums[path]:
result[k_URL] = k_urlbase + "and_wordlists/" + path
else:
logging.debug( path + " not known" )
s_shelf.close()
if 'close' in s_shelf: s_shelf.close()
return json.dumps( result )
def getApp( params ):
@ -137,6 +158,7 @@ def getApp( params ):
gvers = params[k_GVERS]
if k_INSTALLER in params: installer = params[k_INSTALLER]
else: installer = ''
logging.debug( "name: %s; avers: %s; installer: %s; gvers: %s"
% (name, avers, installer, gvers) )
if k_DEVOK in params and params[k_DEVOK]: versions = k_versions_dbg
@ -166,18 +188,19 @@ def getDicts( params ):
if not name.endswith(k_suffix): name += k_suffix
path = lang + "/" + name
if not path in dictSums:
sum = md5Checksum( dictSums, path )
if sum:
dictSums[path] = sum
sums = md5Checksums( dictSums, path )
if sums:
dictSums[path] = sums
s_shelf[k_SUMS] = dictSums
if path in dictSums:
if dictSums[path] != md5sum:
if not md5sum in dictSums[path]:
cur = { k_URL : k_urlbase + "and_wordlists/" + path,
k_INDEX : index }
k_INDEX : index, k_ISUM: dictSums[path][1] }
result.append( cur )
else:
logging.debug( path + " not known" )
if 'close' in s_shelf: s_shelf.close()
if 0 == len(result): result = None
return result
@ -201,7 +224,8 @@ def clearShelf():
def usage():
print "usage:", sys.argv[0], '--get-sums [lang/dict]*'
print " | --test-get-app app <org.eehouse.app.name> avers gvers"
print ' | --test-get-app app <org.eehouse.app.name> avers gvers'
print ' | --test-get-dicts name lang curSum'
print ' | --clear-shelf'
sys.exit(-1)
@ -213,9 +237,9 @@ def main():
elif arg == '--get-sums':
dictSums = getDictSums()
for arg in sys.argv[2:]:
print arg, md5Checksum(dictSums, arg)
print arg, md5Checksums(dictSums, arg)
s_shelf[k_SUMS] = dictSums
s_shelf.close()
if 'close' in s_shelf: s_shelf.close()
elif arg == '--test-get-app':
if not 5 == len(sys.argv): usage()
params = { k_NAME: sys.argv[2],
@ -223,6 +247,14 @@ def main():
k_GVERS: sys.argv[4],
}
print getApp( params )
elif arg == '--test-get-dicts':
if not 5 == len(sys.argv): usage()
params = { k_NAME: sys.argv[2],
k_LANG : sys.argv[3],
k_MD5SUM : sys.argv[4],
k_INDEX : 0,
}
print getDicts( [params] )
else:
usage()

View file

@ -1,4 +1,4 @@
#!/usr/bin/perl -CS
#!/usr/bin/perl -C
#
# Copyright 2004 - 2009 by Eric House (xwords@eehouse.org)
#
@ -24,10 +24,11 @@ use strict;
use Fcntl;
use Encode 'from_to';
use Encode;
use Digest::MD5;
my $gInFile;
my $gSumOnly = 0;
my $gSum;
my $gSum = '';
my $gDescOnly = 0;
my $gDesc;
my $gDoRaw = 0;
@ -95,24 +96,37 @@ sub readXWDFaces($$$) {
$nChars = unpack( 'c', $buf );
printf STDERR "nChars of faces: %d\n", $nChars;
binmode( $fh, ":encoding(utf8)" ) or die "binmode(:utf-8) failed\n";
sysread( $fh, $buf, $nChars );
length($buf) == $nChars or die "didn't read expected number of bytes\n";
binmode( $fh ) or die "binmode failed\n";
# At this point $fh is pointing at the start of data
if ( $gSumOnly ) {
my $sum = sumRestOfFile( $fh );
if ( $sum eq $gSum ) {
print STDERR "got: $sum, $gSum twice!!!\n";
} elsif ( !$gSum ) {
$gSum = $sum;
} else {
print STDERR "disagreement: $sum vs $gSum\n";
exit( -1 );
}
} else {
binmode( $fh, ":encoding(utf8)" ) or die "binmode(:utf-8) failed\n";
sysread( $fh, $buf, $nChars );
length($buf) == $nChars or die "didn't read expected number of bytes\n";
binmode( $fh ) or die "binmode failed\n";
print STDERR "string now: $buf\n";
my @faces;
for ( my $ii = 0; $ii < $nChars; ++$ii ) {
my $chr = substr( $buf, $ii, 1 );
print STDERR "pushing $chr \n";
push( @faces, $chr );
print STDERR "string now: $buf\n";
my @faces;
for ( my $ii = 0; $ii < $nChars; ++$ii ) {
my $chr = substr( $buf, $ii, 1 );
print STDERR "pushing $chr \n";
push( @faces, $chr );
}
printf STDERR "at 0x%x after reading faces\n", systell($fh);
${$nSpecials} = countSpecials( \@faces );
@{$facRef} = @faces;
printf STDERR "readXWDFaces=>%d\n", $nChars;
}
printf STDERR "at 0x%x after reading faces\n", systell($fh);
${$nSpecials} = countSpecials( \@faces );
@{$facRef} = @faces;
printf STDERR "readXWDFaces=>%d\n", $nChars;
return $nChars;
} # readXWDFaces
@ -170,7 +184,12 @@ sub printHeader($$) {
my ( $buf, $len ) = @_;
printf STDERR "skipped %d bytes of header:\n", $len + 2;
my $count;
($count, $gDesc, $gSum) = unpack( 'N Z* Z*', $buf );
if ( $len == 4 ) {
($count) = unpack( 'N', $buf );
} else {
print STDERR 'unpacking...\n';
($count, $gDesc, $gSum) = unpack( 'N Z* Z*', $buf );
}
printf STDERR "has %d words\n", $count;
}
@ -190,7 +209,7 @@ sub nodeSizeFromFlags($$) {
if ( $flags == 2 || $ flags == 4 ) {
return 3;
} elsif ( $flags == 5 ) {
} elsif ( $flags == 3 || $flags == 5 ) {
return 4;
} else {
die "invalid dict flags $flags";
@ -208,6 +227,21 @@ sub mergeSpecials($$) {
}
}
sub sumRestOfFile($) {
my ( $fh ) = @_;
my $buf;
my $count = 0;
my $md5 = Digest::MD5->new;
for ( ; ; ) {
my $nRead = sysread( $fh, $buf, 128 );
0 == $nRead && last;
$count += $nRead;
$md5->add( $buf );
}
# print STDERR "read $count bytes\n";
return $md5->hexdigest();
}
sub prepXWD($$$$) {
my ( $fh, $facRef, $nodesRef, $startRef ) = @_;
my $done = 1;
@ -219,18 +253,14 @@ sub prepXWD($$$$) {
$gNodeSize = nodeSizeFromFlags( $fh, $flags );
my $nSpecials;
my $faceCount = readXWDFaces( $fh, $facRef, \$nSpecials ); # does checksum
if ( $gSumOnly ) {
print STDOUT $gSum, "\n";
} elsif( $gDescOnly ) {
print STDOUT $gDesc, "\n";
} else {
$done = 0;
}
if ( !$done ) {
my $nSpecials;
my $faceCount = readXWDFaces( $fh, $facRef, \$nSpecials );
printf STDERR "at 0x%x before header read\n", systell($fh);
# skip xloc header
$nRead = sysread( $fh, $buf, 2 );