mirror of
git://xwords.git.sourceforge.net/gitroot/xwords/xwords
synced 2025-01-29 08:34:37 +01:00
Merge branch 'android_branch' into android_dictdb
This commit is contained in:
commit
1e12464934
2 changed files with 110 additions and 48 deletions
|
@ -1,6 +1,7 @@
|
|||
#!/usr/bin/python
|
||||
# Script meant to be installed on eehouse.org.
|
||||
|
||||
import logging, shelve, hashlib, sys, json
|
||||
import logging, shelve, hashlib, sys, json, subprocess
|
||||
try:
|
||||
from mod_python import apache
|
||||
apacheAvailable = True
|
||||
|
@ -18,6 +19,7 @@ k_DICTS = 'dicts'
|
|||
k_LANG = 'lang'
|
||||
k_MD5SUM = 'md5sum'
|
||||
k_INDEX = 'index'
|
||||
k_ISUM = 'isum'
|
||||
k_SUCCESS = 'success'
|
||||
k_URL = 'url'
|
||||
|
||||
|
@ -26,7 +28,7 @@ k_COUNT = 'count'
|
|||
|
||||
k_suffix = '.xwd'
|
||||
k_filebase = "/var/www/"
|
||||
k_shelfFile = k_filebase + 'xw4/info_shelf'
|
||||
k_shelfFile = k_filebase + 'xw4/info_shelf_2'
|
||||
k_urlbase = "http://eehouse.org/"
|
||||
k_versions = { 'org.eehouse.android.xw4': {
|
||||
'version' : 43,
|
||||
|
@ -49,8 +51,8 @@ k_versions_dbg = { 'org.eehouse.android.xw4': {
|
|||
'org.eehouse.android.xw4sms' : {
|
||||
'version' : 43,
|
||||
k_AVERS : 43,
|
||||
k_GVERS : 'android_beta_51',
|
||||
k_URL : 'xw4/android/sms/XWords4-release_android_beta_51.apk',
|
||||
k_GVERS : 'android_beta_51-50-g0ccc233',
|
||||
k_URL : 'xw4/android/sms/XWords4-release_android_beta_51-50-g0ccc233.apk',
|
||||
},
|
||||
}
|
||||
s_shelf = None
|
||||
|
@ -63,24 +65,43 @@ logging.basicConfig(level=logging.DEBUG
|
|||
# ,filemode='w')
|
||||
|
||||
# This seems to be required to prime the pump somehow.
|
||||
logging.debug( "loaded...." )
|
||||
# logging.debug( "loaded...." )
|
||||
|
||||
def md5Checksum(sums, filePath):
|
||||
def getInternalSum( filePath ):
|
||||
filePath = k_filebase + "and_wordlists/" + filePath
|
||||
proc = subprocess.Popen(['/usr/bin/perl',
|
||||
'--',
|
||||
'/var/www/xw4/dawg2dict.pl',
|
||||
'-get-sum',
|
||||
'-dict', filePath ],
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.PIPE)
|
||||
return proc.communicate()[0].strip()
|
||||
|
||||
def md5Checksums( sums, filePath ):
|
||||
if not filePath.endswith(k_suffix): filePath += k_suffix
|
||||
if not filePath in sums:
|
||||
logging.debug( "opening %s" % (k_filebase + "and_wordlists/" + filePath))
|
||||
file = open( k_filebase + "and_wordlists/" + filePath, 'rb' )
|
||||
md5 = hashlib.md5()
|
||||
while True:
|
||||
buffer = file.read(128)
|
||||
if not buffer: break
|
||||
md5.update( buffer )
|
||||
sums[filePath] = md5.hexdigest()
|
||||
|
||||
sums[filePath] = [ md5.hexdigest(), getInternalSum( filePath ) ]
|
||||
logging.debug( "figured sum for %s" % filePath )
|
||||
return sums[filePath]
|
||||
|
||||
def getDictSums():
|
||||
global s_shelf
|
||||
s_shelf = shelve.open(k_shelfFile)
|
||||
# shelve will fail if permissions are wrong. That's ok for some
|
||||
# testing: just make a fake shelf and test before saving it later.
|
||||
try:
|
||||
s_shelf = shelve.open(k_shelfFile)
|
||||
except:
|
||||
s_shelf = {}
|
||||
|
||||
if not k_SUMS in s_shelf: s_shelf[k_SUMS] = {}
|
||||
if not k_COUNT in s_shelf: s_shelf[k_COUNT] = 0
|
||||
s_shelf[k_COUNT] += 1
|
||||
|
@ -115,16 +136,16 @@ def dictVersion( req, name, lang, md5sum ):
|
|||
dictSums = getDictSums()
|
||||
path = lang + "/" + name
|
||||
if not path in dictSums:
|
||||
sum = md5Checksum( dictSums, path )
|
||||
if sum:
|
||||
dictSums[path] = sum
|
||||
sums = md5Checksums( dictSums, path )
|
||||
if sums:
|
||||
dictSums[path] = sums
|
||||
s_shelf[k_SUMS] = dictSums
|
||||
if path in dictSums:
|
||||
if dictSums[path] != md5sum:
|
||||
if not md5sum in dictSums[path]:
|
||||
result[k_URL] = k_urlbase + "and_wordlists/" + path
|
||||
else:
|
||||
logging.debug( path + " not known" )
|
||||
s_shelf.close()
|
||||
if 'close' in s_shelf: s_shelf.close()
|
||||
return json.dumps( result )
|
||||
|
||||
def getApp( params ):
|
||||
|
@ -137,6 +158,7 @@ def getApp( params ):
|
|||
gvers = params[k_GVERS]
|
||||
if k_INSTALLER in params: installer = params[k_INSTALLER]
|
||||
else: installer = ''
|
||||
|
||||
logging.debug( "name: %s; avers: %s; installer: %s; gvers: %s"
|
||||
% (name, avers, installer, gvers) )
|
||||
if k_DEVOK in params and params[k_DEVOK]: versions = k_versions_dbg
|
||||
|
@ -166,18 +188,19 @@ def getDicts( params ):
|
|||
if not name.endswith(k_suffix): name += k_suffix
|
||||
path = lang + "/" + name
|
||||
if not path in dictSums:
|
||||
sum = md5Checksum( dictSums, path )
|
||||
if sum:
|
||||
dictSums[path] = sum
|
||||
sums = md5Checksums( dictSums, path )
|
||||
if sums:
|
||||
dictSums[path] = sums
|
||||
s_shelf[k_SUMS] = dictSums
|
||||
if path in dictSums:
|
||||
if dictSums[path] != md5sum:
|
||||
if not md5sum in dictSums[path]:
|
||||
cur = { k_URL : k_urlbase + "and_wordlists/" + path,
|
||||
k_INDEX : index }
|
||||
k_INDEX : index, k_ISUM: dictSums[path][1] }
|
||||
result.append( cur )
|
||||
else:
|
||||
logging.debug( path + " not known" )
|
||||
|
||||
if 'close' in s_shelf: s_shelf.close()
|
||||
if 0 == len(result): result = None
|
||||
return result
|
||||
|
||||
|
@ -201,7 +224,8 @@ def clearShelf():
|
|||
|
||||
def usage():
|
||||
print "usage:", sys.argv[0], '--get-sums [lang/dict]*'
|
||||
print " | --test-get-app app <org.eehouse.app.name> avers gvers"
|
||||
print ' | --test-get-app app <org.eehouse.app.name> avers gvers'
|
||||
print ' | --test-get-dicts name lang curSum'
|
||||
print ' | --clear-shelf'
|
||||
sys.exit(-1)
|
||||
|
||||
|
@ -213,9 +237,9 @@ def main():
|
|||
elif arg == '--get-sums':
|
||||
dictSums = getDictSums()
|
||||
for arg in sys.argv[2:]:
|
||||
print arg, md5Checksum(dictSums, arg)
|
||||
print arg, md5Checksums(dictSums, arg)
|
||||
s_shelf[k_SUMS] = dictSums
|
||||
s_shelf.close()
|
||||
if 'close' in s_shelf: s_shelf.close()
|
||||
elif arg == '--test-get-app':
|
||||
if not 5 == len(sys.argv): usage()
|
||||
params = { k_NAME: sys.argv[2],
|
||||
|
@ -223,6 +247,14 @@ def main():
|
|||
k_GVERS: sys.argv[4],
|
||||
}
|
||||
print getApp( params )
|
||||
elif arg == '--test-get-dicts':
|
||||
if not 5 == len(sys.argv): usage()
|
||||
params = { k_NAME: sys.argv[2],
|
||||
k_LANG : sys.argv[3],
|
||||
k_MD5SUM : sys.argv[4],
|
||||
k_INDEX : 0,
|
||||
}
|
||||
print getDicts( [params] )
|
||||
else:
|
||||
usage()
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/perl -CS
|
||||
#!/usr/bin/perl -C
|
||||
#
|
||||
# Copyright 2004 - 2009 by Eric House (xwords@eehouse.org)
|
||||
#
|
||||
|
@ -24,10 +24,11 @@ use strict;
|
|||
use Fcntl;
|
||||
use Encode 'from_to';
|
||||
use Encode;
|
||||
use Digest::MD5;
|
||||
|
||||
my $gInFile;
|
||||
my $gSumOnly = 0;
|
||||
my $gSum;
|
||||
my $gSum = '';
|
||||
my $gDescOnly = 0;
|
||||
my $gDesc;
|
||||
my $gDoRaw = 0;
|
||||
|
@ -95,24 +96,37 @@ sub readXWDFaces($$$) {
|
|||
$nChars = unpack( 'c', $buf );
|
||||
printf STDERR "nChars of faces: %d\n", $nChars;
|
||||
|
||||
binmode( $fh, ":encoding(utf8)" ) or die "binmode(:utf-8) failed\n";
|
||||
sysread( $fh, $buf, $nChars );
|
||||
length($buf) == $nChars or die "didn't read expected number of bytes\n";
|
||||
binmode( $fh ) or die "binmode failed\n";
|
||||
# At this point $fh is pointing at the start of data
|
||||
if ( $gSumOnly ) {
|
||||
my $sum = sumRestOfFile( $fh );
|
||||
if ( $sum eq $gSum ) {
|
||||
print STDERR "got: $sum, $gSum twice!!!\n";
|
||||
} elsif ( !$gSum ) {
|
||||
$gSum = $sum;
|
||||
} else {
|
||||
print STDERR "disagreement: $sum vs $gSum\n";
|
||||
exit( -1 );
|
||||
}
|
||||
} else {
|
||||
binmode( $fh, ":encoding(utf8)" ) or die "binmode(:utf-8) failed\n";
|
||||
sysread( $fh, $buf, $nChars );
|
||||
length($buf) == $nChars or die "didn't read expected number of bytes\n";
|
||||
binmode( $fh ) or die "binmode failed\n";
|
||||
|
||||
print STDERR "string now: $buf\n";
|
||||
my @faces;
|
||||
for ( my $ii = 0; $ii < $nChars; ++$ii ) {
|
||||
my $chr = substr( $buf, $ii, 1 );
|
||||
print STDERR "pushing $chr \n";
|
||||
push( @faces, $chr );
|
||||
print STDERR "string now: $buf\n";
|
||||
my @faces;
|
||||
for ( my $ii = 0; $ii < $nChars; ++$ii ) {
|
||||
my $chr = substr( $buf, $ii, 1 );
|
||||
print STDERR "pushing $chr \n";
|
||||
push( @faces, $chr );
|
||||
}
|
||||
|
||||
printf STDERR "at 0x%x after reading faces\n", systell($fh);
|
||||
|
||||
${$nSpecials} = countSpecials( \@faces );
|
||||
@{$facRef} = @faces;
|
||||
printf STDERR "readXWDFaces=>%d\n", $nChars;
|
||||
}
|
||||
|
||||
printf STDERR "at 0x%x after reading faces\n", systell($fh);
|
||||
|
||||
${$nSpecials} = countSpecials( \@faces );
|
||||
@{$facRef} = @faces;
|
||||
printf STDERR "readXWDFaces=>%d\n", $nChars;
|
||||
return $nChars;
|
||||
} # readXWDFaces
|
||||
|
||||
|
@ -170,7 +184,12 @@ sub printHeader($$) {
|
|||
my ( $buf, $len ) = @_;
|
||||
printf STDERR "skipped %d bytes of header:\n", $len + 2;
|
||||
my $count;
|
||||
($count, $gDesc, $gSum) = unpack( 'N Z* Z*', $buf );
|
||||
if ( $len == 4 ) {
|
||||
($count) = unpack( 'N', $buf );
|
||||
} else {
|
||||
print STDERR 'unpacking...\n';
|
||||
($count, $gDesc, $gSum) = unpack( 'N Z* Z*', $buf );
|
||||
}
|
||||
printf STDERR "has %d words\n", $count;
|
||||
}
|
||||
|
||||
|
@ -190,7 +209,7 @@ sub nodeSizeFromFlags($$) {
|
|||
|
||||
if ( $flags == 2 || $ flags == 4 ) {
|
||||
return 3;
|
||||
} elsif ( $flags == 5 ) {
|
||||
} elsif ( $flags == 3 || $flags == 5 ) {
|
||||
return 4;
|
||||
} else {
|
||||
die "invalid dict flags $flags";
|
||||
|
@ -208,6 +227,21 @@ sub mergeSpecials($$) {
|
|||
}
|
||||
}
|
||||
|
||||
sub sumRestOfFile($) {
|
||||
my ( $fh ) = @_;
|
||||
my $buf;
|
||||
my $count = 0;
|
||||
my $md5 = Digest::MD5->new;
|
||||
for ( ; ; ) {
|
||||
my $nRead = sysread( $fh, $buf, 128 );
|
||||
0 == $nRead && last;
|
||||
$count += $nRead;
|
||||
$md5->add( $buf );
|
||||
}
|
||||
# print STDERR "read $count bytes\n";
|
||||
return $md5->hexdigest();
|
||||
}
|
||||
|
||||
sub prepXWD($$$$) {
|
||||
my ( $fh, $facRef, $nodesRef, $startRef ) = @_;
|
||||
my $done = 1;
|
||||
|
@ -219,18 +253,14 @@ sub prepXWD($$$$) {
|
|||
|
||||
$gNodeSize = nodeSizeFromFlags( $fh, $flags );
|
||||
|
||||
my $nSpecials;
|
||||
my $faceCount = readXWDFaces( $fh, $facRef, \$nSpecials ); # does checksum
|
||||
|
||||
if ( $gSumOnly ) {
|
||||
print STDOUT $gSum, "\n";
|
||||
} elsif( $gDescOnly ) {
|
||||
print STDOUT $gDesc, "\n";
|
||||
} else {
|
||||
$done = 0;
|
||||
}
|
||||
|
||||
if ( !$done ) {
|
||||
my $nSpecials;
|
||||
my $faceCount = readXWDFaces( $fh, $facRef, \$nSpecials );
|
||||
|
||||
printf STDERR "at 0x%x before header read\n", systell($fh);
|
||||
# skip xloc header
|
||||
$nRead = sysread( $fh, $buf, 2 );
|
||||
|
|
Loading…
Add table
Reference in a new issue