Miroirs/noDRM_DeDRM_tools
1
0
Fork 0
mirror of https://github.com/noDRM/DeDRM_tools synced 2024-12-27 21:59:11 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

topaz tools 1.0 (I think)

Browse source
This commit is contained in:
some_updates 2010-01-17 12:10:35 +00:00 committed by Apprentice Alf
parent 1fc40376cf
commit 0a437510f6
9 changed files with 2931 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

865
Topaz_Tools/lib/cmbtc_dump.py Normal file
View file

@ -0,0 +1,865 @@
#! /usr/bin/python
"""
Comprehensive Mazama Book DRM with Topaz Cryptography V2.0
-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDdBHJ4CNc6DNFCw4MRCw4SWAK6
M8hYfnNEI0yQmn5Ti+W8biT7EatpauE/5jgQMPBmdNrDr1hbHyHBSP7xeC2qlRWC
B62UCxeu/fpfnvNHDN/wPWWH4jynZ2M6cdcnE5LQ+FfeKqZn7gnG2No1U9h7oOHx
y2/pHuYme7U1TsgSjwIDAQAB
-----END PUBLIC KEY-----
"""
from __future__ import with_statement
import csv
import sys
import os
import getopt
import zlib
from struct import pack
from struct import unpack
from ctypes import windll, c_char_p, c_wchar_p, c_uint, POINTER, byref, \
create_unicode_buffer, create_string_buffer, CFUNCTYPE, addressof, \
string_at, Structure, c_void_p, cast
import _winreg as winreg
import Tkinter
import Tkconstants
import tkMessageBox
import traceback
import hashlib
MAX_PATH = 255
kernel32 = windll.kernel32
advapi32 = windll.advapi32
crypt32 = windll.crypt32
global kindleDatabase
global bookFile
global bookPayloadOffset
global bookHeaderRecords
global bookMetadata
global bookKey
global command
#
# Various character maps used to decrypt books. Probably supposed to act as obfuscation
#
charMap1 = "n5Pr6St7Uv8Wx9YzAb0Cd1Ef2Gh3Jk4M"
charMap2 = "AaZzB0bYyCc1XxDdW2wEeVv3FfUuG4g-TtHh5SsIiR6rJjQq7KkPpL8lOoMm9Nn_"
charMap3 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
charMap4 = "ABCDEFGHIJKLMNPQRSTUVWXYZ123456789"
#
# Exceptions for all the problems that might happen during the script
#
class CMBDTCError(Exception):
pass
class CMBDTCFatal(Exception):
pass
#
# Stolen stuff
#
class DataBlob(Structure):
_fields_ = [('cbData', c_uint),
('pbData', c_void_p)]
DataBlob_p = POINTER(DataBlob)
def GetSystemDirectory():
GetSystemDirectoryW = kernel32.GetSystemDirectoryW
GetSystemDirectoryW.argtypes = [c_wchar_p, c_uint]
GetSystemDirectoryW.restype = c_uint
def GetSystemDirectory():
buffer = create_unicode_buffer(MAX_PATH + 1)
GetSystemDirectoryW(buffer, len(buffer))
return buffer.value
return GetSystemDirectory
GetSystemDirectory = GetSystemDirectory()
def GetVolumeSerialNumber():
GetVolumeInformationW = kernel32.GetVolumeInformationW
GetVolumeInformationW.argtypes = [c_wchar_p, c_wchar_p, c_uint,
POINTER(c_uint), POINTER(c_uint),
POINTER(c_uint), c_wchar_p, c_uint]
GetVolumeInformationW.restype = c_uint
def GetVolumeSerialNumber(path):
vsn = c_uint(0)
GetVolumeInformationW(path, None, 0, byref(vsn), None, None, None, 0)
return vsn.value
return GetVolumeSerialNumber
GetVolumeSerialNumber = GetVolumeSerialNumber()
def GetUserName():
GetUserNameW = advapi32.GetUserNameW
GetUserNameW.argtypes = [c_wchar_p, POINTER(c_uint)]
GetUserNameW.restype = c_uint
def GetUserName():
buffer = create_unicode_buffer(32)
size = c_uint(len(buffer))
while not GetUserNameW(buffer, byref(size)):
buffer = create_unicode_buffer(len(buffer) * 2)
size.value = len(buffer)
return buffer.value.encode('utf-16-le')[::2]
return GetUserName
GetUserName = GetUserName()
def CryptUnprotectData():
_CryptUnprotectData = crypt32.CryptUnprotectData
_CryptUnprotectData.argtypes = [DataBlob_p, c_wchar_p, DataBlob_p,
c_void_p, c_void_p, c_uint, DataBlob_p]
_CryptUnprotectData.restype = c_uint
def CryptUnprotectData(indata, entropy):
indatab = create_string_buffer(indata)
indata = DataBlob(len(indata), cast(indatab, c_void_p))
entropyb = create_string_buffer(entropy)
entropy = DataBlob(len(entropy), cast(entropyb, c_void_p))
outdata = DataBlob()
if not _CryptUnprotectData(byref(indata), None, byref(entropy),
None, None, 0, byref(outdata)):
raise CMBDTCFatal("Failed to Unprotect Data")
return string_at(outdata.pbData, outdata.cbData)
return CryptUnprotectData
CryptUnprotectData = CryptUnprotectData()
#
# Returns the MD5 digest of "message"
#
def MD5(message):
ctx = hashlib.md5()
ctx.update(message)
return ctx.digest()
#
# Returns the MD5 digest of "message"
#
def SHA1(message):
ctx = hashlib.sha1()
ctx.update(message)
return ctx.digest()
#
# Open the book file at path
#
def openBook(path):
try:
return open(path,'rb')
except:
raise CMBDTCFatal("Could not open book file: " + path)
#
# Encode the bytes in data with the characters in map
#
def encode(data, map):
result = ""
for char in data:
value = ord(char)
Q = (value ^ 0x80) // len(map)
R = value % len(map)
result += map[Q]
result += map[R]
return result
#
# Hash the bytes in data and then encode the digest with the characters in map
#
def encodeHash(data,map):
return encode(MD5(data),map)
#
# Decode the string in data with the characters in map. Returns the decoded bytes
#
def decode(data,map):
result = ""
for i in range (0,len(data),2):
high = map.find(data[i])
low = map.find(data[i+1])
value = (((high * 0x40) ^ 0x80) & 0xFF) + low
result += pack("B",value)
return result
#
# Locate and open the Kindle.info file (Hopefully in the way it is done in the Kindle application)
#
def openKindleInfo():
regkey = winreg.OpenKey(winreg.HKEY_CURRENT_USER, "Software\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Shell Folders\\")
path = winreg.QueryValueEx(regkey, 'Local AppData')[0]
return open(path+'\\Amazon\\Kindle For PC\\{AMAwzsaPaaZAzmZzZQzgZCAkZ3AjA_AY}\\kindle.info','r')
#
# Parse the Kindle.info file and return the records as a list of key-values
#
def parseKindleInfo():
DB = {}
infoReader = openKindleInfo()
infoReader.read(1)
data = infoReader.read()
items = data.split('{')
for item in items:
splito = item.split(':')
DB[splito[0]] =splito[1]
return DB
#
# Find if the original string for a hashed/encoded string is known. If so return the original string othwise return an empty string. (Totally not optimal)
#
def findNameForHash(hash):
names = ["kindle.account.tokens","kindle.cookie.item","eulaVersionAccepted","login_date","kindle.token.item","login","kindle.key.item","kindle.name.info","kindle.device.info", "MazamaRandomNumber"]
result = ""
for name in names:
if hash == encodeHash(name, charMap2):
result = name
break
return name
#
# Print all the records from the kindle.info file (option -i)
#
def printKindleInfo():
for record in kindleDatabase:
name = findNameForHash(record)
if name != "" :
print (name)
print ("--------------------------\n")
else :
print ("Unknown Record")
print getKindleInfoValueForHash(record)
print "\n"
#
# Get a record from the Kindle.info file for the key "hashedKey" (already hashed and encoded). Return the decoded and decrypted record
#
def getKindleInfoValueForHash(hashedKey):
global kindleDatabase
encryptedValue = decode(kindleDatabase[hashedKey],charMap2)
return CryptUnprotectData(encryptedValue,"")
#
# Get a record from the Kindle.info file for the string in "key" (plaintext). Return the decoded and decrypted record
#
def getKindleInfoValueForKey(key):
return getKindleInfoValueForHash(encodeHash(key,charMap2))
#
# Get a 7 bit encoded number from the book file
#
def bookReadEncodedNumber():
flag = False
data = ord(bookFile.read(1))
if data == 0xFF:
flag = True
data = ord(bookFile.read(1))
if data >= 0x80:
datax = (data & 0x7F)
while data >= 0x80 :
data = ord(bookFile.read(1))
datax = (datax <<7) + (data & 0x7F)
data = datax
if flag:
data = -data
return data
#
# Encode a number in 7 bit format
#
def encodeNumber(number):
result = ""
negative = False
flag = 0
if number < 0 :
number = -number + 1
negative = True
while True:
byte = number & 0x7F
number = number >> 7
byte += flag
result += chr(byte)
flag = 0x80
if number == 0 : break
if negative:
result += chr(0xFF)
return result[::-1]
#
# Get a length prefixed string from the file
#
def bookReadString():
stringLength = bookReadEncodedNumber()
return unpack(str(stringLength)+"s",bookFile.read(stringLength))[0]
#
# Returns a length prefixed string
#
def lengthPrefixString(data):
return encodeNumber(len(data))+data
#
# Read and return the data of one header record at the current book file position [[offset,decompressedLength,compressedLength],...]
#
def bookReadHeaderRecordData():
nbValues = bookReadEncodedNumber()
values = []
for i in range (0,nbValues):
values.append([bookReadEncodedNumber(),bookReadEncodedNumber(),bookReadEncodedNumber()])
return values
#
# Read and parse one header record at the current book file position and return the associated data [[offset,decompressedLength,compressedLength],...]
#
def parseTopazHeaderRecord():
if ord(bookFile.read(1)) != 0x63:
raise CMBDTCFatal("Parse Error : Invalid Header")
tag = bookReadString()
record = bookReadHeaderRecordData()
return [tag,record]
#
# Parse the header of a Topaz file, get all the header records and the offset for the payload
#
def parseTopazHeader():
global bookHeaderRecords
global bookPayloadOffset
magic = unpack("4s",bookFile.read(4))[0]
if magic != 'TPZ0':
raise CMBDTCFatal("Parse Error : Invalid Header, not a Topaz file")
nbRecords = bookReadEncodedNumber()
bookHeaderRecords = {}
for i in range (0,nbRecords):
result = parseTopazHeaderRecord()
print result[0], result[1]
bookHeaderRecords[result[0]] = result[1]
if ord(bookFile.read(1)) != 0x64 :
raise CMBDTCFatal("Parse Error : Invalid Header")
bookPayloadOffset = bookFile.tell()
#
# Get a record in the book payload, given its name and index. If necessary the record is decrypted. The record is not decompressed
# Correction, the record is correctly decompressed too
#
def getBookPayloadRecord(name, index):
encrypted = False
compressed = False
try:
recordOffset = bookHeaderRecords[name][index][0]
except:
raise CMBDTCFatal("Parse Error : Invalid Record, record not found")
bookFile.seek(bookPayloadOffset + recordOffset)
tag = bookReadString()
if tag != name :
raise CMBDTCFatal("Parse Error : Invalid Record, record name doesn't match")
recordIndex = bookReadEncodedNumber()
if recordIndex < 0 :
encrypted = True
recordIndex = -recordIndex -1
if recordIndex != index :
raise CMBDTCFatal("Parse Error : Invalid Record, index doesn't match")
if (bookHeaderRecords[name][index][2] > 0):
compressed = True
record = bookFile.read(bookHeaderRecords[name][index][2])
else:
record = bookFile.read(bookHeaderRecords[name][index][1])
if encrypted:
ctx = topazCryptoInit(bookKey)
record = topazCryptoDecrypt(record,ctx)
if compressed:
record = zlib.decompress(record)
return record
#
# Extract, decrypt and decompress a book record indicated by name and index and print it or save it in "filename"
#
def extractBookPayloadRecord(name, index, filename):
compressed = False
try:
compressed = bookHeaderRecords[name][index][2] != 0
record = getBookPayloadRecord(name,index)
except:
print("Could not find record")
# if compressed:
# try:
# record = zlib.decompress(record)
# except:
# raise CMBDTCFatal("Could not decompress record")
if filename != "":
try:
file = open(filename,"wb")
file.write(record)
file.close()
except:
raise CMBDTCFatal("Could not write to destination file")
else:
print(record)
#
# return next record [key,value] from the book metadata from the current book position
#
def readMetadataRecord():
return [bookReadString(),bookReadString()]
#
# Parse the metadata record from the book payload and return a list of [key,values]
#
def parseMetadata():
global bookHeaderRecords
global bookPayloadAddress
global bookMetadata
bookMetadata = {}
bookFile.seek(bookPayloadOffset + bookHeaderRecords["metadata"][0][0])
tag = bookReadString()
if tag != "metadata" :
raise CMBDTCFatal("Parse Error : Record Names Don't Match")
flags = ord(bookFile.read(1))
nbRecords = ord(bookFile.read(1))
for i in range (0,nbRecords) :
record =readMetadataRecord()
bookMetadata[record[0]] = record[1]
#
# Returns two bit at offset from a bit field
#
def getTwoBitsFromBitField(bitField,offset):
byteNumber = offset // 4
bitPosition = 6 - 2*(offset % 4)
return ord(bitField[byteNumber]) >> bitPosition & 3
#
# Returns the six bits at offset from a bit field
#
def getSixBitsFromBitField(bitField,offset):
offset *= 3
value = (getTwoBitsFromBitField(bitField,offset) <<4) + (getTwoBitsFromBitField(bitField,offset+1) << 2) +getTwoBitsFromBitField(bitField,offset+2)
return value
#
# 8 bits to six bits encoding from hash to generate PID string
#
def encodePID(hash):
global charMap3
PID = ""
for position in range (0,8):
PID += charMap3[getSixBitsFromBitField(hash,position)]
return PID
#
# Context initialisation for the Topaz Crypto
#
def topazCryptoInit(key):
ctx1 = 0x0CAFFE19E
for keyChar in key:
keyByte = ord(keyChar)
ctx2 = ctx1
ctx1 = ((((ctx1 >>2) * (ctx1 >>7))&0xFFFFFFFF) ^ (keyByte * keyByte * 0x0F902007)& 0xFFFFFFFF )
return [ctx1,ctx2]
#
# decrypt data with the context prepared by topazCryptoInit()
#
def topazCryptoDecrypt(data, ctx):
ctx1 = ctx[0]
ctx2 = ctx[1]
plainText = ""
for dataChar in data:
dataByte = ord(dataChar)
m = (dataByte ^ ((ctx1 >> 3) &0xFF) ^ ((ctx2<<3) & 0xFF)) &0xFF
ctx2 = ctx1
ctx1 = (((ctx1 >> 2) * (ctx1 >> 7)) &0xFFFFFFFF) ^((m * m * 0x0F902007) &0xFFFFFFFF)
plainText += chr(m)
return plainText
#
# Decrypt a payload record with the PID
#
def decryptRecord(data,PID):
ctx = topazCryptoInit(PID)
return topazCryptoDecrypt(data, ctx)
#
# Try to decrypt a dkey record (contains the book PID)
#
def decryptDkeyRecord(data,PID):
record = decryptRecord(data,PID)
fields = unpack("3sB8sB8s3s",record)
if fields[0] != "PID" or fields[5] != "pid" :
raise CMBDTCError("Didn't find PID magic numbers in record")
elif fields[1] != 8 or fields[3] != 8 :
raise CMBDTCError("Record didn't contain correct length fields")
elif fields[2] != PID :
raise CMBDTCError("Record didn't contain PID")
return fields[4]
#
# Decrypt all the book's dkey records (contain the book PID)
#
def decryptDkeyRecords(data,PID):
nbKeyRecords = ord(data[0])
records = []
data = data[1:]
for i in range (0,nbKeyRecords):
length = ord(data[0])
try:
key = decryptDkeyRecord(data[1:length+1],PID)
records.append(key)
except CMBDTCError:
pass
data = data[1+length:]
return records
#
# Encryption table used to generate the device PID
#
def generatePidEncryptionTable() :
table = []
for counter1 in range (0,0x100):
value = counter1
for counter2 in range (0,8):
if (value & 1 == 0) :
value = value >> 1
else :
value = value >> 1
value = value ^ 0xEDB88320
table.append(value)
return table
#
# Seed value used to generate the device PID
#
def generatePidSeed(table,dsn) :
value = 0
for counter in range (0,4) :
index = (ord(dsn[counter]) ^ value) &0xFF
value = (value >> 8) ^ table[index]
return value
#
# Generate the device PID
#
def generateDevicePID(table,dsn,nbRoll):
seed = generatePidSeed(table,dsn)
pidAscii = ""
pid = [(seed >>24) &0xFF,(seed >> 16) &0xff,(seed >> 8) &0xFF,(seed) & 0xFF,(seed>>24) & 0xFF,(seed >> 16) &0xff,(seed >> 8) &0xFF,(seed) & 0xFF]
index = 0
for counter in range (0,nbRoll):
pid[index] = pid[index] ^ ord(dsn[counter])
index = (index+1) %8
for counter in range (0,8):
index = ((((pid[counter] >>5) & 3) ^ pid[counter]) & 0x1f) + (pid[counter] >> 7)
pidAscii += charMap4[index]
return pidAscii
#
# Create decrypted book payload
#
def createDecryptedPayload(payload):
for headerRecord in bookHeaderRecords:
name = headerRecord
if name != "dkey" :
ext = '.dat'
if name == 'img' : ext = '.jpg'
for index in range (0,len(bookHeaderRecords[name])) :
fnum = "%04d" % index
fname = name + fnum + ext
destdir = payload
if name == 'img':
destdir = os.path.join(payload,'img')
if name == 'page':
destdir = os.path.join(payload,'page')
if name == 'glyphs':
destdir = os.path.join(payload,'glyphs')
outputFile = os.path.join(destdir,fname)
file(outputFile, 'wb').write(getBookPayloadRecord(name, index))
# Create decrypted book
#
def createDecryptedBook(outdir):
if not os.path.exists(outdir):
os.makedirs(outdir)
destdir = os.path.join(outdir,'img')
if not os.path.exists(destdir):
os.makedirs(destdir)
destdir = os.path.join(outdir,'page')
if not os.path.exists(destdir):
os.makedirs(destdir)
destdir = os.path.join(outdir,'glyphs')
if not os.path.exists(destdir):
os.makedirs(destdir)
createDecryptedPayload(outdir)
#
# Set the command to execute by the programm according to cmdLine parameters
#
def setCommand(name) :
global command
if command != "" :
raise CMBDTCFatal("Invalid command line parameters")
else :
command = name
#
# Program usage
#
def usage():
print("\nUsage:")
print("\ncmbtc_dump.py [options] bookFileName\n")
print("-p Adds a PID to the list of PIDs that are tried to decrypt the book key (can be used several times)")
print("-d Dumps the unencrypted book as files to outdir")
print("-o Output directory to save book files to")
print("-v Verbose (can be used several times)")
print("-i Prints kindle.info database")
#
# Main
#
def main(argv=sys.argv):
global kindleDatabase
global bookMetadata
global bookKey
global bookFile
global command
progname = os.path.basename(argv[0])
verbose = 0
recordName = ""
recordIndex = 0
outdir = ""
PIDs = []
kindleDatabase = None
command = ""
try:
opts, args = getopt.getopt(sys.argv[1:], "vi:o:p:d")
except getopt.GetoptError, err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
if len(opts) == 0 and len(args) == 0 :
usage()
sys.exit(2)
for o, a in opts:
if o == "-v":
verbose+=1
if o == "-i":
setCommand("printInfo")
if o =="-o":
if a == None :
raise CMBDTCFatal("Invalid parameter for -o")
outdir = a
if o =="-p":
PIDs.append(a)
if o =="-d":
setCommand("doit")
if command == "" :
raise CMBDTCFatal("No action supplied on command line")
#
# Read the encrypted database
#
try:
kindleDatabase = parseKindleInfo()
except Exception as message:
if verbose>0:
print(message)
if kindleDatabase != None :
if command == "printInfo" :
printKindleInfo()
#
# Compute the DSN
#
# Get the Mazama Random number
MazamaRandomNumber = getKindleInfoValueForKey("MazamaRandomNumber")
# Get the HDD serial
encodedSystemVolumeSerialNumber = encodeHash(str(GetVolumeSerialNumber(GetSystemDirectory().split('\\')[0] + '\\')),charMap1)
# Get the current user name
encodedUsername = encodeHash(GetUserName(),charMap1)
# concat, hash and encode
DSN = encode(SHA1(MazamaRandomNumber+encodedSystemVolumeSerialNumber+encodedUsername),charMap1)
if verbose >1:
print("DSN: " + DSN)
#
# Compute the device PID
#
table = generatePidEncryptionTable()
devicePID = generateDevicePID(table,DSN,4)
PIDs.append(devicePID)
if verbose > 0:
print("Device PID: " + devicePID)
#
# Open book and parse metadata
#
if len(args) == 1:
bookFile = openBook(args[0])
parseTopazHeader()
parseMetadata()
#
# Compute book PID
#
# Get the account token
if kindleDatabase != None:
kindleAccountToken = getKindleInfoValueForKey("kindle.account.tokens")
if verbose >1:
print("Account Token: " + kindleAccountToken)
keysRecord = bookMetadata["keys"]
keysRecordRecord = bookMetadata[keysRecord]
pidHash = SHA1(DSN+kindleAccountToken+keysRecord+keysRecordRecord)
bookPID = encodePID(pidHash)
PIDs.append(bookPID)
if verbose > 0:
print ("Book PID: " + bookPID )
#
# Decrypt book key
#
dkey = getBookPayloadRecord('dkey', 0)
bookKeys = []
for PID in PIDs :
bookKeys+=decryptDkeyRecords(dkey,PID)
if len(bookKeys) == 0 :
if verbose > 0 :
print ("Book key could not be found. Maybe this book is not registered with this device.")
else :
bookKey = bookKeys[0]
if verbose > 0:
print("Book key: " + bookKey.encode('hex'))
if command == "printRecord" :
extractBookPayloadRecord(recordName,int(recordIndex),outputFile)
if outputFile != "" and verbose>0 :
print("Wrote record to file: "+outputFile)
elif command == "doit" :
if outdir != "" :
createDecryptedBook(outdir)
if verbose >0 :
print ("Decrypted book saved. Don't pirate!")
elif verbose > 0:
print("Output directory name was not supplied.")
return 0
if __name__ == '__main__':
sys.exit(main())

821
Topaz_Tools/lib/convert2xml.py Normal file
View file

@ -0,0 +1,821 @@
#! /usr/bin/python
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
from __future__ import with_statement
import csv
import sys
import os
import getopt
from struct import pack
from struct import unpack
# Get a 7 bit encoded number from string. The most
# significant byte comes first and has the high bit (8th) set
def readEncodedNumber(file):
flag = False
c = file.read(1)
if (len(c) == 0):
return None
data = ord(c)
if data == 0xFF:
flag = True
c = file.read(1)
if (len(c) == 0):
return None
data = ord(c)
if data >= 0x80:
datax = (data & 0x7F)
while data >= 0x80 :
c = file.read(1)
if (len(c) == 0):
return None
data = ord(c)
datax = (datax <<7) + (data & 0x7F)
data = datax
if flag:
data = -data
return data
# returns a binary string that encodes a number into 7 bits
# most significant byte first which has the high bit set
def encodeNumber(number):
result = ""
negative = False
flag = 0
if number < 0 :
number = -number + 1
negative = True
while True:
byte = number & 0x7F
number = number >> 7
byte += flag
result += chr(byte)
flag = 0x80
if number == 0 : break
if negative:
result += chr(0xFF)
return result[::-1]
# create / read a length prefixed string from the file
def lengthPrefixString(data):
return encodeNumber(len(data))+data
def readString(file):
stringLength = readEncodedNumber(file)
if (stringLength == None):
return ""
sv = file.read(stringLength)
if (len(sv) != stringLength):
return ""
return unpack(str(stringLength)+"s",sv)[0]
# convert a binary string generated by encodeNumber (7 bit encoded number)
# to the value you would find inside the page*.dat files to be processed
def convert(i):
result = ''
val = encodeNumber(i)
for j in xrange(len(val)):
c = ord(val[j:j+1])
result += '%02x' % c
return result
# the complete string table used to store all book text content
# as well as the xml tokens and values that make sense out of it
class Dictionary(object):
def __init__(self, dictFile):
self.filename = dictFile
self.size = 0
self.fo = file(dictFile,'rb')
self.stable = []
self.size = readEncodedNumber(self.fo)
for i in xrange(self.size):
self.stable.append(self.escapestr(readString(self.fo)))
self.pos = 0
def escapestr(self, str):
str = str.replace('&','&amp;')
str = str.replace('<','&lt;')
str = str.replace('>','&gt;')
str = str.replace('=','&#61;')
return str
def lookup(self,val):
if ((val >= 0) and (val < self.size)) :
self.pos = val
return self.stable[self.pos]
else:
print "Error - %d outside of string table limits" % val
sys.exit(-1)
def getSize(self):
return self.size
def getPos(self):
return self.pos
def dumpDict(self):
for i in xrange(self.size):
print "%d %s %s" % (i, convert(i), self.stable[i])
return
# parses the xml snippets that are represented by each page*.dat file.
# also parses the other0.dat file - the main stylesheet
# and information used to inject the xml snippets into page*.dat files
class PageParser(object):
def __init__(self, filename, dict, debug, flat_xml):
self.fo = file(filename,'rb')
self.id = os.path.basename(filename).replace('.dat','')
self.dict = dict
self.debug = debug
self.flat_xml = flat_xml
self.tagpath = []
self.doc = []
self.snippetList = []
# hash table used to enable the decoding process
# This has all been developed by trial and error so it may still have omissions or
# contain errors
# Format:
# tag : (number of arguments, argument type, subtags present, special case of subtags presents when escaped)
token_tags = {
'book' : (1, 'snippets', 1, 0),
'version' : (1, 'snippets', 1, 0),
'stylesheet' : (1, 'snippets', 1, 0),
'links' : (0, 'number', 0, 1),
'pages' : (0, 'number', 0, 1),
'page' : (1, 'snippets', 1, 0),
'group' : (1, 'snippets', 1, 0),
'region' : (1, 'snippets', 1, 0),
'reflow' : (1, 'number', 1, 0),
'img' : (1, 'snippets', 1, 0),
'paragraph' : (1, 'snippets', 1, 0),
'extratokens' : (1, 'snippets', 1, 0),
'style' : (1, 'snippets', 1, 0),
'rule' : (1, 'snippets', 1, 0),
'_span' : (1, 'snippets', 1, 0),
'word_semantic': (1, 'snippets', 1, 1),
'value' : (1, 'scalar_text', 0, 0),
'h' : (1, 'scalar_number', 0, 0),
'w' : (1, 'scalar_number', 0, 0),
'firstWord' : (1, 'scalar_number', 0, 0),
'lastWord' : (1, 'scalar_number', 0, 0),
'x' : (1, 'number', 0, 0),
'y' : (1, 'number', 0, 0),
'links.page' : (1, 'number', 0, 0),
'link_id' : (1, 'number', 0, 0),
'glyph' : (0, 'number', 1, 1),
'glyph.h' : (1, 'number', 0, 0),
'glyph.w' : (1, 'number', 0, 0),
'sh' : (1, 'number', 0, 0),
'word' : (0, 'number', 1, 1),
'src' : (1, 'scalar_number', 0, 0),
'rel' : (1, 'number', 0, 0),
'row' : (1, 'number', 0, 0),
'startID' : (1, 'number', 0, 1),
'startID.page' : (1, 'number', 0, 0),
'glyphID' : (1, 'number', 0, 0),
'rootID' : (1, 'number', 0, 0),
'stemID' : (1, 'number', 0, 0),
'margin-top' : (1, 'number', 0, 0),
'stemPage' : (1, 'number', 0, 0),
'dehyphen' : (1, 'number', 1, 1),
'rootID' : (1, 'number', 0, 0),
'paraCont' : (1, 'number', 1, 1),
'paraStems' : (1, 'number', 1, 1),
'wordStems' : (1, 'number', 1, 1),
'original' : (0, 'number', 0, 1),
'use' : (1, 'number', 0, 0),
'vtx' : (1, 'number', 0, 1),
'len' : (1, 'number', 0, 1),
'dpi' : (1, 'number', 0, 0),
'n' : (1, 'number', 0, 0),
'id' : (1, 'number', 0, 0),
'ref' : (1, 'number', 0, 0),
'pnum' : (1, 'number', 0, 0),
'pid' : (1, 'text', 0, 0),
'info' : (0, 'number', 1, 0),
'bl' : (1, 'raw', 0, 0),
'firstGlyph' : (1, 'raw', 0, 0),
'lastGlyph' : (1, 'raw', 0, 0),
'ocrText' : (1, 'text', 0, 0),
'title' : (1, 'text', 0, 0),
'href' : (1, 'text', 0, 0),
'_parent_type' : (1, 'text', 0, 0),
'attr' : (1, 'scalar_text', 0, 0),
'justify' : (1, 'scalar_text', 0, 0),
'align' : (1, 'scalar_text', 0, 0),
'layout' : (1, 'scalar_text', 0, 0),
'pageid' : (1, 'scalar_text', 0, 0),
'pagelabel' : (1, 'scalar_text', 0, 0),
'type' : (1, 'text', 0, 0),
'class' : (1, 'scalar_text', 0, 0),
'container' : (1, 'scalar_text', 0, 0),
'_after_class' : (1, 'scalar_text', 0, 0),
'_tag' : (1, 'scalar_text', 0, 0),
'pos' : (1, 'scalar_text', 0, 0),
'page_num' : (1, 'scalar_text', 0, 0),
'page_type' : (1, 'scalar_text', 0, 0),
'findlists' : (1, 'scalar_text', 0, 0),
'FlowEdit_1_id' : (1, 'scalar_text', 0, 0),
'FlowEdit_1_version' : (1, 'scalar_text', 0, 0),
'Schema_id' : (1, 'scalar_text', 0, 0),
'Schema_version' : (1, 'scalar_text', 0, 0),
'Topaz_version' : (1, 'scalar_text', 0, 0),
'WordDetailEdit_1_id' : (1, 'scalar_text', 0, 0),
'WordDetailEdit_1_version' : (1, 'scalar_text', 0, 0),
'ZoneEdit_1_id' : (1, 'scalar_text', 0, 0),
'ZoneEdit_1_version' : (1, 'scalar_text', 0, 0),
'chapterheaders' : (1, 'scalar_text', 0, 0),
'creation_date' : (1, 'scalar_text', 0, 0),
'header_footer' : (1, 'scalar_text', 0, 0),
'init_from_ocr' : (1, 'scalar_text', 0, 0),
'letter_insertion' : (1, 'scalar_text', 0, 0),
'xmlinj_convert' : (1, 'scalar_text', 0, 0),
'xmlinj_reflow' : (1, 'scalar_text', 0, 0),
'xmlinj_transform' : (1, 'scalar_text', 0, 0),
}
# full tag path record keeping routines
def tag_push(self, token):
self.tagpath.append(token)
def tag_pop(self):
if len(self.tagpath) > 0 :
self.tagpath.pop()
def tagpath_len(self):
return len(self.tagpath)
def get_tagpath(self, i):
cnt = len(self.tagpath)
if i < cnt : result = self.tagpath[i]
for j in xrange(i+1, cnt) :
result += '.' + self.tagpath[j]
return result
# list of absolute command byte values values that indicate
# various types of loop meachanisms typically used to generate vectors
cmd_list = (0x76, 0x76)
# peek at and return 1 byte that is ahead by i bytes
def peek(self, aheadi):
c = self.fo.read(aheadi)
if (len(c) == 0):
return None
self.fo.seek(-aheadi,1)
c = c[-1:]
return ord(c)
# get the next value from the file being processed
def getNext(self):
nbyte = self.peek(1);
if (nbyte == None):
return None
val = readEncodedNumber(self.fo)
return val
# format an arg by argtype
def formatArg(self, arg, argtype):
if (argtype == 'text') or (argtype == 'scalar_text') :
result = self.dict.lookup(arg)
elif (argtype == 'raw') or (argtype == 'number') or (argtype == 'scalar_number') :
result = arg
elif (argtype == 'snippets') :
result = arg
else :
print "Error Unknown argtype %s" % argtype
sys.exit(-2)
return result
# process the next tag token, recursively handling subtags,
# arguments, and commands
def procToken(self, token):
known_token = False
self.tag_push(token)
if self.debug : print 'Processing: ', self.get_tagpath(0)
cnt = self.tagpath_len()
for j in xrange(cnt):
tkn = self.get_tagpath(j)
if tkn in self.token_tags :
num_args = self.token_tags[tkn][0]
argtype = self.token_tags[tkn][1]
subtags = self.token_tags[tkn][2]
splcase = self.token_tags[tkn][3]
ntags = -1
known_token = True
break
if known_token :
# handle subtags if present
subtagres = []
if (splcase == 1):
# this type of tag uses of escape marker 0x74 indicate subtag count
if self.peek(1) == 0x74:
skip = readEncodedNumber(self.fo)
subtags = 1
num_args = 0
if (subtags == 1):
ntags = readEncodedNumber(self.fo)
if self.debug : print 'subtags: ' + token + ' has ' + str(ntags)
for j in xrange(ntags):
val = readEncodedNumber(self.fo)
subtagres.append(self.procToken(self.dict.lookup(val)))
# arguments can be scalars or vectors of text or numbers
argres = []
if num_args > 0 :
firstarg = self.peek(1)
if (firstarg in self.cmd_list) and (argtype != 'scalar_number') and (argtype != 'scalar_text'):
# single argument is a variable length vector of data
arg = readEncodedNumber(self.fo)
argres = self.decodeCMD(arg,argtype)
else :
# num_arg scalar arguments
for i in xrange(num_args):
argres.append(self.formatArg(readEncodedNumber(self.fo), argtype))
# build the return tag
result = []
tkn = self.get_tagpath(0)
result.append(tkn)
result.append(subtagres)
result.append(argtype)
result.append(argres)
self.tag_pop()
return result
# all tokens that need to be processed should be in the hash
# table if it may indicate a problem, either new token
# or an out of sync condition
else:
result = []
if (self.debug):
print 'Unknown Token:', token
self.tag_pop()
return result
# special loop used to process code snippets
# it is NEVER used to format arguments.
# builds the snippetList
def doLoop72(self, argtype):
cnt = readEncodedNumber(self.fo)
if self.debug :
result = 'Set of '+ str(cnt) + ' xml snippets. The overall structure \n'
result += 'of the document is indicated by snippet number sets at the\n'
result += 'end of each snippet. \n'
print result
for i in xrange(cnt):
if self.debug: print 'Snippet:',str(i)
snippet = []
snippet.append(i)
val = readEncodedNumber(self.fo)
snippet.append(self.procToken(self.dict.lookup(val)))
self.snippetList.append(snippet)
return
# loop: pass though values unchanged
# DO NOT CHANGE - this has proven to be correct
def doLoop76Mode0(self, argtype, cnt):
result = []
for i in xrange(cnt):
result.append(self.formatArg(readEncodedNumber(self.fo), argtype))
return result
# loop generating values relative to the *negative*
# of the offset - don't ask why - it just is
# DO NOT CHANGE - this has proven to be correct
def doLoop76Mode1(self, argtype, cnt):
result = []
offset = -readEncodedNumber(self.fo)
for i in xrange(cnt):
val = readEncodedNumber(self.fo) + offset
result.append(self.formatArg(val, argtype))
return result
# loop generating values with starting value and accumulation
# DO NOT CHANGE - this has proven to be the correct
def doLoop76Mode2(self, argtype, cnt):
result = []
ptr = readEncodedNumber(self.fo)
result.append(self.formatArg(ptr, argtype))
for i in xrange(cnt-1):
ptr = ptr + readEncodedNumber(self.fo)
result.append(self.formatArg(ptr, argtype))
return result
# loop generating values with starting value and accumulation
# **after** subtracting adjustment value from each
# DO NOT CHANGE - this has been proven to be correct
def doLoop76Mode3(self, argtype, cnt):
result = []
adj = readEncodedNumber(self.fo)
ptr = readEncodedNumber(self.fo)
ptr = ptr - adj
result.append(self.formatArg(ptr, argtype))
for i in xrange(cnt-1):
ptr = ptr + readEncodedNumber(self.fo) - adj
result.append(self.formatArg(ptr,argtype))
return result
# loop using runing sum of data values and starting value
# with accumulation to get new value
# Again, don't ask it took me forever to figure this out
# DO NOT CHANGE - this has been proven to be correct
def doLoop76Mode4(self, argtype, cnt):
result = []
val = readEncodedNumber(self.fo)
runsum = val
ptr = val
result.append(self.formatArg(ptr, argtype))
for i in xrange(cnt-1):
runsum += readEncodedNumber(self.fo)
ptr = ptr + runsum
result.append(self.formatArg(ptr,argtype))
return result
# loop using and extra value as an adjustment
# and a running sum of the values after subtracting
# the adjustment, added to a ptr to get a new pointer
def doLoop76Mode5(self, argtype, cnt):
result = []
adj = readEncodedNumber(self.fo)
ptr = 0
runsum = 0
for i in xrange(cnt):
val = readEncodedNumber(self.fo)
runsum += (val - adj)
ptr = ptr +runsum
result.append(self.formatArg(ptr,argtype))
return result
# FIXME: I have only 4 points to work this out with inside my book
# So may be wrong but it is correct for my 4 points
def doLoop76Mode6(self, argtype, cnt):
result = []
oldval = 0
for i in xrange(cnt):
val = readEncodedNumber(self.fo)
ptr= (3 * oldval) + val + 1
result.append(self.formatArg(ptr,argtype))
oldval = val
return result
# dispatches loop commands bytes with various modes
# The 0x76 style loops are used to build vectors
# This was all derived by trial and error and
# new loop types may exist that are not handled here
# since they did not appear in the test cases
def decodeCMD(self, cmd, argtype):
# if (cmd == 0x72):
# self.doLoop72(argtype)
# result =[]
# return result
if (cmd == 0x76):
# loop with cnt, and mode to control loop styles
cnt = readEncodedNumber(self.fo)
mode = readEncodedNumber(self.fo)
if self.debug : print 'Loop for', cnt, 'with mode', mode, ': '
if (mode == 0x00):
return self.doLoop76Mode0(argtype, cnt)
elif (mode == 0x01):
return self.doLoop76Mode1(argtype, cnt)
elif (mode == 0x02):
return self.doLoop76Mode2(argtype, cnt)
elif (mode == 0x03):
return self.doLoop76Mode3(argtype, cnt)
elif (mode == 0x04):
return self.doLoop76Mode4(argtype, cnt)
elif (mode == 0x05):
return self.doLoop76Mode5(argtype, cnt)
elif (mode == 0x06):
return self.doLoop76Mode6(argtype, cnt)
else:
if self.debug :
# try to mark any unknown loop comands
# if they exist, unless they are used to process
# text or some other known list, we won't be able to prove them correct
print '*** Unknown Loop 0x%x %d %d :' % (cmd, cnt, mode)
for i in xrange(cnt):
val = readEncodedNumber(self.fo)
print ' 0x%x' % val,
print ' '
result = []
return result
if self.dbug: print "Unknown command", cmd
result = []
return result
# add full tag path to injected snippets
def updateName(self, tag, prefix):
name = tag[0]
subtagList = tag[1]
argtype = tag[2]
argList = tag[3]
nname = prefix + '.' + name
nsubtaglist = []
for j in subtagList:
nsubtaglist.append(self.updateName(j,prefix))
ntag = []
ntag.append(nname)
ntag.append(nsubtaglist)
ntag.append(argtype)
ntag.append(argList)
return ntag
# perform depth first injection of specified snippets into this one
def injectSnippets(self, snippet):
snipno, tag = snippet
name = tag[0]
subtagList = tag[1]
argtype = tag[2]
argList = tag[3]
nsubtagList = []
if len(argList) > 0 :
for j in argList:
asnip = self.snippetList[j]
aso, atag = self.injectSnippets(asnip)
atag = self.updateName(atag, name)
nsubtagList.append(atag)
argtype='number'
argList=[]
if len(nsubtagList) > 0 :
subtagList.extend(nsubtagList)
tag = []
tag.append(name)
tag.append(subtagList)
tag.append(argtype)
tag.append(argList)
snippet = []
snippet.append(snipno)
snippet.append(tag)
return snippet
# format the tag for output
def formatTag(self, node):
name = node[0]
subtagList = node[1]
argtype = node[2]
argList = node[3]
fullpathname = name.split('.')
nodename = fullpathname.pop()
ilvl = len(fullpathname)
indent = ' ' * (3 * ilvl)
result = indent + '<' + nodename + '>'
if len(argList) > 0:
argres = ''
for j in argList:
if (argtype == 'text') or (argtype == 'scalar_text') :
argres += j + '|'
else :
argres += str(j) + ','
argres = argres[0:-1]
if argtype == 'snippets' :
result += 'snippets:' + argres
else :
result += argres
if len(subtagList) > 0 :
result += '\n'
for j in subtagList:
if len(j) > 0 :
result += self.formatTag(j)
result += indent + '</' + nodename + '>\n'
else:
result += '</' + nodename + '>\n'
return result
# flatten tag
def flattenTag(self, node):
name = node[0]
subtagList = node[1]
argtype = node[2]
argList = node[3]
result = name
if (len(argList) > 0):
argres = ''
for j in argList:
if (argtype == 'text') or (argtype == 'scalar_text') :
argres += j + '|'
else :
argres += str(j) + '|'
argres = argres[0:-1]
if argtype == 'snippets' :
result += '.snippets=' + argres
else :
result += '=' + argres
result += '\n'
for j in subtagList:
if len(j) > 0 :
result += self.flattenTag(j)
return result
# reduce create xml output
def formatDoc(self, flat_xml):
result = ''
for j in self.doc :
if len(j) > 0:
if flat_xml:
result += self.flattenTag(j)
else:
result += self.formatTag(j)
if self.debug : print result
return result
# main loop - parse the page.dat files
# to create structured document and snippets
# FIXME: value at end of magic appears to be a subtags count
# but for what? For now, inject an 'info" tag as it is in
# every dictionary and seems close to what is meant
# The alternative is to special case the last _ "0x5f" to mean something
def process(self):
# peek at the first bytes to see what type of file it is
magic = self.fo.read(11)
if (magic[0:1] == 'p') and (magic[2:10] == '__PAGE__'):
first_token = 'info'
elif (magic[0:1] == 'g') and (magic[2:11] == '__GLYPH__'):
skip = self.fo.read(1)
first_token = 'info'
else :
# other0.dat file
first_token = None
self.fo.seek(-11,1)
# main loop to read and build the document tree
while True:
if first_token != None :
# use "inserted" first token 'info' for page and glyph files
tag = self.procToken(first_token)
if len(tag) > 0 :
self.doc.append(tag)
first_token = None
v = self.getNext()
if (v == None):
break
if (v == 0x72):
self.doLoop72('number')
elif (v > 0) and (v < self.dict.getSize()) :
tag = self.procToken(self.dict.lookup(v))
if len(tag) > 0 :
self.doc.append(tag)
else:
if self.debug:
print "Mina Loop: Unknown value: %x" % v
# now do snippet injection
if len(self.snippetList) > 0 :
if self.debug : print 'Injecting Snippets:'
snippet = self.injectSnippets(self.snippetList[0])
snipno = snippet[0]
tag_add = snippet[1]
if self.debug : print self.formatTag(tag_add)
if len(tag_add) > 0:
self.doc.append(tag_add)
# handle generation of xml output
xmlpage = self.formatDoc(self.flat_xml)
return xmlpage
def usage():
print 'Usage: '
print ' convert2xml.py dict0000.dat infile.dat '
print ' '
print ' Options:'
print ' -h print this usage help message '
print ' -d turn on debug output to check for potential errors '
print ' --flat-xml output the flattened xml page description only '
print ' '
print ' This program will attempt to convert a page*.dat file or '
print ' glyphs*.dat file, using the dict0000.dat file, to its xml description. '
print ' '
print ' Use "cmbtc_dump.py" first to unencrypt, uncompress, and dump '
print ' the *.dat files from a Topaz format e-book.'
#
# Main
#
def main(argv):
dictFile = ""
pageFile = ""
debug = False
flat_xml = False
printOutput = False
if len(argv) == 0:
printOutput = True
argv = sys.argv
else :
argv = argv.split()
try:
opts, args = getopt.getopt(argv[1:], "hd", ["flat-xml"])
except getopt.GetoptError, err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
if len(opts) == 0 and len(args) == 0 :
usage()
sys.exit(2)
for o, a in opts:
if o =="-d":
debug=True
if o =="-h":
usage()
sys.exit(0)
if o =="--flat-xml":
flat_xml = True
dictFile, pageFile = args[0], args[1]
# read in the string table dictionary
dict = Dictionary(dictFile)
# create a page parser
pp = PageParser(pageFile, dict, debug, flat_xml)
xmlpage = pp.process()
if printOutput:
print xmlpage
return 0
return xmlpage
if __name__ == '__main__':
sys.exit(main(''))

109
Topaz_Tools/lib/decode_meta.py Normal file
View file

@ -0,0 +1,109 @@
#! /usr/bin/python
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
from __future__ import with_statement
import csv
import sys
import os
import getopt
from struct import pack
from struct import unpack
#
# Get a 7 bit encoded number from string
#
def readEncodedNumber(file):
flag = False
c = file.read(1)
if (len(c) == 0):
return None
data = ord(c)
if data == 0xFF:
flag = True
c = file.read(1)
if (len(c) == 0):
return None
data = ord(c)
if data >= 0x80:
datax = (data & 0x7F)
while data >= 0x80 :
c = file.read(1)
if (len(c) == 0):
return None
data = ord(c)
datax = (datax <<7) + (data & 0x7F)
data = datax
if flag:
data = -data
return data
#
# Encode a number in 7 bit format
#
def encodeNumber(number):
result = ""
negative = False
flag = 0
if number < 0 :
number = -number + 1
negative = True
while True:
byte = number & 0x7F
number = number >> 7
byte += flag
result += chr(byte)
flag = 0x80
if number == 0 : break
if negative:
result += chr(0xFF)
return result[::-1]
#
# Get a length prefixed string from the file
#
def lengthPrefixString(data):
return encodeNumber(len(data))+data
def readString(file):
stringLength = readEncodedNumber(file)
if (stringLength == None):
return None
sv = file.read(stringLength)
if (len(sv) != stringLength):
return ""
return unpack(str(stringLength)+"s",sv)[0]
def getMetaArray(metaFile):
# parse the meta file into a Python dictionary (associative array)
result = {}
fo = file(metaFile,'rb')
size = readEncodedNumber(fo)
for i in xrange(size):
temp = readString(fo)
result[temp] = readString(fo)
fo.close()
return result
def getMetaData(metaFile):
# parse the meta file
result = ''
fo = file(metaFile,'rb')
size = readEncodedNumber(fo)
for i in xrange(size):
result += readString(fo) + '|'
result += readString(fo) + '\n'
fo.close()
return result

299
Topaz_Tools/lib/flatxml2html.py Normal file
View file

@ -0,0 +1,299 @@
#! /usr/bin/python
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
from __future__ import with_statement
import csv
import sys
import os
import getopt
from struct import pack
from struct import unpack
class DocParser(object):
def __init__(self, flatxml, fileid):
self.id = os.path.basename(fileid).replace('.dat','')
self.flatdoc = flatxml.split('\n')
self.ocrtext = []
self.link_id = []
self.link_title = []
self.link_page = []
self.dehyphen_rootid = []
self.paracont_stemid = []
self.parastems_stemid = []
# find tag if within pos to end inclusive
def findinDoc(self, tagpath, pos, end) :
result = None
docList = self.flatdoc
cnt = len(docList)
if end == -1 :
end = cnt
else:
end = min(cnt,end)
foundat = -1
for j in xrange(pos, end):
item = docList[j]
if item.find('=') >= 0:
(name, argres) = item.split('=')
else :
name = item
argres = ''
if name.endswith(tagpath) :
result = argres
foundat = j
break
return foundat, result
# return list of start positions for the tagpath
def posinDoc(self, tagpath):
startpos = []
pos = 0
res = ""
while res != None :
(foundpos, res) = self.findinDoc(tagpath, pos, -1)
if res != None :
startpos.append(foundpos)
pos = foundpos + 1
return startpos
# get a description of the paragraph
def getParaDescription(self, start, end):
# normal paragraph
(pos, pclass) = self.findinDoc('paragraph.class',start,end)
# class names are an issue given topaz starts them with numerals (not allowed)
# use a mix of cases, (which cause some browsers problems), and actually
# attach numbers after "reclustered*" to the end to deal with reflow issues
# so we clean this up by lowercasing, prepend 'cl_', and remove all end pieces after reclustered
pclass = pclass.lower()
pclass = 'cl_' + pclass
p = pclass.find('reclustered')
if p > 0 : pclass = pclass[0:p+11]
(pos, sfirst) = self.findinDoc('paragraph.firstWord',start,end)
(pos, slast) = self.findinDoc('paragraph.lastWord',start,end)
if (sfirst != None) and (slast != None) :
return pclass, int(sfirst), int(slast)
# some paragraphs are instead split into multiple spans and some even have word_semantic tags as well
# so walk through this region keeping track of the first firstword, and the last lastWord
# on any items that have it
(pos, sfirst) = self.findinDoc('firstWord',start, end)
first = int(sfirst)
last = -1
for i in xrange(pos+1,end):
(pos, slast) = self.findinDoc('lastWord',i,i+1)
if slast != None:
last = int(slast)
return pclass, first, last
def buildParagraph(self, cname, first, last, type, regtype) :
parares = ''
sep =''
br_lb = False
if (regtype == 'fixed') or (regtype == 'chapterheading') :
br_lb = True
handle_links = False
if len(self.link_id) > 0:
handle_links = True
if (type == 'full') or (type == 'begin') :
parares += '<p class="' + cname + '">'
if (type == 'end'):
parares += ' '
for j in xrange(first, last) :
word = self.ocrtext[j]
sep = ' '
if handle_links:
link = self.link_id[j]
if (link > 0):
title = self.link_title[link-1]
if title == "": title='_link_'
ptarget = self.link_page[link-1] - 1
linkhtml = '<a href="#page%04d">' % ptarget
linkhtml += title + '</a>'
pos = parares.rfind(title)
if pos >= 0:
parares = parares[0:pos] + linkhtml + parares[pos+len(title):]
else :
parares += linkhtml
if word == '_link_' : word = ''
elif (link < 0) :
if word == '_link_' : word = ''
if word == '_lb_':
if (j-1) in self.dehyphen_rootid :
word = ''
sep = ''
elif handle_links :
word = ''
sep = ''
elif br_lb :
word = '<br />\n'
sep = ''
else :
word = '\n'
sep = ''
if j in self.dehyphen_rootid :
word = word[0:-1]
sep = ''
parares += word + sep
if len(sep) > 0 : parares = parares[0:-1]
if (type == 'full') or (type == 'end') :
parares += '</p>'
return parares
# walk the document tree collecting the information needed
# to build an html page using the ocrText
def process(self):
htmlpage = ''
# first collect information from the xml doc that describes this page
(pos, argres) = self.findinDoc('info.word.ocrText',0,-1)
if argres : self.ocrtext = argres.split('|')
(pos, argres) = self.findinDoc('info.dehyphen.rootID',0,-1)
if argres:
argList = argres.split('|')
self.dehyphen_rootid = [ int(strval) for strval in argList]
(pos, self.parastems_stemid) = self.findinDoc('info.paraStems.stemID',0,-1)
if self.parastems_stemid == None : self.parastems_stemid = []
(pos, self.paracont_stemid) = self.findinDoc('info.paraCont.stemID',0,-1)
if self.paracont_stemid == None : self.paracont_stemid = []
(pos, argres) = self.findinDoc('info.word.link_id',0,-1)
if argres:
argList = argres.split('|')
self.link_id = [ int(strval) for strval in argList]
(pos, argres) = self.findinDoc('info.links.page',0,-1)
if argres :
argList = argres.split('|')
self.link_page = [ int(strval) for strval in argList]
(pos, argres) = self.findinDoc('info.links.title',0,-1)
if argres :
self.link_title = argres.split('|')
else:
self.link_title.append('')
(pos, pagetype) = self.findinDoc('page.type',0,-1)
# generate a list of each region starting point
# each region has one paragraph,, or one image, or one chapterheading
regionList= self.posinDoc('region')
regcnt = len(regionList)
regionList.append(-1)
anchorSet = False
breakSet = False
# process each region tag and convert what you can to html
for j in xrange(regcnt):
start = regionList[j]
end = regionList[j+1]
(pos, regtype) = self.findinDoc('region.type',start,end)
if regtype == 'graphic' :
if not anchorSet:
htmlpage += '<div id="' + self.id + '" class="page_' + pagetype + '">&nbsp</div>\n'
anchorSet = True
(pos, simgsrc) = self.findinDoc('img.src',start,end)
if simgsrc:
htmlpage += '<div class="graphic"><img src="img/img%04d.jpg" alt="" /></div>' % int(simgsrc)
elif regtype == 'chapterheading' :
(pclass, first, last) = self.getParaDescription(start,end)
if not breakSet:
htmlpage += '<div style="page-break-after: always;">&nbsp;</div>\n'
breakSet = True
if not anchorSet:
htmlpage += '<div id="' + self.id + '" class="page_' + pagetype + '">&nbsp</div>\n'
anchorSet = True
tag = 'h1'
if pclass[3:7] == 'ch1-' : tag = 'h1'
if pclass[3:7] == 'ch2-' : tag = 'h2'
if pclass[3:7] == 'ch3-' : tag = 'h3'
htmlpage += '<' + tag + ' class="' + pclass + '">'
htmlpage += self.buildParagraph(pclass,first,last,'middle', regtype)
htmlpage += '</' + tag + '>'
elif (regtype == 'text') or (regtype == 'fixed') or (regtype == 'insert') :
ptype = 'full'
# check to see if this is a continution from the previous page
if (len(self.parastems_stemid) > 0):
ptype = 'end'
self.parastems_stemid=[]
else:
if not anchorSet:
htmlpage += '<div id="' + self.id + '" class="page_' + pagetype + '">&nbsp</div>\n'
anchorSet = True
(pclass, first, last) = self.getParaDescription(start,end)
if ptype == 'full' :
tag = 'p'
if pclass[3:6] == 'h1-' : tag = 'h4'
if pclass[3:6] == 'h2-' : tag = 'h5'
if pclass[3:6] == 'h3-' : tag = 'h6'
htmlpage += '<' + tag + ' class="' + pclass + '">'
htmlpage += self.buildParagraph(pclass, first, last, 'middle', regtype)
htmlpage += '</' + tag + '>'
else :
htmlpage += self.buildParagraph(pclass, first, last, ptype, regtype)
elif (regtype == 'tocentry') :
ptype = 'full'
# check to see if this is a continution from the previous page
if (len(self.parastems_stemid) > 0) and (j == 0):
# process the first paragraph as a continuation from the last page
ptype = 'end'
self.parastems_stemid = []
else:
if not anchorSet:
htmlpage += '<div id="' + self.id + '" class="page_' + pagetype + '">&nbsp</div>\n'
anchorSet = True
(pclass, first, last) = self.getParaDescription(start,end)
htmlpage += self.buildParagraph(pclass, first, last, ptype, regtype)
else :
print 'Unknown region type', regtype
print 'Warning: skipping this region'
if len(self.paracont_stemid) > 0 :
if htmlpage[-4:] == '</p>':
htmlpage = htmlpage[0:-4]
return htmlpage
return self.convert2HTML()
def convert2HTML(flatxml, fileid):
# create a document parser
dp = DocParser(flatxml, fileid)
htmlpage = dp.process()
return htmlpage

125
Topaz_Tools/lib/genhtml.py Normal file
View file

@ -0,0 +1,125 @@
#! /usr/bin/python
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
import os, sys, getopt
# local routines
import convert2xml
import flatxml2html
import decode_meta
import stylexml2css
def usage():
print 'Usage: '
print ' '
print ' genhtml.py unencryptedBookDir'
print ' '
def main(argv):
bookDir = ''
if len(argv) == 0:
argv = sys.argv
else :
argv = argv.split()
try:
opts, args = getopt.getopt(argv[1:], "h:")
except getopt.GetoptError, err:
print str(err)
usage()
sys.exit(2)
if len(opts) == 0 and len(args) == 0 :
usage()
sys.exit(2)
for o, a in opts:
if o =="-h":
usage()
sys.exit(0)
bookDir = args[0]
if not os.path.exists(bookDir) :
print "Can not find directory with unencrypted book"
sys.exit(-1)
dictFile = os.path.join(bookDir,'dict0000.dat')
if not os.path.exists(dictFile) :
print "Can not find dict0000.dat file"
sys.exit(-1)
pageDir = os.path.join(bookDir,'page')
if not os.path.exists(pageDir) :
print "Can not find page directory in unencrypted book"
sys.exit(-1)
imgDir = os.path.join(bookDir,'img')
if not os.path.exists(imgDir) :
print "Can not find image directory in unencrypted book"
sys.exit(-1)
otherFile = os.path.join(bookDir,'other0000.dat')
if not os.path.exists(otherFile) :
print "Can not find other0000.dat in unencrypted book"
sys.exit(-1)
metaFile = os.path.join(bookDir,'metadata0000.dat')
if not os.path.exists(metaFile) :
print "Can not find metadata0000.dat in unencrypted book"
sys.exit(-1)
htmlFileName = "book.html"
htmlstr = '<html>\n'
filenames = os.listdir(pageDir)
filenames = sorted(filenames)
print 'Processing ... '
htmlstr += '<head>\n'
print ' ', 'metadata0000.dat'
fname = os.path.join(bookDir,'metadata0000.dat')
xname = os.path.join(bookDir, 'metadata.txt')
metastr = decode_meta.getMetaData(fname)
file(xname, 'wb').write(metastr)
meta_array = decode_meta.getMetaArray(fname)
htmlstr += '<meta name="Author" content="' + meta_array['Authors'] + '" />\n'
htmlstr += '<meta name="Title" content="' + meta_array['Title'] + '" />\n'
print ' ', 'other0000.dat'
fname = os.path.join(bookDir,'other0000.dat')
xname = os.path.join(bookDir, 'style.css')
xmlstr = convert2xml.main('convert2xml.py --flat-xml ' + dictFile + ' ' + fname)
cssstr = '<style>\n'
cssstr += stylexml2css.convert2CSS(xmlstr)
cssstr += '</style>\n'
file(xname, 'wb').write(cssstr)
htmlstr += cssstr
htmlstr += '</head>\n<body>\n'
for filename in filenames:
print ' ', filename
fname = os.path.join(pageDir,filename)
flat_xml = convert2xml.main('convert2xml.py --flat-xml ' + dictFile + ' ' + fname)
htmlstr += flatxml2html.convert2HTML(flat_xml, fname)
htmlstr += '</body>\n</html>\n'
file(os.path.join(bookDir, htmlFileName), 'wb').write(htmlstr)
print 'Processing Complete'
return 0
if __name__ == '__main__':
sys.exit(main(''))

295
Topaz_Tools/lib/gensvg.py Normal file
View file

@ -0,0 +1,295 @@
#! /usr/bin/python
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
import os, sys, getopt
# local routines
import convert2xml
import flatxml2html
import decode_meta
class GParser(object):
def __init__(self, flatxml):
self.flatdoc = flatxml.split('\n')
self.dpi = 1440
self.gh = self.getData('info.glyph.h')
self.gw = self.getData('info.glyph.w')
self.guse = self.getData('info.glyph.use')
self.count = len(self.guse)
self.gvtx = self.getData('info.glyph.vtx')
self.glen = self.getData('info.glyph.len')
self.gdpi = self.getData('info.glyph.dpi')
self.vx = self.getData('info.vtx.x')
self.vy = self.getData('info.vtx.y')
self.vlen = self.getData('info.len.n')
self.glen.append(len(self.vlen))
self.gvtx.append(len(self.vx))
def getData(self, path):
result = None
cnt = len(self.flatdoc)
for j in xrange(cnt):
item = self.flatdoc[j]
if item.find('=') >= 0:
(name, argt) = item.split('=')
argres = argt.split('|')
else:
name = item
argres = []
if (name == path):
result = argres
break
if (len(argres) > 0) :
for j in xrange(0,len(argres)):
argres[j] = int(argres[j])
return result
def getPath(self, gly):
path = ''
if (gly < 0) or (gly >= self.count):
return path
tx = self.vx[self.gvtx[gly]:self.gvtx[gly+1]-1]
ty = self.vy[self.gvtx[gly]:self.gvtx[gly+1]-1]
p = 0
for k in xrange(self.glen[gly], self.glen[gly+1]):
if (p == 0):
zx = tx[0:self.vlen[k]+1]
zy = ty[0:self.vlen[k]+1]
else:
zx = tx[self.vlen[k-1]+1:self.vlen[k]+1]
zy = ty[self.vlen[k-1]+1:self.vlen[k]+1]
p += 1
for j in xrange(0, len(zx)):
if (j == 0):
path += 'M %d %d ' % (zx[j] * self.dpi / self.gdpi[gly], zy[j] * self.dpi / self.gdpi[gly])
else:
path += 'L %d %d ' % (zx[j] * self.dpi / self.gdpi[gly], zy[j] * self.dpi / self.gdpi[gly])
path += 'z'
return path
class PParser(object):
def __init__(self, flatxml):
self.flatdoc = flatxml.split('\n')
self.temp = []
self.ph = self.getData('page.h')[0]
self.pw = self.getData('page.w')[0]
self.gx = self.getData('info.glyph.x')
self.gy = self.getData('info.glyph.y')
self.gid = self.getData('info.glyph.glyphID')
def getData(self, path):
result = None
cnt = len(self.flatdoc)
for j in xrange(cnt):
item = self.flatdoc[j]
if item.find('=') >= 0:
(name, argt) = item.split('=')
argres = argt.split('|')
else:
name = item
argres = []
if (name.endswith(path)):
result = argres
break
if (len(argres) > 0) :
for j in xrange(0,len(argres)):
argres[j] = int(argres[j])
return result
def getDataTemp(self, path):
result = None
cnt = len(self.temp)
for j in xrange(cnt):
item = self.temp[j]
if item.find('=') >= 0:
(name, argt) = item.split('=')
argres = argt.split('|')
else:
name = item
argres = []
if (name.endswith(path)):
result = argres
self.temp.pop(j)
break
if (len(argres) > 0) :
for j in xrange(0,len(argres)):
argres[j] = int(argres[j])
return result
def getImages(self):
result = []
self.temp = self.flatdoc
while (self.getDataTemp('region.img') != None):
h = self.getDataTemp('region.img.h')[0]
w = self.getDataTemp('region.img.w')[0]
x = self.getDataTemp('region.img.x')[0]
y = self.getDataTemp('region.img.y')[0]
src = self.getDataTemp('region.img.src')[0]
result.append('<image xlink:href="../img/img%04d.jpg" x="%d" y="%d" width="%d" height="%d" />\n' % (src, x, y, w, h))
return result
def getGlyphs(self,glyfname):
result = []
if (self.gid != None) and (len(self.gid) > 0):
glyphs = []
for j in set(self.gid):
glyphs.append(j)
glyphs.sort()
gfile = open(glyfname, 'r')
j = 0
while True :
inp = gfile.readline()
if (inp == ''):
break
id='id="gl%d"' % glyphs[j]
if (inp.find(id) > 0):
result.append(inp)
j += 1
if (j == len(glyphs)):
break
gfile.close()
return result
def usage():
print 'Usage: '
print ' '
print ' gensvg.py unencryptedBookDir'
print ' '
def main(argv):
bookDir = ''
if len(argv) == 0:
argv = sys.argv
else :
argv = argv.split()
try:
opts, args = getopt.getopt(argv[1:], "h:")
except getopt.GetoptError, err:
print str(err)
usage()
sys.exit(2)
if len(opts) == 0 and len(args) == 0 :
usage()
sys.exit(2)
for o, a in opts:
if o =="-h":
usage()
sys.exit(0)
bookDir = args[0]
if not os.path.exists(bookDir) :
print "Can not find directory with unencrypted book"
sys.exit(-1)
dictFile = os.path.join(bookDir,'dict0000.dat')
if not os.path.exists(dictFile) :
print "Can not find dict0000.dat file"
sys.exit(-1)
pageDir = os.path.join(bookDir,'page')
if not os.path.exists(pageDir) :
print "Can not find page directory in unencrypted book"
sys.exit(-1)
imgDir = os.path.join(bookDir,'img')
if not os.path.exists(imgDir) :
print "Can not find image directory in unencrypted book"
sys.exit(-1)
glyphsDir = os.path.join(bookDir,'glyphs')
if not os.path.exists(glyphsDir) :
print "Can not find glyphs directory in unencrypted book"
sys.exit(-1)
metaFile = os.path.join(bookDir,'metadata0000.dat')
if not os.path.exists(metaFile) :
print "Can not find metadata0000.dat in unencrypted book"
sys.exit(-1)
svgDir = os.path.join(bookDir,'svg')
if not os.path.exists(svgDir) :
os.makedirs(svgDir)
print 'Processing Meta Data ... '
print ' ', 'metadata0000.dat'
fname = os.path.join(bookDir,'metadata0000.dat')
metadata = decode_meta.getMetaArray(fname)
print 'Processing Glyphs ... '
filenames = os.listdir(glyphsDir)
filenames = sorted(filenames)
glyfname = os.path.join(svgDir,'glyphs.svg')
glyfile = open(glyfname, 'w')
glyfile.write('<?xml version="1.0" standalone="no"?>\n')
glyfile.write('<!DOCTYPE svg PUBLIC "-//W3C/DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">\n')
glyfile.write('<svg width="512" height="512" viewBox="0 0 511 511" xmlns="http://www.w3.org/2000/svg" version="1.1">\n')
glyfile.write('<title>Glyphs for %s</title>\n' % metadata['Title'])
glyfile.write('<defs>\n')
counter = 0
for filename in filenames:
print ' ', filename
fname = os.path.join(glyphsDir,filename)
flat_xml = convert2xml.main('convert2xml.py --flat-xml ' + dictFile + ' ' + fname)
gp = GParser(flat_xml)
for i in xrange(0, gp.count):
path = gp.getPath(i)
glyfile.write('<path id="gl%d" d="%s" fill="black" />\n' % (counter * 256 + i, path))
counter += 1
glyfile.write('</defs>\n')
glyfile.write('</svg>\n')
glyfile.close()
print 'Processing Pages ... '
scaledpi = 720
filenames = os.listdir(pageDir)
filenames = sorted(filenames)
counter = 0
for filename in filenames:
print ' ', filename
fname = os.path.join(pageDir,filename)
flat_xml = convert2xml.main('convert2xml.py --flat-xml ' + dictFile + ' ' + fname)
pp = PParser(flat_xml)
pfile = open(os.path.join(svgDir,filename.replace('.dat','.svg')), 'w')
pfile.write('<?xml version="1.0" standalone="no"?>\n')
pfile.write('<!DOCTYPE svg PUBLIC "-//W3C/DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">\n')
pfile.write('<svg width="%fin" height="%fin" viewBox="0 0 %d %d" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1">\n' % (pp.pw / scaledpi, pp.ph / scaledpi, pp.pw -1, pp.ph -1))
pfile.write('<title>Page %d - %s by %s</title>\n' % (counter, metadata['Title'],metadata['Authors']))
if (pp.gid != None):
pfile.write('<defs>\n')
gdefs = pp.getGlyphs(glyfname)
for j in xrange(0,len(gdefs)):
pfile.write(gdefs[j])
pfile.write('</defs>\n')
for j in xrange(0,len(pp.gid)):
pfile.write('<use xlink:href="#gl%d" x="%d" y="%d" />\n' % (pp.gid[j], pp.gx[j], pp.gy[j]))
img = pp.getImages()
if (img != None):
for j in xrange(0,len(img)):
pfile.write(img[j])
pfile.write('</svg>')
pfile.close()
counter += 1
print 'Processing Complete'
return 0
if __name__ == '__main__':
sys.exit(main(''))

121
Topaz_Tools/lib/genxml.py Normal file
View file

@ -0,0 +1,121 @@
#! /usr/bin/python
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
import os, sys, getopt
# local routines
import convert2xml
import flatxml2html
import decode_meta
def usage():
print 'Usage: '
print ' '
print ' genxml.py dict0000.dat unencryptedBookDir'
print ' '
def main(argv):
bookDir = ''
if len(argv) == 0:
argv = sys.argv
else :
argv = argv.split()
try:
opts, args = getopt.getopt(argv[1:], "h:")
except getopt.GetoptError, err:
print str(err)
usage()
sys.exit(2)
if len(opts) == 0 and len(args) == 0 :
usage()
sys.exit(2)
for o, a in opts:
if o =="-h":
usage()
sys.exit(0)
bookDir = args[0]
if not os.path.exists(bookDir) :
print "Can not find directory with unencrypted book"
sys.exit(-1)
dictFile = os.path.join(bookDir,'dict0000.dat')
if not os.path.exists(dictFile) :
print "Can not find dict0000.dat file"
sys.exit(-1)
pageDir = os.path.join(bookDir,'page')
if not os.path.exists(pageDir) :
print "Can not find page directory in unencrypted book"
sys.exit(-1)
glyphsDir = os.path.join(bookDir,'glyphs')
if not os.path.exists(glyphsDir) :
print "Can not find glyphs directory in unencrypted book"
sys.exit(-1)
otherFile = os.path.join(bookDir,'other0000.dat')
if not os.path.exists(otherFile) :
print "Can not find other0000.dat in unencrypted book"
sys.exit(-1)
metaFile = os.path.join(bookDir,'metadata0000.dat')
if not os.path.exists(metaFile) :
print "Can not find metadata0000.dat in unencrypted book"
sys.exit(-1)
xmlDir = os.path.join(bookDir,'xml')
if not os.path.exists(xmlDir):
os.makedirs(xmlDir)
print 'Processing ... '
print ' ', 'metadata0000.dat'
fname = os.path.join(bookDir,'metadata0000.dat')
xname = os.path.join(xmlDir, 'metadata.txt')
metastr = decode_meta.getMetaData(fname)
file(xname, 'wb').write(metastr)
print ' ', 'other0000.dat'
fname = os.path.join(bookDir,'other0000.dat')
xname = os.path.join(xmlDir, 'stylesheet.xml')
xmlstr = convert2xml.main('convert2xml.py ' + dictFile + ' ' + fname)
file(xname, 'wb').write(xmlstr)
filenames = os.listdir(pageDir)
filenames = sorted(filenames)
for filename in filenames:
print ' ', filename
fname = os.path.join(pageDir,filename)
xname = os.path.join(xmlDir, filename.replace('.dat','.xml'))
xmlstr = convert2xml.main('convert2xml.py ' + dictFile + ' ' + fname)
file(xname, 'wb').write(xmlstr)
filenames = os.listdir(glyphsDir)
filenames = sorted(filenames)
for filename in filenames:
print ' ', filename
fname = os.path.join(glyphsDir,filename)
xname = os.path.join(xmlDir, filename.replace('.dat','.xml'))
xmlstr = convert2xml.main('convert2xml.py ' + dictFile + ' ' + fname)
file(xname, 'wb').write(xmlstr)
print 'Processing Complete'
return 0
if __name__ == '__main__':
sys.exit(main(''))

75
Topaz_Tools/lib/readme.txt Normal file
View file

@ -0,0 +1,75 @@
This is experimental and it will probably not work for you but...
ALSO: Please do not use any of this to steal. Theft is wrong.
This is meant to allow conversion of Topaz books for other book readers you own
Here are the steps:
1. Unzip the topazscripts.zip file to get the full set of python scripts.
The files you should have after unzipping are:
cmbtc_dump.py - (author: cmbtc) unencrypts and dumps sections into separate files
decode_meta.py - converts metadata0000.dat to human readable text (for the most part)
convert2xml.py - converts page*.dat, other*.dat, and glyphs*.dat files to pseudo xml descriptions
flatxml2html.py - converts a "flattened" xml description to html using the ocrtext
stylexml2css.py - converts stylesheet "flattened" xml into css (as best it can)
genxml.py - main program to convert everything to xml
genhtml.py - main program to generate "book.html"
gensvg.py - (author: clarknova) main program to create an svg grpahic of each page
Please note, gensvg.py, genhtml.py, and genxml.py import and use
decode_meta.py, convert2xml.py, flatxml2html.py, and stylexml2css.py
so please keep all of these python scripts together in the same place.
2. Remove the DRM from the Topaz book and build a directory
of its contents as files
All Thanks go to CMBTC who broke the DRM for Topaz - without it nothing else
would be possible
cmbtc_dump.py -d -o TARGETDIR [-p pid] YOURTOPAZBOOKNAMEHERE
This should create a directory called "TARGETDIR" in your current directory.
It should have the following files in it:
metadata0000.dat - metadata info
other0000.dat - information used to create a style sheet
dict0000.dat - dictionary of words used to build page descriptions
page - directory filled with page*.dat files
glyphs - directory filled with glyphs*.dat files
3. Convert the files in "TARGETDIR" to their xml descriptions
which can be found in TARGETDIR/xml/ upon completion.
genxml.py TARGETDIR
4. Create book.html which can be found in "TARGETDIR" after
completion. This html conversion can not fully capture
all of the layouts actually used in the book and needs to
be edited to include special font handling such as bold
or italics that can not be determined from the ocrText
information or the style information. If you want to
see things exactly as they were, see step 5 below.
genhtml.py TARGETDIR
5. Create an svg description of each page which can
be found in TARGETDIR/svg/ upon completion.
All thanks go to CLARKNOVA for this program. This program is
needed to actually see the true image of each page so that hand
editing of the html created by step 4 can be done.
Or use the resulting svg files to read each page of the book
exactly as it has been laid out originally.
gensvg.py TARGETDIR

221
Topaz_Tools/lib/stylexml2css.py Normal file
View file

@ -0,0 +1,221 @@
#! /usr/bin/python
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
from __future__ import with_statement
import csv
import sys
import os
import getopt
from struct import pack
from struct import unpack
class DocParser(object):
def __init__(self, flatxml):
self.flatdoc = flatxml.split('\n')
stags = {
'paragraph' : 'p',
'graphic' : '.graphic'
}
attr_val_map = {
'hang' : ('text-indent: ', 135),
'indent' : ('text-indent: ', 135),
'line-space' : ('line-height: ', 190),
'margin-bottom' : ('margin-bottom: ', 135),
'margin-left' : ('margin-left: ', 135),
'margin-right' : ('margin-right: ', 135),
'margin-top' : ('margin-top: ', 135),
'space-after' : ('padding-bottom: ', 135),
}
attr_str_map = {
'align-center' : 'text-align: center; margin-left: auto; margin-right: auto;',
'align-left' : 'text-align: left;',
'align-right' : 'text-align: right;',
'align-justify' : 'text-align: justify;',
'display-inline' : 'display: inline;',
'pos-left' : 'text-align: left;',
'pos-right' : 'text-align: right;',
'pos-center' : 'text-align: center; margin-left: auto; margin-right: auto;',
}
# find tag if within pos to end inclusive
def findinDoc(self, tagpath, pos, end) :
result = None
docList = self.flatdoc
cnt = len(docList)
if end == -1 :
end = cnt
else:
end = min(cnt,end)
foundat = -1
for j in xrange(pos, end):
item = docList[j]
if item.find('=') >= 0:
(name, argres) = item.split('=')
else :
name = item
argres = ''
if name.endswith(tagpath) :
result = argres
foundat = j
break
return foundat, result
# return list of start positions for the tagpath
def posinDoc(self, tagpath):
startpos = []
pos = 0
res = ""
while res != None :
(foundpos, res) = self.findinDoc(tagpath, pos, -1)
if res != None :
startpos.append(foundpos)
pos = foundpos + 1
return startpos
def process(self):
csspage = ''
# generate a list of each <style> starting point in the stylesheet
styleList= self.posinDoc('book.stylesheet.style')
stylecnt = len(styleList)
styleList.append(-1)
# process each style converting what you can
for j in xrange(stylecnt):
start = styleList[j]
end = styleList[j+1]
(pos, tag) = self.findinDoc('style._tag',start,end)
if tag == None :
(pos, tag) = self.findinDoc('style.type',start,end)
# Is this something we know how to convert to css
if tag in self.stags :
# get the style class
(pos, sclass) = self.findinDoc('style.class',start,end)
if sclass != None:
sclass = '.cl_' + sclass.lower()
else :
sclass = ''
# check for any "after class" specifiers
(pos, aftclass) = self.findinDoc('style._after_class',start,end)
if aftclass != None:
aftclass = '.cl_' + aftclass.lower()
else :
aftclass = ''
cssargs = {}
while True :
(pos, attr) = self.findinDoc('style.rule.attr', start, end)
(pos, val) = self.findinDoc('style.rule.value', start, end)
if attr == None : break
if (attr == 'display') or (attr == 'pos') or (attr == 'align'):
# handle text based attributess
attr = attr + '-' + val
if attr in self.attr_str_map :
cssargs[attr] = (self.attr_str_map[attr], '')
else :
# handle value based attributes
if attr in self.attr_val_map :
(name, scale) = self.attr_val_map[attr]
if not ((attr == 'hang') and (int(val) == 0)) :
ems = int(val)/scale
cssargs[attr] = (self.attr_val_map[attr][0], ems)
keep = True
start = pos + 1
# disable all of the after class tags until I figure out how to handle them
# remove all numerals after the "reclustered"
if aftclass != "" : keep = False
p = sclass.find('reclustered')
if p >= 0:
sclass = sclass[0:p+11]
if keep :
# make sure line-space does not go below 1em
if 'line-space' in cssargs:
seg = cssargs['line-space'][0]
val = cssargs['line-space'][1]
if val < 1.0: val = 1.0
del cssargs['line-space']
cssargs['line-space'] = (self.attr_val_map['line-space'][0], val)
# handle modifications for css style hanging indents
if 'hang' in cssargs:
hseg = cssargs['hang'][0]
hval = cssargs['hang'][1]
del cssargs['hang']
cssargs['hang'] = (self.attr_val_map['hang'][0], -hval)
mval = 0
mseg = 'margin-left: '
if 'margin-left' in cssargs:
mseg = cssargs['margin-left'][0]
mval = cssargs['margin-left'][1]
mval = hval + mval
cssargs['margin-left'] = (mseg, mval)
if 'indent' in cssargs:
del cssargs['indent']
cssline = sclass + ' { '
for key in iter(cssargs):
mseg = cssargs[key][0]
mval = cssargs[key][1]
if mval == '':
cssline += mseg + ' '
else :
aseg = mseg + '%.1fem;' % mval
cssline += aseg + ' '
cssline += '}'
# handle special case of paragraph class used inside chapter heading
# and non-chapter headings
if sclass != '' :
ctype = sclass[4:7]
if ctype == 'ch1' :
csspage += 'h1' + cssline + '\n'
if ctype == 'ch2' :
csspage += 'h2' + cssline + '\n'
if ctype == 'ch3' :
csspage += 'h3' + cssline + '\n'
if ctype == 'h1-' :
csspage += 'h4' + cssline + '\n'
if ctype == 'h2-' :
csspage += 'h5' + cssline + '\n'
if ctype == 'h3_' :
csspage += 'h6' + cssline + '\n'
csspage += self.stags[tag] + cssline + '\n'
return csspage
def convert2CSS(flatxml):
# create a document parser
dp = DocParser(flatxml)
csspage = dp.process()
return csspage