Merge of 0.04 (missing) and 0.05 to get ereader2html 0.06

eReader_Tools/lib/ereader2html.py

@@ -1,17 +1,90 @@
+#!/usr/bin/env python
+# -*- coding: ascii -*-
+# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
+#
 # This is a python script. You need a Python interpreter to run it.
 # For example, ActiveState Python, which exists for windows.
 # Changelog
 #  0.01 - Initial version
 #  0.02 - Support more eReader files. Support bold text and links. Fix PML decoder parsing bug.
 #  0.03 - Fix incorrect variable usage at one place.
+#  0.03b - enhancement by DeBockle (version 259 support)
+# Custom version 0.03 - no change to eReader support, only usability changes
+#   - start of pep-8 indentation (spaces not tab), fix trailing blanks
+#   - version variable, only one place to change
+#   - added main routine, now callable as a library/module, 
+#     means tools can add optional support for ereader2html
+#   - outdir is no longer a mandatory parameter (defaults based on input name if missing)
+#   - time taken output to stdout
+#   - Psyco support - reduces runtime by a factor of (over) 3!
+#     E.g. (~600Kb file) 90 secs down to 24 secs
+#       - newstyle classes
+#       - changed map call to list comprehension
+#         may not work with python 2.3
+#         without Psyco this reduces runtime to 90%
+#         E.g. 90 secs down to 77 secs
+#         Psyco with map calls takes longer, do not run with map in Psyco JIT!
+#       - izip calls used instead of zip (if available), further reduction
+#         in run time (factor of 4.5).
+#         E.g. (~600Kb file) 90 secs down to 20 secs
+#   - Python 2.6+ support, avoid DeprecationWarning with sha/sha1
+#  0.04 - Footnote support, PML output, correct charset in html, support more PML tags
+#   - Feature change, dump out PML file
+#   - Added supprt for footnote tags. NOTE footnote ids appear to be bad (not usable)
+#       in some pdb files :-( due to the same id being used multiple times
+#   - Added correct charset encoding (pml is based on cp1252)
+#   - Added logging support.
+#   
+#   TODO run this through a profiler - speed increases so far was from
+#   applying "quick known fixes", added (commented out) cprofiler call
 #  0.05 - Support for more things in type 272
-import struct, binascii, zlib, os, sha, sys, os.path
+#  0.06 - Merge of 0.04 and 0.05. Improved HTML output
+#         Placed images in subfolder, so that it's possible to just
+#         drop the book.pml file onto DropBook to make an unencrypted
+#         copy of the eReader file.
+#         Using that with Calibre works a lot better than the HTML
+#         conversion in this code.
 
+__version__='0.06'
 DEBUG = 0
 
+
+# Import Psyco if available
+try:
+    # Dumb speed hack 1
+    # http://psyco.sourceforge.net
+    import psyco
+    psyco.full()
+    pass
+except ImportError:
+    pass
+try:
+    # Dumb speed hack 2
+    # All map() calls converted to list comprehension (some use zip)
+    # override zip with izip - saves memory and in rough testing
+    # appears to be faster zip() is only used in the converted map() calls
+    from itertools import izip as zip
+except ImportError:
+    pass
+
+import struct, binascii, zlib, os, sys, os.path, urllib
+try:
+    from hashlib import sha1
+except ImportError:
+    # older Python release
+    import sha
+    sha1 = lambda s: sha.new(s)
+import cgi
+import logging
+
+logging.basicConfig()
+#logging.basicConfig(level=logging.DEBUG)
+
+write_pml=False
+
 ECB = 0
 CBC = 1
-class Des:
+class Des(object):
     __pc1 = [56, 48, 40, 32, 24, 16,  8,  0, 57, 49, 41, 33, 25, 17,
           9,  1, 58, 50, 42, 34, 26, 18, 10,  2, 59, 51, 43, 35,
          62, 54, 46, 38, 30, 22, 14,  6, 61, 53, 45, 37, 29, 21,
@@ -128,7 +201,7 @@ class Des:
             pos += 1
         return result
     def __permutate(self, table, block):
-		return map(lambda x: block[x], table)
+        return [block[x] for x in table]
     def __create_sub_keys(self):
         key = self.__permutate(Des.__pc1, self.__String_to_BitList(self.getKey()))
         i = 0
@@ -158,7 +231,7 @@ class Des:
         while i < 16:
             tempR = self.R[:]
             self.R = self.__permutate(Des.__expansion_table, self.R)
-			self.R = map(lambda x, y: x ^ y, self.R, self.Kn[iteration])
+            self.R = [x ^ y for x,y in zip(self.R, self.Kn[iteration])]
             B = [self.R[:6], self.R[6:12], self.R[12:18], self.R[18:24], self.R[24:30], self.R[30:36], self.R[36:42], self.R[42:]]
             j = 0
             Bn = [0] * 32
@@ -174,7 +247,7 @@ class Des:
                 pos += 4
                 j += 1
             self.R = self.__permutate(Des.__p, Bn)
-			self.R = map(lambda x, y: x ^ y, self.R, self.L)
+            self.R = [x ^ y for x, y in zip(self.R, self.L)]
             self.L = tempR
             i += 1
             iteration += iteration_adjustment
@@ -202,10 +275,10 @@ class Des:
             block = self.__String_to_BitList(data[i:i+8])
             if self.getMode() == CBC:
                 if crypt_type == Des.ENCRYPT:
-					block = map(lambda x, y: x ^ y, block, iv)
+                    block = [x ^ y for x, y in zip(block, iv)]
                 processed_block = self.__des_crypt(block, crypt_type)
                 if crypt_type == Des.DECRYPT:
-					processed_block = map(lambda x, y: x ^ y, processed_block, iv)
+                    processed_block = [x ^ y for x, y in zip(processed_block, iv)]
                     iv = block
                 else:
                     iv = processed_block
@@ -226,7 +299,7 @@ class Des:
         self.__padding = pad
         return self.crypt(data, Des.DECRYPT)
 
-class Sectionizer:
+class Sectionizer(object):
     def __init__(self, filename, ident):
         self.contents = file(filename, 'rb').read()
         self.header = self.contents[0:72]
@@ -275,8 +348,7 @@ def deXOR(text, sp, table):
             j = 0
     return r
 
-
+class EreaderProcessor(object):
-class EreaderProcessor:
     def __init__(self, section_reader, username, creditcard):
         self.section_reader = section_reader
         data = section_reader(0)
@@ -284,19 +356,21 @@ class EreaderProcessor:
         if DEBUG:
             file("data0.dat", 'wb').write(self.data0)
         version,  = struct.unpack('>H', data[0:2])
-		if version != 272 and version != 260:
+        self.version = version
+        logging.info('eReader file format version %s', version)
+        if version != 272 and version != 260 and version != 259:
             raise ValueError('incorrect eReader version %d (error 1)' % version)
         data = section_reader(1)
         self.data = data
-		des = Des(fixKey(data[0:8]))
         # first key is used on last 8 bytes of data to get cookie_shuf and cookie_size
         self.first_key = data[0:8]
+        des = Des(fixKey(self.first_key))
         self.end_key = des.decrypt(data[-8:])
-		cookie_shuf, cookie_size = struct.unpack('>LL', des.decrypt(data[-8:]))
+        cookie_shuf, cookie_size = struct.unpack('>LL', self.end_key)
         if cookie_shuf < 3 or cookie_shuf > 0x14 or cookie_size < 0xf0 or cookie_size > 0x200:
             raise ValueError('incorrect eReader version (error 2)')
         # first key is also used to decrypt all remaining bytes in data into their shuffled version
-		input = des.decrypt(data[-cookie_size:])
+        cookie = des.decrypt(data[-cookie_size:])
         
         # now unshuffle it
         def unshuff(data, shuf):
@@ -307,7 +381,9 @@ class EreaderProcessor:
                 r[j] = data[i]
             assert len("".join(r)) == len(data)
             return "".join(r)
-		r = unshuff(input[0:-8], cookie_shuf)
+        r = unshuff(cookie[0:-8], cookie_shuf)
+        if DEBUG:
+            file("mainrecord.dat", 'wb').write(r)
         
         def fixUsername(s):
             r = ''
@@ -316,13 +392,10 @@ class EreaderProcessor:
                     r += c
             return r
         
-		user_key = struct.pack('>LL', binascii.crc32(fixUsername(username)) & 0xffffffff, binascii.crc32(creditcard[-8:])& 0xffffffff)
         # second key is made of of user private info i.e. name and crediticard info
+        user_key = struct.pack('>LL', binascii.crc32(fixUsername(username)) & 0xffffffff, binascii.crc32(creditcard[-8:])& 0xffffffff)
         self.user_key = user_key
         
-		if DEBUG:
-			file("mainrecord.dat", 'wb').write(r)
-
         # the unshuffled data is examined to find record information
         drm_sub_version = struct.unpack('>H', r[0:2])[0]
         self.num_text_pages = struct.unpack('>H', r[2:4])[0] - 1
@@ -339,8 +412,8 @@ class EreaderProcessor:
             self.num_bkinfo_pages = struct.unpack('>H', r[34:34+2])[0]
             self.first_bkinfo_page = struct.unpack('>H', r[32:32+2])[0]
 
-			self.num_fnote_pages = struct.unpack('>H', r[46:46+2])[0]
+            self.num_footnote_pages = struct.unpack('>H', r[46:46+2])[0]
-			self.first_fnote_page = struct.unpack('>H', r[44:44+2])[0]
+            self.first_footnote_page = struct.unpack('>H', r[44:44+2])[0]
 
             self.num_xtextsize_pages = struct.unpack('>H', r[54:54+2])[0]
             self.first_xtextsize_page = struct.unpack('>H', r[52:52+2])[0]
@@ -357,15 +430,15 @@ class EreaderProcessor:
             self.num_chapter_pages = 0
             self.num_link_pages = 0
             self.num_bkinfo_pages = 0
-			self.num_fnote_pages = 0
+            self.num_footnote_pages = 0
             self.num_xtextsize_pages = 0
             self.num_sidebar_pages = 0
-			self.first_chapter_pages = -1
+            self.first_chapter_page = -1
-			self.first_link_pages = -1
+            self.first_link_page = -1
-			self.first_bkinfo_pages = -1
+            self.first_bkinfo_page = -1
-			self.first_fnote_pages = -1
+            self.first_footnote_page = -1
-			self.first_xtextsize_pages = -1
+            self.first_xtextsize_page = -1
-			self.first_sidebar_pages = -1
+            self.first_sidebar_page = -1
 
         if DEBUG:
             print "num_text_pages: %d" % self.num_text_pages
@@ -377,8 +450,8 @@ class EreaderProcessor:
             print "num_image_pages: %d" % self.num_image_pages
             print "first_image_page: %d" % self.first_image_page
 
-			print "num_fnote_pages: %d" % self.num_fnote_pages
+            print "num_footnote_pages: %d" % self.num_footnote_pages
-			print "first_fnote_page: %d" % self.first_fnote_page
+            print "first_footnote_page: %d" % self.first_footnote_page
 
             print "num_link_pages: %d" % self.num_link_pages
             print "first_link_page: %d" % self.first_link_page
@@ -392,6 +465,9 @@ class EreaderProcessor:
             print "num_sidebar_pages: %d" % self.num_sidebar_pages
             print "first_sidebar_page: %d" % self.first_sidebar_page
 
+        logging.debug('self.num_text_pages %d', self.num_text_pages)
+        logging.debug('self.num_footnote_pages %d, self.first_footnote_page %d', self.num_footnote_pages , self.first_footnote_page)
+        
         self.flags = struct.unpack('>L', r[4:8])[0]
         reqd_flags = (1<<9) | (1<<7) | (1<<10)
         if (self.flags & reqd_flags) != reqd_flags:
@@ -399,8 +475,12 @@ class EreaderProcessor:
             raise ValueError('incompatible eReader file')
         # the user_key is used to unpack the encrypted key which is stored in the unshuffled data
         des = Des(fixKey(user_key))
-		
+        if version == 259:
-		if version == 260:
+            if drm_sub_version != 7:
+                raise ValueError('incorrect eReader version %d (error 3)' % drm_sub_version)
+            encrypted_key_sha = r[44:44+20]
+            encrypted_key = r[64:64+8]
+        elif version == 260:
             if drm_sub_version != 13:
                 raise ValueError('incorrect eReader version %d (error 3)' % drm_sub_version)
             encrypted_key = r[44:44+8]
@@ -412,10 +492,9 @@ class EreaderProcessor:
         # the decrypted version of encrypted_key is the content_key
         self.content_key = des.decrypt(encrypted_key)
         
-		if sha.new(self.content_key).digest() != encrypted_key_sha:
+        if sha1(self.content_key).digest() != encrypted_key_sha:
             raise ValueError('Incorrect Name and/or Credit Card')
             
-		
     def getNumImages(self):
         return self.num_image_pages
         
@@ -425,6 +504,7 @@ class EreaderProcessor:
         data = sect[62:]
         return sanitizeFileName(name), data
         
+    
     def getChapterNamePMLOffsetData(self):
         cv = ''
         if self.num_chapter_pages > 0:
@@ -476,24 +556,25 @@ class EreaderProcessor:
         des = Des(fixKey(self.content_key))
         r = ''
         for i in xrange(self.num_text_pages):
+            logging.debug('get page %d', i)
             r += zlib.decompress(des.decrypt(self.section_reader(1 + i)))
              
         # now handle footnotes pages
-                if self.num_fnote_pages > 0:
+        if self.num_footnote_pages > 0:
             # the first record of the footnote section must pass through the Xor Table to make it useful
-			sect = self.section_reader(self.first_fnote_page)
+            sect = self.section_reader(self.first_footnote_page)
             fnote_ids = deXOR(sect, 0, self.xortable)
             p = 0
             # the remaining records of the footnote sections need to be decoded with the content_key and zlib inflated
             des = Des(fixKey(self.content_key))
             r += '\\w="100%"'
             r += 'Footnotes\p'
-			for i in xrange(1,self.num_fnote_pages):
+            for i in xrange(1,self.num_footnote_pages):
                 id_len = ord(fnote_ids[2])
                 id = fnote_ids[3:3+id_len]
                 fmarker='\Q="%s"' % id
                 r+=fmarker
-				r += zlib.decompress(des.decrypt(self.section_reader(self.first_fnote_page + i)))
+                r += zlib.decompress(des.decrypt(self.section_reader(self.first_footnote_page + i)))
                 r += '\n'
                 fnote_ids = fnote_ids[id_len+4:]
 
@@ -501,8 +582,7 @@ class EreaderProcessor:
 
         return r
 
-
+class PmlConverter(object):
-class PmlConverter:
     def __init__(self, s):
         self.s = s
         self.pos = 0
@@ -552,12 +632,37 @@ class PmlConverter:
         return None, None, None
     def linkPrinter(link):
         return '<a href="%s">' % link
-	def ilinkPrinter(link):
+    def footnotePrinter(link):
-		return '<a href="#%s">' % link
+        return '<a href="#%s">' % link ## TODO may need to cgi.escape name....
+    def LinePrinter(link):
+        return '<hr width="%s">' % link
+    def ChapterTitle(link):
+        print "Nonfatal Error: ChapterTitle not implemented."
+        return '<!-- ChapterTitle %s -->' %link
+    def IndentPercent(link):
+        print "Nonfatal Error: IndentPercent not implemented."
+        return '<!-- IndentPercent: %s -->' %link
+    def NormalFont(link):
+        print "Nonfatal Error: NormalFont not implemented."
+        return '<!-- NormalFont %s -->' %link
+    def StdFont(link):
+        print "Nonfatal Error: StdFont not implemented."
+        return '<!-- StdFont: %s -->' %link
+    def SingleBackslash(link):
+        print "Nonfatal Error: SingleBackslash not implemented."
+        return '<!-- SingleBackslash: %s -->' %link
+    def SoftHyphen(link):
+        print "Nonfatal Error: SoftHyphen not implemented."
+        return '<!-- SoftHyphen: %s -->' %link
+    def ReferenceIndexItem(link):
+        print "Nonfatal Error: ReferenceIndexItem not implemented."
+        return '<!-- CReferenceIndexItem: %s -->' %link
 
+    
+    # See http://wiki.mobileread.com/wiki/PML#Palm_Markup_Language
     html_tags = {
-		'c' : ('<p align="center">', '</p>'),
+        'c' : ('<div class="center">', '</div>\n'),
-		'r' : ('<p align="right">', '</p>'),
+        'r' : ('<div class="right">', '</div>\n'),
         'i' : ('<i>', '</i>'),
         'u' : ('<u>', '</u>'),
         'b' : ('<strong>', '</strong>'),
@@ -567,17 +672,33 @@ class PmlConverter:
         't' : ('', ''),
         'Sb' : ('<sub>', '</sub>'),
         'Sp' : ('<sup>', '</sup>'),
-		'X0' : ('<h1>', '</h1>'),
+        'X0' : ('<h1>', '</h1>\n'),
-		'X1' : ('<h2>', '</h2>'),
+        'X1' : ('<h2>', '</h2>\n'),
-		'X2' : ('<h3>', '</h3>'),
+        'X2' : ('<h3>', '</h3>\n'),
-		'X3' : ('<h4>', '</h4>'),
+        'X3' : ('<h4>', '</h4>\n'),
-		'X4' : ('<h5>', '</h5>'),
+        'X4' : ('<h5>', '</h5>\n'),
         'l' : ('<font size="+2">', '</font>'),
         'q' : (linkPrinter, '</a>'),
-		'Fn' : (ilinkPrinter, '</a>'),
+        'Fn' : (footnotePrinter, '</a>'),
+        'w' : (LinePrinter, ''),
+        #'m' : handled in if block, 
+        #'Q' : handled in if block, 
+        #'a' : handled in if block, 
+        #'U' : handled in if block, 
+        'x' : ('<h1 class="breakbefore">', '</h1>\n'),
+        'Cn' : (ChapterTitle, ''),
+        'T' : (IndentPercent, ''),
+        'n' : (NormalFont, ''),
+        #'s' : (StdFont, ''),
+        's' : ('', ''),
+        'k' : ('<span style="font-variant: small-caps;">', '</span>'), # NOTE some pdb's then go ahead and use uppercase letters - which doesn't format the way one would expect (perhaps post process the output with html dom and lower case if only upper case letters are found?)
+        '\\' : (SingleBackslash, ''),
+        '-' : (SoftHyphen, ''),
+        'I' : (ReferenceIndexItem, ''),
+        'Sd' : (footnotePrinter, '</a>'), ## untested
     }
     html_one_tags = {
-		'p' : '<br><br>'
+        'p' : '<p class="breakafter">&nbsp;</p>\n'
     }
     pml_chars = {
         160 : '&nbsp;',130 : '&#8212;',131: '&#402;',132: '&#8222;',
@@ -588,7 +709,7 @@ class PmlConverter:
         156: '&#339;',159: '&#376;'
     }
     def process(self):
-		final = '<html><body>\n'
+        final = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">\n<html>\n<head>\n<meta http-equiv="content-type" content="text/html; charset=windows-1252">\n<style type="text/css">\ndiv.center { text-align:center; }\ndiv.right { text-align:right; }\n.breakbefore { page-break-before: always; }\n.breakafter { page-break-after: always; }\n</style>\n</head>\n<body>\n'
         in_tags = []
         def makeText(s):
             s = s.replace('&', '&amp;')
@@ -632,7 +753,9 @@ class PmlConverter:
                 if cmd in self.html_one_tags:
                     final += self.html_one_tags[cmd]
                 if cmd == 'm':
-					final += '<img src="%s">' % attr
+                    unquotedimagepath = bookname + "_img/" + attr
+                    imagepath = urllib.quote( unquotedimagepath )
+                    final += '<img src="%s" alt="">' % imagepath
                 if cmd == 'Q':
                     final += '<a name="%s" id="%s"> </a>' % (attr, attr)
                 if cmd == 'a':
@@ -653,15 +776,21 @@ def convertEreaderToHtml(infile, name, cc, outdir):
     sect = Sectionizer(infile, 'PNRdPPrs')
     er = EreaderProcessor(sect.loadSection, name, cc)
 
+    if er.getNumImages() > 0:
+        imagedir = bookname + "_img"
+        imagedirpath = os.path.join(outdir,imagedir)
+        if not os.path.exists(imagedirpath):
+            os.makedirs(imagedirpath)
         for i in xrange(er.getNumImages()):
             name, contents = er.getImage(i)
-		file(os.path.join(outdir, name), 'wb').write(contents)
+            file(os.path.join(imagedirpath, name), 'wb').write(contents)
 
-        rawpml = er.getText()
+    pml_string = er.getText()
-	file(os.path.join(outdir, 'book.pml'),'wb').write(rawpml)
+    pml = PmlConverter(pml_string)
-
+    pmlfilename = bookname + ".pml"
-        pml = PmlConverter(rawpml)
+    htmlfilename = bookname + ".html"
-	file(os.path.join(outdir, 'book.html'),'wb').write(pml.process())
+    file(os.path.join(outdir, pmlfilename),'wb').write(pml_string)
+    file(os.path.join(outdir, htmlfilename),'wb').write(pml.process())
     
     ts = er.getExpandedTextSizesData()
     file(os.path.join(outdir, 'xtextsizes.dat'), 'wb').write(ts)
@@ -675,19 +804,45 @@ def convertEreaderToHtml(infile, name, cc, outdir):
     lv = er.getLinkNamePMLOffsetData()
     file(os.path.join(outdir, 'links.dat'), 'wb').write(lv)
 
+def main(argv=None):
+    global bookname
+    if argv is None:
+        argv = sys.argv
     
-print "eReader2Html v0.05. Copyright (c) 2008 The Dark Reverser"
+    print "eReader2Html v%s. Copyright (c) 2008 The Dark Reverser" % __version__
-if len(sys.argv)!=5:
+
-	print "Converts eReader books to HTML"
+    if len(argv)!=4 and len(argv)!=5:
+        print "Converts DRMed eReader books to PML Source and HTML"
         print "Usage:"
-	print "  ereader2html infile.pdb outdir \"your name\" credit_card_number "
+        print "  ereader2html infile.pdb [outdir] \"your name\" credit_card_number "
         print "Note:"
+        print "  if ommitted, outdir defaults based on 'infile.pdb'"
         print "  It's enough to enter the last 8 digits of the credit card number"
     else:
-	infile, outdir, name, cc = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
+        if len(argv)==4:
+            infile, name, cc = argv[1], argv[2], argv[3]
+            outdir = infile[:-4] + '_Source'
+        elif len(argv)==5:
+            infile, outdir, name, cc = argv[1], argv[2], argv[3], argv[4]
+        bookname = os.path.splitext(os.path.basename(infile))[0]
+        
         try:
             print "Processing..."
+            import time
+            start_time = time.time()
             convertEreaderToHtml(infile, name, cc, outdir)
+            end_time = time.time()
+            search_time = end_time - start_time
+            print 'elapsed time: %.2f seconds' % (search_time, ) 
+            print 'output in %s' % outdir
             print "done"
         except ValueError, e:
             print "Error: %s" % e
+
+if __name__ == "__main__":
+    #import cProfile
+    #command = """sys.exit(main())"""
+    #cProfile.runctx( command, globals(), locals(), filename="cprofile.profile" )
+    
+    sys.exit(main())
+