Update Hachoir library 1.3.3 to 1.3.4 (r1383)

lib/hachoir_core/__init__.py

@@ -1,2 +1,2 @@
-from lib.hachoir_core.version import VERSION as __version__, PACKAGE, WEBSITE, LICENSE
+from hachoir_core.version import VERSION as __version__, PACKAGE, WEBSITE, LICENSE
 

lib/hachoir_core/benchmark.py

@@ -1,5 +1,5 @@
-from lib.hachoir_core.tools import humanDurationNanosec
-from lib.hachoir_core.i18n import _
+from hachoir_core.tools import humanDurationNanosec
+from hachoir_core.i18n import _
 from math import floor
 from time import time
 

lib/hachoir_core/bits.py

@@ -3,8 +3,8 @@ Utilities to convert integers and binary strings to binary (number), binary
 string, number, hexadecimal, etc.
 """
 
-from lib.hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
-from lib.hachoir_core.compatibility import reversed
+from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN, MIDDLE_ENDIAN
+from hachoir_core.compatibility import reversed
 from itertools import chain, repeat
 from struct import calcsize, unpack, error as struct_error
 
@@ -30,6 +30,28 @@ def swap32(value):
          | ((value & 0x00FF0000L) >> 8) \
          | ((value & 0xFF000000L) >> 24)
 
+def arrswapmid(data):
+    r"""
+    Convert an array of characters from middle-endian to big-endian and vice-versa.
+
+    >>> arrswapmid("badcfehg")
+    ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']
+    """
+    assert len(data)%2 == 0
+    ret = ['']*len(data)
+    ret[1::2] = data[0::2]
+    ret[0::2] = data[1::2]
+    return ret
+
+def strswapmid(data):
+    r"""
+    Convert raw data from middle-endian to big-endian and vice-versa.
+
+    >>> strswapmid("badcfehg")
+    'abcdefgh'
+    """
+    return ''.join(arrswapmid(data))
+
 def bin2long(text, endian):
     """
     Convert binary number written in a string into an integer.
@@ -45,9 +67,10 @@ def bin2long(text, endian):
     assert endian in (LITTLE_ENDIAN, BIG_ENDIAN)
     bits = [ (ord(character)-ord("0")) \
         for character in text if character in "01" ]
-    assert len(bits) != 0
     if endian is not BIG_ENDIAN:
-        bits = reversed(bits)
+        bits = bits[::-1]
+    size = len(bits)
+    assert 0 < size
     value = 0
     for bit in bits:
         value *= 2
@@ -142,7 +165,7 @@ def long2raw(value, endian, size=None):
     '\x19\x12\x00\x00'
     """
     assert (not size and 0 < value) or (0 <= value)
-    assert endian in (LITTLE_ENDIAN, BIG_ENDIAN)
+    assert endian in (LITTLE_ENDIAN, BIG_ENDIAN, MIDDLE_ENDIAN)
     text = []
     while (value != 0 or text == ""):
         byte = value % 256
@@ -153,13 +176,15 @@ def long2raw(value, endian, size=None):
     else:
         need = 0
     if need:
-        if endian is BIG_ENDIAN:
-            text = chain(repeat("\0", need), reversed(text))
-        else:
+        if endian is LITTLE_ENDIAN:
             text = chain(text, repeat("\0", need))
+        else:
+            text = chain(repeat("\0", need), reversed(text))
     else:
-        if endian is BIG_ENDIAN:
+        if endian is not LITTLE_ENDIAN:
             text = reversed(text)
+    if endian is MIDDLE_ENDIAN:
+        text = arrswapmid(text)
     return "".join(text)
 
 def long2bin(size, value, endian, classic_mode=False):
@@ -257,6 +282,8 @@ def str2long(data, endian):
     True
     >>> str2long("\xff\xff\xff\xff\xff\xff\xff\xff", BIG_ENDIAN) == (2**64-1)
     True
+    >>> str2long("\x0b\x0a\x0d\x0c", MIDDLE_ENDIAN) == 0x0a0b0c0d
+    True
     """
     assert 1 <= len(data) <= 32   # arbitrary limit: 256 bits
     try:
@@ -264,14 +291,15 @@ def str2long(data, endian):
     except KeyError:
         pass
 
-    assert endian in (BIG_ENDIAN, LITTLE_ENDIAN)
+    assert endian in (BIG_ENDIAN, LITTLE_ENDIAN, MIDDLE_ENDIAN)
     shift = 0
     value = 0
     if endian is BIG_ENDIAN:
         data = reversed(data)
+    elif endian is MIDDLE_ENDIAN:
+        data = reversed(strswapmid(data))
     for character in data:
         byte = ord(character)
         value += (byte << shift)
         shift += 8
     return value
-

lib/hachoir_core/cmd_line.py

@@ -1,8 +1,8 @@
 from optparse import OptionGroup
-from lib.hachoir_core.log import log
-from lib.hachoir_core.i18n import _, getTerminalCharset
-from lib.hachoir_core.tools import makePrintable
-import lib.hachoir_core.config as config
+from hachoir_core.log import log
+from hachoir_core.i18n import _, getTerminalCharset
+from hachoir_core.tools import makePrintable
+import hachoir_core.config as config
 
 def getHachoirOptions(parser):
     """

lib/hachoir_core/dict.py

@@ -2,8 +2,8 @@
 Dictionnary classes which store values order.
 """
 
-from lib.hachoir_core.error import HachoirError
-from lib.hachoir_core.i18n import _
+from hachoir_core.error import HachoirError
+from hachoir_core.i18n import _
 
 class UniqKeyError(HachoirError):
     """

lib/hachoir_core/endian.py

@@ -2,14 +2,15 @@
 Constant values about endian.
 """
 
-from lib.hachoir_core.i18n import _
+from hachoir_core.i18n import _
 
 BIG_ENDIAN = "ABCD"
 LITTLE_ENDIAN = "DCBA"
+MIDDLE_ENDIAN = "BADC"
 NETWORK_ENDIAN = BIG_ENDIAN
 
 endian_name = {
     BIG_ENDIAN: _("Big endian"),
     LITTLE_ENDIAN: _("Little endian"),
+    MIDDLE_ENDIAN: _("Middle endian"),
 }
-

lib/hachoir_core/error.py

@@ -2,8 +2,8 @@
 Functions to display an error (error, warning or information) message.
 """
 
-from lib.hachoir_core.log import log
-from lib.hachoir_core.tools import makePrintable
+from hachoir_core.log import log
+from hachoir_core.tools import makePrintable
 import sys, traceback
 
 def getBacktrace(empty="Empty backtrace."):

lib/hachoir_core/field/__init__.py

@@ -1,44 +1,44 @@
 # Field classes
-from lib.hachoir_core.field.field import Field, FieldError, MissingField, joinPath
-from lib.hachoir_core.field.bit_field import Bit, Bits, RawBits
-from lib.hachoir_core.field.byte_field import Bytes, RawBytes
-from lib.hachoir_core.field.sub_file import SubFile, CompressedField
-from lib.hachoir_core.field.character import Character
-from lib.hachoir_core.field.integer import (
+from hachoir_core.field.field import Field, FieldError, MissingField, joinPath
+from hachoir_core.field.bit_field import Bit, Bits, RawBits
+from hachoir_core.field.byte_field import Bytes, RawBytes
+from hachoir_core.field.sub_file import SubFile, CompressedField
+from hachoir_core.field.character import Character
+from hachoir_core.field.integer import (
     Int8,  Int16,  Int24,  Int32,  Int64,
     UInt8, UInt16, UInt24, UInt32, UInt64,
     GenericInteger)
-from lib.hachoir_core.field.enum import Enum
-from lib.hachoir_core.field.string_field import (GenericString,
+from hachoir_core.field.enum import Enum
+from hachoir_core.field.string_field import (GenericString,
     String, CString, UnixLine,
     PascalString8, PascalString16, PascalString32)
-from lib.hachoir_core.field.padding import (PaddingBits, PaddingBytes,
+from hachoir_core.field.padding import (PaddingBits, PaddingBytes,
     NullBits, NullBytes)
 
 # Functions
-from lib.hachoir_core.field.helper import (isString, isInteger,
+from hachoir_core.field.helper import (isString, isInteger,
     createPaddingField, createNullField, createRawField,
     writeIntoFile, createOrphanField)
 
 # FieldSet classes
-from lib.hachoir_core.field.fake_array import FakeArray
-from lib.hachoir_core.field.basic_field_set import (BasicFieldSet,
+from hachoir_core.field.fake_array import FakeArray
+from hachoir_core.field.basic_field_set import (BasicFieldSet,
     ParserError, MatchError)
-from lib.hachoir_core.field.generic_field_set import GenericFieldSet
-from lib.hachoir_core.field.seekable_field_set import SeekableFieldSet, RootSeekableFieldSet
-from lib.hachoir_core.field.field_set import FieldSet
-from lib.hachoir_core.field.static_field_set import StaticFieldSet
-from lib.hachoir_core.field.parser import Parser
-from lib.hachoir_core.field.vector import GenericVector, UserVector
+from hachoir_core.field.generic_field_set import GenericFieldSet
+from hachoir_core.field.seekable_field_set import SeekableFieldSet, RootSeekableFieldSet
+from hachoir_core.field.field_set import FieldSet
+from hachoir_core.field.static_field_set import StaticFieldSet
+from hachoir_core.field.parser import Parser
+from hachoir_core.field.vector import GenericVector, UserVector
 
 # Complex types
-from lib.hachoir_core.field.float import Float32, Float64, Float80
-from lib.hachoir_core.field.timestamp import (GenericTimestamp,
+from hachoir_core.field.float import Float32, Float64, Float80
+from hachoir_core.field.timestamp import (GenericTimestamp,
     TimestampUnix32, TimestampUnix64, TimestampMac32, TimestampUUID60, TimestampWin64,
     DateTimeMSDOS32, TimeDateMSDOS32, TimedeltaWin64)
 
 # Special Field classes
-from lib.hachoir_core.field.link import Link, Fragment
+from hachoir_core.field.link import Link, Fragment
 
 available_types = (
     Bit, Bits, RawBits,

lib/hachoir_core/field/basic_field_set.py

@@ -1,7 +1,7 @@
-from lib.hachoir_core.field import Field, FieldError
-from lib.hachoir_core.stream import InputStream
-from lib.hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
-from lib.hachoir_core.event_handler import EventHandler
+from hachoir_core.field import Field, FieldError
+from hachoir_core.stream import InputStream
+from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN, MIDDLE_ENDIAN
+from hachoir_core.event_handler import EventHandler
 
 class ParserError(FieldError):
     """
@@ -60,7 +60,7 @@ class BasicFieldSet(Field):
             self._global_event_handler = None
 
         # Sanity checks (post-conditions)
-        assert self.endian in (BIG_ENDIAN, LITTLE_ENDIAN)
+        assert self.endian in (BIG_ENDIAN, LITTLE_ENDIAN, MIDDLE_ENDIAN)
         if (self._size is not None) and (self._size <= 0):
             raise ParserError("Invalid parser '%s' size: %s" % (self.path, self._size))
 

lib/hachoir_core/field/bit_field.py

@@ -5,9 +5,9 @@ Bit sized classes:
 - RawBits: unknown content with a size in bits.
 """
 
-from lib.hachoir_core.field import Field
-from lib.hachoir_core.i18n import _
-from lib.hachoir_core import config
+from hachoir_core.field import Field
+from hachoir_core.i18n import _
+from hachoir_core import config
 
 class RawBits(Field):
     """

lib/hachoir_core/field/byte_field.py

@@ -3,10 +3,10 @@ Very basic field: raw content with a size in byte. Use this class for
 unknown content.
 """
 
-from lib.hachoir_core.field import Field, FieldError
-from lib.hachoir_core.tools import makePrintable
-from lib.hachoir_core.bits import str2hex
-from lib.hachoir_core import config
+from hachoir_core.field import Field, FieldError
+from hachoir_core.tools import makePrintable
+from hachoir_core.bits import str2hex
+from hachoir_core import config
 
 MAX_LENGTH = (2**64)
 

lib/hachoir_core/field/character.py

@@ -2,9 +2,9 @@
 Character field class: a 8-bit character
 """
 
-from lib.hachoir_core.field import Bits
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.tools import makePrintable
+from hachoir_core.field import Bits
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.tools import makePrintable
 
 class Character(Bits):
     """
@@ -24,4 +24,3 @@ class Character(Bits):
 
     def createDisplay(self):
         return makePrintable(self.value, "ASCII", quote="'", to_unicode=True)
-

lib/hachoir_core/field/enum.py

@@ -1,7 +1,7 @@
 def Enum(field, enum, key_func=None):
     """
     Enum is an adapter to another field: it will just change its display
-    attribute. It uses a dictionnary to associate a value to another.
+    attribute. It uses a dictionary to associate a value to another.
 
     key_func is an optional function with prototype "def func(key)->key"
     which is called to transform key.
@@ -23,4 +23,3 @@ def Enum(field, enum, key_func=None):
     field.createDisplay = createDisplay
     field.getEnum = lambda: enum
     return field
-

lib/hachoir_core/field/fake_array.py

@@ -1,5 +1,5 @@
 import itertools
-from lib.hachoir_core.field import MissingField
+from hachoir_core.field import MissingField
 
 class FakeArray:
     """

lib/hachoir_core/field/field.py

@@ -2,12 +2,12 @@
 Parent of all (field) classes in Hachoir: Field.
 """
 
-from lib.hachoir_core.compatibility import reversed
-from lib.hachoir_core.stream import InputFieldStream
-from lib.hachoir_core.error import HachoirError, HACHOIR_ERRORS
-from lib.hachoir_core.log import Logger
-from lib.hachoir_core.i18n import _
-from lib.hachoir_core.tools import makePrintable
+from hachoir_core.compatibility import reversed
+from hachoir_core.stream import InputFieldStream
+from hachoir_core.error import HachoirError, HACHOIR_ERRORS
+from hachoir_core.log import Logger
+from hachoir_core.i18n import _
+from hachoir_core.tools import makePrintable
 from weakref import ref as weakref_ref
 
 class FieldError(HachoirError):
@@ -70,6 +70,8 @@ class Field(Logger):
         assert issubclass(parent.__class__, Field)
         assert (size is None) or (0 <= size)
         self._parent = parent
+        if not name:
+            raise ValueError("empty field name")
         self._name = name
         self._address = parent.nextFieldAddress()
         self._size = size
@@ -166,7 +168,7 @@ class Field(Logger):
             return '/'
         names = []
         field = self
-        while field:
+        while field is not None:
             names.append(field._name)
             field = field._parent
         names[-1] = ''

lib/hachoir_core/field/field_set.py

@@ -1,4 +1,4 @@
-from lib.hachoir_core.field import BasicFieldSet, GenericFieldSet
+from hachoir_core.field import BasicFieldSet, GenericFieldSet
 
 class FieldSet(GenericFieldSet):
     def __init__(self, parent, name, *args, **kw):

lib/hachoir_core/field/float.py

@@ -1,5 +1,5 @@
-from lib.hachoir_core.field import Bit, Bits, FieldSet
-from lib.hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
+from hachoir_core.field import Bit, Bits, FieldSet
+from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
 import struct
 
 # Make sure that we use right struct types
@@ -85,15 +85,15 @@ def floatFactory(name, format, mantissa_bits, exponent_bits, doc):
     cls.__name__ = name
     return cls
 
-# 32-bit float (standart: IEEE 754/854)
+# 32-bit float (standard: IEEE 754/854)
 Float32 = floatFactory("Float32", "f", 23, 8,
     "Floating point number: format IEEE 754 int 32 bit")
 
-# 64-bit float (standart: IEEE 754/854)
+# 64-bit float (standard: IEEE 754/854)
 Float64 = floatFactory("Float64", "d", 52, 11,
     "Floating point number: format IEEE 754 in 64 bit")
 
-# 80-bit float (standart: IEEE 754/854)
+# 80-bit float (standard: IEEE 754/854)
 Float80 = floatFactory("Float80", None, 64, 15,
     "Floating point number: format IEEE 754 in 80 bit")
 

lib/hachoir_core/field/generic_field_set.py

@@ -1,9 +1,9 @@
-from lib.hachoir_core.field import (MissingField, BasicFieldSet, Field, ParserError,
+from hachoir_core.field import (MissingField, BasicFieldSet, Field, ParserError,
     createRawField, createNullField, createPaddingField, FakeArray)
-from lib.hachoir_core.dict import Dict, UniqKeyError
-from lib.hachoir_core.error import HACHOIR_ERRORS
-from lib.hachoir_core.tools import lowerBound
-import lib.hachoir_core.config as config
+from hachoir_core.dict import Dict, UniqKeyError
+from hachoir_core.error import HACHOIR_ERRORS
+from hachoir_core.tools import lowerBound, makeUnicode
+import hachoir_core.config as config
 
 class GenericFieldSet(BasicFieldSet):
     """
@@ -12,8 +12,8 @@ class GenericFieldSet(BasicFieldSet):
     document).
 
     Class attributes:
-    - endian: Bytes order (L{BIG_ENDIAN} or L{LITTLE_ENDIAN}). Optional if the
-      field set has a parent ;
+    - endian: Bytes order (L{BIG_ENDIAN}, L{LITTLE_ENDIAN} or L{MIDDLE_ENDIAN}).
+      Optional if the field set has a parent ;
     - static_size: (optional) Size of FieldSet in bits. This attribute should
       be used in parser of constant size.
 
@@ -310,7 +310,7 @@ class GenericFieldSet(BasicFieldSet):
         """
         if self._size is None or not self.autofix:
             return False
-        self.warning(unicode(exception))
+        self.warning(makeUnicode(exception))
         return self._fixLastField()
 
     def _feedUntil(self, field_name):

lib/hachoir_core/field/helper.py

@@ -1,9 +1,9 @@
-from lib.hachoir_core.field import (FieldError,
+from hachoir_core.field import (FieldError,
     RawBits, RawBytes,
     PaddingBits, PaddingBytes,
     NullBits, NullBytes,
     GenericString, GenericInteger)
-from lib.hachoir_core.stream import FileOutputStream
+from hachoir_core.stream import FileOutputStream
 
 def createRawField(parent, size, name="raw[]", description=None):
     if size <= 0:

lib/hachoir_core/field/integer.py

@@ -4,15 +4,15 @@ Integer field classes:
 - Int8, Int16, Int24, Int32, Int64: signed integer of 8, 16, 32, 64 bits.
 """
 
-from lib.hachoir_core.field import Bits, FieldError
+from hachoir_core.field import Bits, FieldError
 
 class GenericInteger(Bits):
     """
     Generic integer class used to generate other classes.
     """
     def __init__(self, parent, name, signed, size, description=None):
-        if not (8 <= size <= 256):
-            raise FieldError("Invalid integer size (%s): have to be in 8..256" % size)
+        if not (8 <= size <= 16384):
+            raise FieldError("Invalid integer size (%s): have to be in 8..16384" % size)
         Bits.__init__(self, parent, name, size, description)
         self.signed = signed
 

lib/hachoir_core/field/link.py

@@ -1,5 +1,5 @@
-from lib.hachoir_core.field import Field, FieldSet, ParserError, Bytes, MissingField
-from lib.hachoir_core.stream import FragmentedStream
+from hachoir_core.field import Field, FieldSet, ParserError, Bytes, MissingField
+from hachoir_core.stream import FragmentedStream
 
 
 class Link(Field):

lib/hachoir_core/field/new_seekable_field_set.py

@@ -1,5 +1,5 @@
-from lib.hachoir_core.field import BasicFieldSet, GenericFieldSet, ParserError, createRawField
-from lib.hachoir_core.error import HACHOIR_ERRORS
+from hachoir_core.field import BasicFieldSet, GenericFieldSet, ParserError, createRawField
+from hachoir_core.error import HACHOIR_ERRORS
 
 # getgaps(int, int, [listof (int, int)]) -> generator of (int, int)
 # Gets all the gaps not covered by a block in `blocks` from `start` for `length` units.

lib/hachoir_core/field/padding.py

@@ -1,6 +1,6 @@
-from lib.hachoir_core.field import Bits, Bytes
-from lib.hachoir_core.tools import makePrintable, humanFilesize
-from lib.hachoir_core import config
+from hachoir_core.field import Bits, Bytes
+from hachoir_core.tools import makePrintable, humanFilesize
+from hachoir_core import config
 
 class PaddingBits(Bits):
     """

lib/hachoir_core/field/parser.py

@@ -1,13 +1,13 @@
-from lib.hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
-from lib.hachoir_core.field import GenericFieldSet
-from lib.hachoir_core.log import Logger
-import lib.hachoir_core.config as config
+from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN, MIDDLE_ENDIAN
+from hachoir_core.field import GenericFieldSet
+from hachoir_core.log import Logger
+import hachoir_core.config as config
 
 class Parser(GenericFieldSet):
     """
     A parser is the root of all other fields. It create first level of fields
     and have special attributes and methods:
-    - endian: Byte order (L{BIG_ENDIAN} or L{LITTLE_ENDIAN}) of input data ;
+    - endian: Byte order (L{BIG_ENDIAN}, L{LITTLE_ENDIAN} or L{MIDDLE_ENDIAN}) of input data ;
     - stream: Data input stream (set in L{__init__()}) ;
     - size: Field set size will be size of input stream.
     """
@@ -21,7 +21,7 @@ class Parser(GenericFieldSet):
         """
         # Check arguments
         assert hasattr(self, "endian") \
-            and self.endian in (BIG_ENDIAN, LITTLE_ENDIAN)
+            and self.endian in (BIG_ENDIAN, LITTLE_ENDIAN, MIDDLE_ENDIAN)
 
         # Call parent constructor
         GenericFieldSet.__init__(self, None, "root", stream, description, stream.askSize(self))

lib/hachoir_core/field/seekable_field_set.py

@@ -1,182 +1,82 @@
-from lib.hachoir_core.field import Field, BasicFieldSet, FakeArray, MissingField, ParserError
-from lib.hachoir_core.tools import makeUnicode
-from lib.hachoir_core.error import HACHOIR_ERRORS
-from itertools import repeat
-import lib.hachoir_core.config as config
-
-class RootSeekableFieldSet(BasicFieldSet):
-    def __init__(self, parent, name, stream, description, size):
-        BasicFieldSet.__init__(self, parent, name, stream, description, size)
-        self._generator = self.createFields()
-        self._offset = 0
-        self._current_size = 0
-        if size:
-            self._current_max_size = size
-        else:
-            self._current_max_size = 0
-        self._field_dict = {}
-        self._field_array = []
-
-    def _feedOne(self):
-        assert self._generator
-        field = self._generator.next()
-        self._addField(field)
-        return field
-
-    def array(self, key):
-        return FakeArray(self, key)
-
-    def getFieldByAddress(self, address, feed=True):
-        for field in self._field_array:
-            if field.address <= address < field.address + field.size:
-                return field
-        for field in self._readFields():
-            if field.address <= address < field.address + field.size:
-                return field
-        return None
-
-    def _stopFeed(self):
-        self._size = self._current_max_size
-        self._generator = None
-    done = property(lambda self: not bool(self._generator))
-
-    def _getSize(self):
-        if self._size is None:
-            self._feedAll()
-        return self._size
-    size = property(_getSize)
-
-    def _getField(self, key, const):
-        field = Field._getField(self, key, const)
-        if field is not None:
-            return field
-        if key in self._field_dict:
-            return self._field_dict[key]
-        if self._generator and not const:
-            try:
-                while True:
-                    field = self._feedOne()
-                    if field.name == key:
-                        return field
-            except StopIteration:
-                self._stopFeed()
-            except HACHOIR_ERRORS, err:
-                self.error("Error: %s" % makeUnicode(err))
-                self._stopFeed()
-        return None
-
-    def getField(self, key, const=True):
-        if isinstance(key, (int, long)):
-            if key < 0:
-                raise KeyError("Key must be positive!")
-            if not const:
-                self.readFirstFields(key+1)
-            if len(self._field_array) <= key:
-                raise MissingField(self, key)
-            return self._field_array[key]
-        return Field.getField(self, key, const)
-
-    def _addField(self, field):
-        if field._name.endswith("[]"):
-            self.setUniqueFieldName(field)
-        if config.debug:
-            self.info("[+] DBG: _addField(%s)" % field.name)
-
-        if field._address != self._offset:
-            self.warning("Set field %s address to %s (was %s)" % (
-                field.path, self._offset//8, field._address//8))
-            field._address = self._offset
-        assert field.name not in self._field_dict
-
-        self._checkFieldSize(field)
-
-        self._field_dict[field.name] = field
-        self._field_array.append(field)
-        self._current_size += field.size
-        self._offset += field.size
-        self._current_max_size = max(self._current_max_size, field.address + field.size)
-
-    def _checkAddress(self, address):
-        if self._size is not None:
-            max_addr = self._size
-        else:
-            # FIXME: Use parent size
-            max_addr = self.stream.size
-        return address < max_addr
-
-    def _checkFieldSize(self, field):
-        size = field.size
-        addr = field.address
-        if not self._checkAddress(addr+size-1):
-            raise ParserError("Unable to add %s: field is too large" % field.name)
-
+from hachoir_core.field import BasicFieldSet, GenericFieldSet, ParserError, createRawField
+from hachoir_core.error import HACHOIR_ERRORS
+
+# getgaps(int, int, [listof (int, int)]) -> generator of (int, int)
+# Gets all the gaps not covered by a block in `blocks` from `start` for `length` units.
+def getgaps(start, length, blocks):
+    '''
+    Example:
+    >>> list(getgaps(0, 20, [(15,3), (6,2), (6,2), (1,2), (2,3), (11,2), (9,5)]))
+    [(0, 1), (5, 1), (8, 1), (14, 1), (18, 2)]
+    '''
+    # done this way to avoid mutating the original
+    blocks = sorted(blocks, key=lambda b: b[0])
+    end = start+length
+    for s, l in blocks:
+        if s > start:
+            yield (start, s-start)
+            start = s
+        if s+l > start:
+            start = s+l
+    if start < end:
+        yield (start, end-start)
+
+class RootSeekableFieldSet(GenericFieldSet):
     def seekBit(self, address, relative=True):
         if not relative:
             address -= self.absolute_address
         if address < 0:
             raise ParserError("Seek below field set start (%s.%s)" % divmod(address, 8))
-        if not self._checkAddress(address):
-            raise ParserError("Seek above field set end (%s.%s)" % divmod(address, 8))
-        self._offset = address
+        self._current_size = address
         return None
 
     def seekByte(self, address, relative=True):
         return self.seekBit(address*8, relative)
 
-    def readMoreFields(self, number):
-        return self._readMoreFields(xrange(number))
-
-    def _feedAll(self):
-        return self._readMoreFields(repeat(1))
-
-    def _readFields(self):
-        while True:
-            added = self._readMoreFields(xrange(1))
-            if not added:
-                break
-            yield self._field_array[-1]
-
-    def _readMoreFields(self, index_generator):
-        added = 0
-        if self._generator:
-            try:
-                for index in index_generator:
-                    self._feedOne()
-                    added += 1
-            except StopIteration:
-                self._stopFeed()
-            except HACHOIR_ERRORS, err:
-                self.error("Error: %s" % makeUnicode(err))
-                self._stopFeed()
-        return added
-
-    current_length = property(lambda self: len(self._field_array))
-    current_size = property(lambda self: self._offset)
-
-    def __iter__(self):
-        for field in self._field_array:
-            yield field
-        if self._generator:
-            try:
-                while True:
-                    yield self._feedOne()
-            except StopIteration:
-                self._stopFeed()
-                raise StopIteration
-
-    def __len__(self):
-        if self._generator:
-            self._feedAll()
-        return len(self._field_array)
-
-    def nextFieldAddress(self):
-        return self._offset
+    def _fixLastField(self):
+        """
+        Try to fix last field when we know current field set size.
+        Returns new added field if any, or None.
+        """
+        assert self._size is not None
+
+        # Stop parser
+        message = ["stop parser"]
+        self._field_generator = None
+
+        # If last field is too big, delete it
+        while self._size < self._current_size:
+            field = self._deleteField(len(self._fields)-1)
+            message.append("delete field %s" % field.path)
+        assert self._current_size <= self._size
+
+        blocks = [(x.absolute_address, x.size) for x in self._fields]
+        fields = []
+        self._size = max(self._size, max(a+b for a,b in blocks) - self.absolute_address)
+        for start, length in getgaps(self.absolute_address, self._size, blocks):
+            self.seekBit(start, relative=False)
+            field = createRawField(self, length, "unparsed[]")
+            self.setUniqueFieldName(field)
+            self._fields.append(field.name, field)
+            fields.append(field)
+            message.append("found unparsed segment: start %s, length %s" % (start, length))
+        self.seekBit(self._size + self.absolute_address, relative=False)
+        message = ", ".join(message)
+        if fields:
+            self.warning("[Autofix] Fix parser error: " + message)
+        return fields
+
+    def _stopFeeding(self):
+        new_field = None
+        if self._size is None:
+            if self._parent:
+                self._size = self._current_size
 
-    def getFieldIndex(self, field):
-        return self._field_array.index(field)
+        new_field = self._fixLastField()
+        self._field_generator = None
+        return new_field
 
 class SeekableFieldSet(RootSeekableFieldSet):
     def __init__(self, parent, name, description=None, size=None):
         assert issubclass(parent.__class__, BasicFieldSet)
         RootSeekableFieldSet.__init__(self, parent, name, parent.stream, description, size)
-

lib/hachoir_core/field/static_field_set.py

@@ -1,4 +1,4 @@
-from lib.hachoir_core.field import FieldSet, ParserError
+from hachoir_core.field import FieldSet, ParserError
 
 class StaticFieldSet(FieldSet):
     """
@@ -20,7 +20,7 @@ class StaticFieldSet(FieldSet):
         if cls._class is not cls.__name__:
             cls._class = cls.__name__
             cls.static_size = cls._computeStaticSize()
-        return object.__new__(cls)
+        return object.__new__(cls, *args, **kw)
 
     @staticmethod
     def _computeItemSize(item):

lib/hachoir_core/field/string_field.py

@@ -15,11 +15,11 @@ Note: For PascalStringXX, prefixed value is the number of bytes and not
       of characters!
 """
 
-from lib.hachoir_core.field import FieldError, Bytes
-from lib.hachoir_core.endian import LITTLE_ENDIAN, BIG_ENDIAN
-from lib.hachoir_core.tools import alignValue, makePrintable
-from lib.hachoir_core.i18n import guessBytesCharset, _
-from lib.hachoir_core import config
+from hachoir_core.field import FieldError, Bytes
+from hachoir_core.endian import LITTLE_ENDIAN, BIG_ENDIAN
+from hachoir_core.tools import alignValue, makePrintable
+from hachoir_core.i18n import guessBytesCharset, _
+from hachoir_core import config
 from codecs import BOM_UTF16_LE, BOM_UTF16_BE, BOM_UTF32_LE, BOM_UTF32_BE
 
 # Default charset used to convert byte string to Unicode

lib/hachoir_core/field/sub_file.py

@@ -1,6 +1,6 @@
-from lib.hachoir_core.field import Bytes
-from lib.hachoir_core.tools import makePrintable, humanFilesize
-from lib.hachoir_core.stream import InputIOStream
+from hachoir_core.field import Bytes
+from hachoir_core.tools import makePrintable, humanFilesize
+from hachoir_core.stream import InputIOStream
 
 class SubFile(Bytes):
     """

lib/hachoir_core/field/timestamp.py

@@ -1,7 +1,7 @@
-from lib.hachoir_core.tools import (humanDatetime, humanDuration,
+from hachoir_core.tools import (humanDatetime, humanDuration,
     timestampUNIX, timestampMac32, timestampUUID60,
     timestampWin64, durationWin64)
-from lib.hachoir_core.field import Bits, FieldSet
+from hachoir_core.field import Bits, FieldSet
 from datetime import datetime
 
 class GenericTimestamp(Bits):
@@ -32,7 +32,7 @@ def timestampFactory(cls_name, handler, size):
 
 TimestampUnix32 = timestampFactory("TimestampUnix32", timestampUNIX, 32)
 TimestampUnix64 = timestampFactory("TimestampUnix64", timestampUNIX, 64)
-TimestampMac32 = timestampFactory("TimestampUnix32", timestampMac32, 32)
+TimestampMac32 = timestampFactory("TimestampMac32", timestampMac32, 32)
 TimestampUUID60 = timestampFactory("TimestampUUID60", timestampUUID60, 60)
 TimestampWin64 = timestampFactory("TimestampWin64", timestampWin64, 64)
 

lib/hachoir_core/field/vector.py

@@ -1,4 +1,4 @@
-from lib.hachoir_core.field import Field, FieldSet, ParserError
+from hachoir_core.field import Field, FieldSet, ParserError
 
 class GenericVector(FieldSet):
     def __init__(self, parent, name, nb_items, item_class, item_name="item", description=None):

lib/hachoir_core/i18n.py

@@ -14,8 +14,8 @@ WARNING: Loading this module indirectly calls initLocale() which sets
          settings.
 """
 
-import lib.hachoir_core.config as config
-import lib.hachoir_core
+import hachoir_core.config as config
+import hachoir_core
 import locale
 from os import path
 import sys
@@ -133,7 +133,7 @@ def _initGettext():
         return (_dummy_gettext, _dummy_ngettext)
 
     # Gettext variables
-    package = lib.hachoir_core.PACKAGE
+    package = hachoir_core.PACKAGE
     locale_dir = path.join(path.dirname(__file__), "..", "locale")
 
     # Initialize gettext module

lib/hachoir_core/iso639.py

@@ -328,7 +328,6 @@ _ISO639 = (
     (u"Micmac", "mic", None),
     (u"Minangkabau", "min", None),
     (u"Mirandese", "mwl", None),
-    (u"Miscellaneous languages", "mis", None),
     (u"Mohawk", "moh", None),
     (u"Moksha", "mdf", None),
     (u"Moldavian", "mol", "mo"),
@@ -513,6 +512,7 @@ _ISO639 = (
     (u"Uighur", "uig", "ug"),
     (u"Ukrainian", "ukr", "uk"),
     (u"Umbundu", "umb", None),
+    (u"Uncoded languages", "mis", None),
     (u"Undetermined", "und", None),
     (u"Upper Sorbian", "hsb", None),
     (u"Urdu", "urd", "ur"),

lib/hachoir_core/language.py

@@ -1,4 +1,4 @@
-from lib.hachoir_core.iso639 import ISO639_2
+from hachoir_core.iso639 import ISO639_2
 
 class Language:
     def __init__(self, code):

lib/hachoir_core/log.py

@@ -1,6 +1,6 @@
 import os, sys, time
-import lib.hachoir_core.config as config
-from lib.hachoir_core.i18n import _
+import hachoir_core.config as config
+from hachoir_core.i18n import _
 
 class Log:
     LOG_INFO   = 0
@@ -75,7 +75,7 @@ class Log:
            level <= self.LOG_INFO and not config.verbose:
             return
         if config.debug:
-            from lib.hachoir_core.error import getBacktrace
+            from hachoir_core.error import getBacktrace
             backtrace = getBacktrace(None)
             if backtrace:
                 text += "\n\n" + backtrace

lib/hachoir_core/stream/__init__.py

@@ -1,11 +1,11 @@
-from lib.hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
-from lib.hachoir_core.stream.stream import StreamError
-from lib.hachoir_core.stream.input import (
+from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
+from hachoir_core.stream.stream import StreamError
+from hachoir_core.stream.input import (
         InputStreamError,
         InputStream, InputIOStream, StringInputStream,
         InputSubStream, InputFieldStream,
         FragmentedStream, ConcatStream)
-from lib.hachoir_core.stream.input_helper import FileInputStream, guessStreamCharset
-from lib.hachoir_core.stream.output import (OutputStreamError,
+from hachoir_core.stream.input_helper import FileInputStream, guessStreamCharset
+from hachoir_core.stream.output import (OutputStreamError,
         FileOutputStream, StringOutputStream, OutputStream)
 

lib/hachoir_core/stream/input.py

@@ -1,14 +1,14 @@
-from lib.hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
-from lib.hachoir_core.error import info
-from lib.hachoir_core.log import Logger
-from lib.hachoir_core.bits import str2long
-from lib.hachoir_core.i18n import getTerminalCharset
-from lib.hachoir_core.tools import lowerBound
-from lib.hachoir_core.i18n import _
-from os import dup, fdopen
+from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN, MIDDLE_ENDIAN
+from hachoir_core.error import info
+from hachoir_core.log import Logger
+from hachoir_core.bits import str2long
+from hachoir_core.i18n import getTerminalCharset
+from hachoir_core.tools import lowerBound
+from hachoir_core.i18n import _
+from hachoir_core.tools import alignValue
 from errno import ESPIPE
 from weakref import ref as weakref_ref
-from lib.hachoir_core.stream import StreamError
+from hachoir_core.stream import StreamError
 
 class InputStreamError(StreamError):
     pass
@@ -168,13 +168,20 @@ class InputStream(Logger):
         raise NotImplementedError
 
     def readBits(self, address, nbits, endian):
-        assert endian in (BIG_ENDIAN, LITTLE_ENDIAN)
-
-        shift, data, missing = self.read(address, nbits)
+        assert endian in (BIG_ENDIAN, LITTLE_ENDIAN, MIDDLE_ENDIAN)
+
+        if endian is MIDDLE_ENDIAN:
+            # read an aligned chunk of words
+            wordaddr, remainder = divmod(address, 16)
+            wordnbits = alignValue(remainder+nbits, 16)
+            _, data, missing = self.read(wordaddr*16, wordnbits)
+            shift = remainder
+        else:
+            shift, data, missing = self.read(address, nbits)
         if missing:
             raise ReadStreamError(nbits, address)
         value = str2long(data, endian)
-        if endian is BIG_ENDIAN:
+        if endian in (BIG_ENDIAN, MIDDLE_ENDIAN):
             value >>= len(data) * 8 - shift - nbits
         else:
             value >>= shift
@@ -404,6 +411,7 @@ class InputIOStream(InputStream):
 
     def file(self):
         if hasattr(self._input, "fileno"):
+            from os import dup, fdopen
             new_fd = dup(self._input.fileno())
             new_file = fdopen(new_fd, "r")
             new_file.seek(0)

lib/hachoir_core/stream/input_helper.py

@@ -1,5 +1,5 @@
-from lib.hachoir_core.i18n import getTerminalCharset, guessBytesCharset, _
-from lib.hachoir_core.stream import InputIOStream, InputSubStream, InputStreamError
+from hachoir_core.i18n import getTerminalCharset, guessBytesCharset, _
+from hachoir_core.stream import InputIOStream, InputSubStream, InputStreamError
 
 def FileInputStream(filename, real_filename=None, **args):
     """

lib/hachoir_core/stream/output.py

@@ -1,7 +1,7 @@
 from cStringIO import StringIO
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.bits import long2raw
-from lib.hachoir_core.stream import StreamError
+from hachoir_core.endian import BIG_ENDIAN, LITTLE_ENDIAN
+from hachoir_core.bits import long2raw
+from hachoir_core.stream import StreamError
 from errno import EBADF
 
 MAX_READ_NBYTES = 2 ** 16
@@ -21,6 +21,7 @@ class OutputStream(object):
     filename = property(_getFilename)
 
     def writeBit(self, state, endian):
+        assert endian in (BIG_ENDIAN, LITTLE_ENDIAN) # middle endian not yet supported
         if self._bit_pos == 7:
             self._bit_pos = 0
             if state:
@@ -39,6 +40,7 @@ class OutputStream(object):
             self._bit_pos += 1
 
     def writeBits(self, count, value, endian):
+        assert endian in (BIG_ENDIAN, LITTLE_ENDIAN) # middle endian not yet supported
         assert 0 <= value < 2**count
 
         # Feed bits to align to byte address

lib/hachoir_core/stream/stream.py

@@ -1,4 +1,4 @@
-from lib.hachoir_core.error import HachoirError
+from hachoir_core.error import HachoirError
 
 class StreamError(HachoirError):
     pass

lib/hachoir_core/text_handler.py

@@ -2,12 +2,12 @@
 Utilities used to convert a field to human classic reprentation of data.
 """
 
-from lib.hachoir_core.tools import (
+from hachoir_core.tools import (
     humanDuration, humanFilesize, alignValue,
     durationWin64 as doDurationWin64,
     deprecated)
 from types import FunctionType, MethodType
-from lib.hachoir_core.field import Field
+from hachoir_core.field import Field
 
 def textHandler(field, handler):
     assert isinstance(handler, (FunctionType, MethodType))

lib/hachoir_core/tools.py

@@ -4,7 +4,7 @@
 Various utilities.
 """
 
-from lib.hachoir_core.i18n import _, ngettext
+from hachoir_core.i18n import _, ngettext
 import re
 import stat
 from datetime import datetime, timedelta, MAXYEAR
@@ -330,7 +330,14 @@ def makeUnicode(text):
     if isinstance(text, str):
         text = unicode(text, "ISO-8859-1")
     elif not isinstance(text, unicode):
-        text = unicode(text)
+        try:
+            text = unicode(text)
+        except UnicodeError:
+            try:
+                text = str(text)
+            except Exception:
+                text = repr(text)
+            return makeUnicode(text)
     text = regex_control_code.sub(
         lambda regs: controlchars[ord(regs.group(1))], text)
     text = re.sub(r"\\x0([0-7])(?=[^0-7]|$)", r"\\\1", text)

lib/hachoir_core/version.py

@@ -1,5 +1,5 @@
 PACKAGE = "hachoir-core"
-VERSION = "1.3.3"
+VERSION = "1.3.4"
 WEBSITE = 'http://bitbucket.org/haypo/hachoir/wiki/hachoir-core'
 LICENSE = 'GNU GPL v2'
 

lib/hachoir_metadata/__init__.py

@@ -1,15 +1,15 @@
-from lib.hachoir_metadata.version import VERSION as __version__
-from lib.hachoir_metadata.metadata import extractMetadata
+from hachoir_metadata.version import VERSION as __version__
+from hachoir_metadata.metadata import extractMetadata
 
 # Just import the module,
 # each module use registerExtractor() method
-import lib.hachoir_metadata.archive
-import lib.hachoir_metadata.audio
-import lib.hachoir_metadata.file_system
-import lib.hachoir_metadata.image
-import lib.hachoir_metadata.jpeg
-import lib.hachoir_metadata.misc
-import lib.hachoir_metadata.program
-import lib.hachoir_metadata.riff
-import lib.hachoir_metadata.video
+import hachoir_metadata.archive
+import hachoir_metadata.audio
+import hachoir_metadata.file_system
+import hachoir_metadata.image
+import hachoir_metadata.jpeg
+import hachoir_metadata.misc
+import hachoir_metadata.program
+import hachoir_metadata.riff
+import hachoir_metadata.video
 

lib/hachoir_metadata/archive.py

@@ -1,11 +1,11 @@
-from lib.hachoir_metadata.metadata_item import QUALITY_BEST, QUALITY_FASTEST
-from lib.hachoir_metadata.safe import fault_tolerant, getValue
-from lib.hachoir_metadata.metadata import (
+from hachoir_metadata.metadata_item import QUALITY_BEST, QUALITY_FASTEST
+from hachoir_metadata.safe import fault_tolerant, getValue
+from hachoir_metadata.metadata import (
     RootMetadata, Metadata, MultipleMetadata, registerExtractor)
-from lib.hachoir_parser.archive import (Bzip2Parser, CabFile, GzipParser,
+from hachoir_parser.archive import (Bzip2Parser, CabFile, GzipParser,
     TarFile, ZipFile, MarFile)
-from lib.hachoir_core.tools import humanUnixAttributes
-from lib.hachoir_core.i18n import _
+from hachoir_core.tools import humanUnixAttributes
+from hachoir_core.i18n import _
 
 def maxNbFile(meta):
     if meta.quality <= QUALITY_FASTEST:
@@ -110,7 +110,7 @@ class CabMetadata(MultipleMetadata):
     def extract(self, cab):
         if "folder[0]" in cab:
             self.useFolder(cab["folder[0]"])
-        self.format_version = "Microsoft Cabinet version %s" % cab["cab_version"].display
+        self.format_version = "Microsoft Cabinet version %s.%s" % (cab["major_version"].display, cab["minor_version"].display)
         self.comment = "%s folders, %s files" % (
             cab["nb_folder"].value, cab["nb_files"].value)
         max_nb = maxNbFile(self)

lib/hachoir_metadata/audio.py

@@ -1,12 +1,12 @@
-from lib.hachoir_metadata.metadata import (registerExtractor,
+from hachoir_metadata.metadata import (registerExtractor,
     Metadata, RootMetadata, MultipleMetadata)
-from lib.hachoir_parser.audio import AuFile, MpegAudioFile, RealAudioFile, AiffFile, FlacParser
-from lib.hachoir_parser.container import OggFile, RealMediaFile
-from lib.hachoir_core.i18n import _
-from lib.hachoir_core.tools import makePrintable, timedelta2seconds, humanBitRate
+from hachoir_parser.audio import AuFile, MpegAudioFile, RealAudioFile, AiffFile, FlacParser
+from hachoir_parser.container import OggFile, RealMediaFile
+from hachoir_core.i18n import _
+from hachoir_core.tools import makePrintable, timedelta2seconds, humanBitRate
 from datetime import timedelta
-from lib.hachoir_metadata.metadata_item import QUALITY_FAST, QUALITY_NORMAL, QUALITY_BEST
-from lib.hachoir_metadata.safe import fault_tolerant, getValue
+from hachoir_metadata.metadata_item import QUALITY_FAST, QUALITY_NORMAL, QUALITY_BEST
+from hachoir_metadata.safe import fault_tolerant, getValue
 
 def computeComprRate(meta, size):
     if not meta.has("duration") \

lib/hachoir_metadata/file_system.py

@@ -1,6 +1,6 @@
-from lib.hachoir_metadata.metadata import RootMetadata, registerExtractor
-from lib.hachoir_metadata.safe import fault_tolerant
-from lib.hachoir_parser.file_system import ISO9660
+from hachoir_metadata.metadata import RootMetadata, registerExtractor
+from hachoir_metadata.safe import fault_tolerant
+from hachoir_parser.file_system import ISO9660
 from datetime import datetime
 
 class ISO9660_Metadata(RootMetadata):

lib/hachoir_metadata/filter.py

@@ -1,4 +1,4 @@
-from lib.hachoir_metadata.timezone import UTC
+from hachoir_metadata.timezone import UTC
 from datetime import date, datetime
 
 # Year in 1850..2030

lib/hachoir_metadata/formatter.py

@@ -1,4 +1,4 @@
-from lib.hachoir_core.i18n import _, ngettext
+from hachoir_core.i18n import _, ngettext
 
 NB_CHANNEL_NAME = {1: _("mono"), 2: _("stereo")}
 

lib/hachoir_metadata/image.py

@@ -1,12 +1,12 @@
-from lib.hachoir_metadata.metadata import (registerExtractor,
+from hachoir_metadata.metadata import (registerExtractor,
     Metadata, RootMetadata, MultipleMetadata)
-from lib.hachoir_parser.image import (
+from hachoir_parser.image import (
     BmpFile, IcoFile, PcxFile, GifFile, PngFile, TiffFile,
     XcfFile, TargaFile, WMF_File, PsdFile)
-from lib.hachoir_parser.image.png import getBitsPerPixel as pngBitsPerPixel
-from lib.hachoir_parser.image.xcf import XcfProperty
-from lib.hachoir_core.i18n import _
-from lib.hachoir_metadata.safe import fault_tolerant
+from hachoir_parser.image.png import getBitsPerPixel as pngBitsPerPixel
+from hachoir_parser.image.xcf import XcfProperty
+from hachoir_core.i18n import _
+from hachoir_metadata.safe import fault_tolerant
 
 def computeComprRate(meta, compr_size):
     """
@@ -240,7 +240,7 @@ class GifMetadata(RootMetadata):
     def useScreen(self, screen):
         self.width = screen["width"].value
         self.height = screen["height"].value
-        self.bits_per_pixel = (1 + screen["bpp"].value)
+        self.bits_per_pixel = (1 + screen["size_global_map"].value)
 
 class TargaMetadata(RootMetadata):
     def extract(self, tga):

lib/hachoir_metadata/jpeg.py

@@ -1,14 +1,14 @@
-from lib.hachoir_metadata.metadata import RootMetadata, registerExtractor
-from lib.hachoir_metadata.image import computeComprRate
-from lib.hachoir_parser.image.exif import ExifEntry
-from lib.hachoir_parser.image.jpeg import (
+from hachoir_metadata.metadata import RootMetadata, registerExtractor
+from hachoir_metadata.image import computeComprRate
+from hachoir_parser.image.exif import IFD, BasicIFDEntry
+from hachoir_parser.image.jpeg import (
     JpegFile, JpegChunk,
     QUALITY_HASH_COLOR, QUALITY_SUM_COLOR,
     QUALITY_HASH_GRAY, QUALITY_SUM_GRAY)
-from lib.hachoir_core.field import MissingField
-from lib.hachoir_core.i18n import _
-from lib.hachoir_core.tools import makeUnicode
-from lib.hachoir_metadata.safe import fault_tolerant
+from hachoir_core.field import MissingField
+from hachoir_core.i18n import _
+from hachoir_core.tools import makeUnicode
+from hachoir_metadata.safe import fault_tolerant
 from datetime import datetime
 
 def deg2float(degree, minute, second):
@@ -17,21 +17,21 @@ def deg2float(degree, minute, second):
 class JpegMetadata(RootMetadata):
     EXIF_KEY = {
         # Exif metadatas
-        ExifEntry.TAG_CAMERA_MANUFACTURER: "camera_manufacturer",
-        ExifEntry.TAG_CAMERA_MODEL: "camera_model",
-        ExifEntry.TAG_ORIENTATION: "image_orientation",
-        ExifEntry.TAG_EXPOSURE: "camera_exposure",
-        ExifEntry.TAG_FOCAL: "camera_focal",
-        ExifEntry.TAG_BRIGHTNESS: "camera_brightness",
-        ExifEntry.TAG_APERTURE: "camera_aperture",
+        "Make": "camera_manufacturer",
+        "Model": "camera_model",
+        "Orientation": "image_orientation",
+        "ExposureTime": "camera_exposure",
+        "FNumber": "camera_focal",
+        "BrightnessValue": "camera_brightness",
+        "MaxApertureValue": "camera_aperture",
 
         # Generic metadatas
-        ExifEntry.TAG_IMG_TITLE: "title",
-        ExifEntry.TAG_SOFTWARE: "producer",
-        ExifEntry.TAG_FILE_TIMESTAMP: "creation_date",
-        ExifEntry.TAG_WIDTH: "width",
-        ExifEntry.TAG_HEIGHT: "height",
-        ExifEntry.TAG_USER_COMMENT: "comment",
+        "ImageDescription": "title",
+        "Software": "producer",
+        "DateTime": "creation_date",
+        "PixelXDimension": "width",
+        "PixelYDimension": "height",
+        "UserComment": "comment",
     }
 
     IPTC_KEY = {
@@ -63,7 +63,8 @@ class JpegMetadata(RootMetadata):
             self.extractAPP0(jpeg["app0/content"])
 
         if "exif/content" in jpeg:
-            for ifd in jpeg.array("exif/content/ifd"):
+            for ifd in jpeg['exif/content']:
+                if not isinstance(ifd, IFD): continue
                 for entry in ifd.array("entry"):
                     self.processIfdEntry(ifd, entry)
                 self.readGPS(ifd)
@@ -156,7 +157,7 @@ class JpegMetadata(RootMetadata):
     @fault_tolerant
     def processIfdEntry(self, ifd, entry):
         # Skip unknown tags
-        tag = entry["tag"].value
+        tag = entry["tag"].display
         if tag not in self.EXIF_KEY:
             return
         key = self.EXIF_KEY[tag]
@@ -166,20 +167,17 @@ class JpegMetadata(RootMetadata):
             return
 
         # Read value
-        if "value" in entry:
-            value = entry["value"].value
-        else:
-            value = ifd["value_%s" % entry.name].value
+        value = ifd.getEntryValues(entry)[0].value
 
         # Convert value to string
-        if tag == ExifEntry.TAG_ORIENTATION:
+        if tag == "Orientation":
             value = self.orientation_name.get(value, value)
-        elif tag == ExifEntry.TAG_EXPOSURE:
+        elif tag == "ExposureTime":
             if not value:
                 return
             if isinstance(value, float):
                 value = (value, u"1/%g" % (1/value))
-        elif entry["type"].value in (ExifEntry.TYPE_RATIONAL, ExifEntry.TYPE_SIGNED_RATIONAL):
+        elif entry["type"].value in (BasicIFDEntry.TYPE_RATIONAL, BasicIFDEntry.TYPE_SIGNED_RATIONAL):
             value = (value, u"%.3g" % value)
 
         # Store information
@@ -197,35 +195,33 @@ class JpegMetadata(RootMetadata):
         timestamp = None
         datestamp = None
         for entry in ifd.array("entry"):
-            tag = entry["tag"].value
-            if tag == ExifEntry.TAG_GPS_LATITUDE_REF:
-                if entry["value"].value == "N":
+            tag = entry["tag"].display
+            values = [v.value for v in ifd.getEntryValues(entry)]
+            if tag == "GPSLatitudeRef":
+                if values[0] == "N":
                     latitude_ref = 1
                 else:
                     latitude_ref = -1
-            elif tag == ExifEntry.TAG_GPS_LONGITUDE_REF:
-                if entry["value"].value == "E":
+            elif tag == "GPSLongitudeRef":
+                if values[0] == "E":
                     longitude_ref = 1
                 else:
                     longitude_ref = -1
-            elif tag == ExifEntry.TAG_GPS_ALTITUDE_REF:
-                if entry["value"].value == 1:
+            elif tag == "GPSAltitudeRef":
+                if values[0] == 1:
                     altitude_ref = -1
                 else:
                     altitude_ref = 1
-            elif tag == ExifEntry.TAG_GPS_LATITUDE:
-                latitude = [ifd["value_%s[%u]" % (entry.name, index)].value for index in xrange(3)]
-            elif tag == ExifEntry.TAG_GPS_LONGITUDE:
-                longitude = [ifd["value_%s[%u]" % (entry.name, index)].value for index in xrange(3)]
-            elif tag == ExifEntry.TAG_GPS_ALTITUDE:
-                altitude = ifd["value_%s" % entry.name].value
-            elif tag == ExifEntry.TAG_GPS_DATESTAMP:
-                datestamp = ifd["value_%s" % entry.name].value
-            elif tag == ExifEntry.TAG_GPS_TIMESTAMP:
-                items = [ifd["value_%s[%u]" % (entry.name, index)].value for index in xrange(3)]
-                items = map(int, items)
-                items = map(str, items)
-                timestamp = ":".join(items)
+            elif tag == "GPSLatitude":
+                latitude = values
+            elif tag == "GPSLongitude":
+                longitude = values
+            elif tag == "GPSAltitude":
+                altitude = values[0]
+            elif tag == "GPSDateStamp":
+                datestamp = values[0]
+            elif tag == "GPSTimeStamp":
+                timestamp = ':'.join(str(int(x)) for x in values)
         if latitude_ref and latitude:
             value = deg2float(*latitude)
             if latitude_ref < 0:

lib/hachoir_metadata/metadata.py

@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
-from lib.hachoir_core.compatibility import any, sorted
-from lib.hachoir_core.endian import endian_name
-from lib.hachoir_core.tools import makePrintable, makeUnicode
-from lib.hachoir_core.dict import Dict
-from lib.hachoir_core.error import error, HACHOIR_ERRORS
-from lib.hachoir_core.i18n import _
-from lib.hachoir_core.log import Logger
-from lib.hachoir_metadata.metadata_item import (
+from hachoir_core.compatibility import any, sorted
+from hachoir_core.endian import endian_name
+from hachoir_core.tools import makePrintable, makeUnicode
+from hachoir_core.dict import Dict
+from hachoir_core.error import error, HACHOIR_ERRORS
+from hachoir_core.i18n import _
+from hachoir_core.log import Logger
+from hachoir_metadata.metadata_item import (
     MIN_PRIORITY, MAX_PRIORITY, QUALITY_NORMAL)
-from lib.hachoir_metadata.register import registerAllItems
+from hachoir_metadata.register import registerAllItems
 
 extractors = {}
 

lib/hachoir_metadata/metadata_item.py

@@ -1,7 +1,7 @@
-from lib.hachoir_core.tools import makeUnicode, normalizeNewline
-from lib.hachoir_core.error import HACHOIR_ERRORS
-from lib.hachoir_metadata import config
-from lib.hachoir_metadata.setter import normalizeString
+from hachoir_core.tools import makeUnicode, normalizeNewline
+from hachoir_core.error import HACHOIR_ERRORS
+from hachoir_metadata import config
+from hachoir_metadata.setter import normalizeString
 
 MIN_PRIORITY = 100
 MAX_PRIORITY = 999

lib/hachoir_metadata/misc.py

@@ -1,11 +1,11 @@
-from lib.hachoir_metadata.metadata import RootMetadata, registerExtractor
-from lib.hachoir_metadata.safe import fault_tolerant
-from lib.hachoir_parser.container import SwfFile
-from lib.hachoir_parser.misc import TorrentFile, TrueTypeFontFile, OLE2_File, PcfFile
-from lib.hachoir_core.field import isString
-from lib.hachoir_core.error import warning
-from lib.hachoir_parser import guessParser
-from lib.hachoir_metadata.setter import normalizeString
+from hachoir_metadata.metadata import RootMetadata, registerExtractor
+from hachoir_metadata.safe import fault_tolerant
+from hachoir_parser.container import SwfFile
+from hachoir_parser.misc import TorrentFile, TrueTypeFontFile, OLE2_File, PcfFile
+from hachoir_core.field import isString
+from hachoir_core.error import warning
+from hachoir_parser import guessParser
+from hachoir_metadata.setter import normalizeString
 
 class TorrentMetadata(RootMetadata):
     KEY_TO_ATTR = {
@@ -109,45 +109,42 @@ class OLE2_Metadata(RootMetadata):
     def extract(self, ole2):
         self._extract(ole2)
 
-    def _extract(self, fieldset, main_document=True):
-        if main_document:
-            # _feedAll() is needed to make sure that we get all root[*] fragments
+    def _extract(self, fieldset):
+        try:
             fieldset._feedAll()
-            if "root[0]" in fieldset:
-                self.useRoot(fieldset["root[0]"])
-        doc_summary = self.getField(fieldset, main_document, "doc_summary[0]")
+        except StopIteration:
+            pass
+        if "root[0]" in fieldset:
+            self._extract(self.getFragment(fieldset["root[0]"]))
+        doc_summary = self.getField(fieldset, "doc_summary[0]")
         if doc_summary:
             self.useSummary(doc_summary, True)
-        word_doc = self.getField(fieldset, main_document, "word_doc[0]")
+        word_doc = self.getField(fieldset, "word_doc[0]")
         if word_doc:
             self.useWordDocument(word_doc)
-        summary = self.getField(fieldset, main_document, "summary[0]")
+        summary = self.getField(fieldset, "summary[0]")
         if summary:
             self.useSummary(summary, False)
 
-    @fault_tolerant
-    def useRoot(self, root):
-        stream = root.getSubIStream()
+    def getFragment(self, frag):
+        stream = frag.getSubIStream()
         ministream = guessParser(stream)
         if not ministream:
             warning("Unable to create the OLE2 mini stream parser!")
-            return
-        self._extract(ministream, main_document=False)
+            return frag
+        return ministream
 
-    def getField(self, fieldset, main_document, name):
-        if name not in fieldset:
-            return None
+    def getField(self, fieldset, name):
         # _feedAll() is needed to make sure that we get all fragments
         # eg. summary[0], summary[1], ..., summary[n]
-        fieldset._feedAll()
+        try:
+            fieldset._feedAll()
+        except StopIteration:
+            pass
+        if name not in fieldset:
+            return None
         field = fieldset[name]
-        if main_document:
-            stream = field.getSubIStream()
-            field = guessParser(stream)
-            if not field:
-                warning("Unable to create the OLE2 parser for %s!" % name)
-                return None
-        return field
+        return self.getFragment(field)
 
     @fault_tolerant
     def useSummary(self, summary, is_doc_summary):
@@ -161,7 +158,7 @@ class OLE2_Metadata(RootMetadata):
 
     @fault_tolerant
     def useWordDocument(self, doc):
-        self.comment = "Encrypted: %s" % doc["fEncrypted"].value
+        self.comment = "Encrypted: %s" % doc["FIB/fEncrypted"].value
 
     @fault_tolerant
     def useProperty(self, summary, property, is_doc_summary):

lib/hachoir_metadata/program.py

@@ -1,6 +1,6 @@
-from lib.hachoir_metadata.metadata import RootMetadata, registerExtractor
-from lib.hachoir_parser.program import ExeFile
-from lib.hachoir_metadata.safe import fault_tolerant, getValue
+from hachoir_metadata.metadata import RootMetadata, registerExtractor
+from hachoir_parser.program import ExeFile
+from hachoir_metadata.safe import fault_tolerant, getValue
 
 class ExeMetadata(RootMetadata):
     KEY_TO_ATTR = {

lib/hachoir_metadata/qt/dialog.ui

@@ -1,64 +0,0 @@
-<ui version="4.0" >
- <class>Form</class>
- <widget class="QWidget" name="Form" >
-  <property name="geometry" >
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>441</width>
-    <height>412</height>
-   </rect>
-  </property>
-  <property name="windowTitle" >
-   <string>hachoir-metadata</string>
-  </property>
-  <layout class="QVBoxLayout" name="verticalLayout" >
-   <item>
-    <layout class="QHBoxLayout" name="horizontalLayout_2" >
-     <item>
-      <widget class="QPushButton" name="open_button" >
-       <property name="text" >
-        <string>Open</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QComboBox" name="files_combo" >
-       <property name="sizePolicy" >
-        <sizepolicy vsizetype="Fixed" hsizetype="Expanding" >
-         <horstretch>0</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-      </widget>
-     </item>
-    </layout>
-   </item>
-   <item>
-    <widget class="QTableWidget" name="metadata_table" >
-     <property name="alternatingRowColors" >
-      <bool>true</bool>
-     </property>
-     <property name="showGrid" >
-      <bool>false</bool>
-     </property>
-     <property name="rowCount" >
-      <number>0</number>
-     </property>
-     <property name="columnCount" >
-      <number>0</number>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <widget class="QPushButton" name="quit_button" >
-     <property name="text" >
-      <string>Quit</string>
-     </property>
-    </widget>
-   </item>
-  </layout>
- </widget>
- <resources/>
- <connections/>
-</ui>

lib/hachoir_metadata/qt/dialog_ui.py

@@ -1,52 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Form implementation generated from reading ui file 'hachoir_metadata/qt/dialog.ui'
-#
-# Created: Mon Jul 26 03:10:06 2010
-#      by: PyQt4 UI code generator 4.7.3
-#
-# WARNING! All changes made in this file will be lost!
-
-from PyQt4 import QtCore, QtGui
-
-class Ui_Form(object):
-    def setupUi(self, Form):
-        Form.setObjectName("Form")
-        Form.resize(441, 412)
-        self.verticalLayout = QtGui.QVBoxLayout(Form)
-        self.verticalLayout.setObjectName("verticalLayout")
-        self.horizontalLayout_2 = QtGui.QHBoxLayout()
-        self.horizontalLayout_2.setObjectName("horizontalLayout_2")
-        self.open_button = QtGui.QPushButton(Form)
-        self.open_button.setObjectName("open_button")
-        self.horizontalLayout_2.addWidget(self.open_button)
-        self.files_combo = QtGui.QComboBox(Form)
-        sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Fixed)
-        sizePolicy.setHorizontalStretch(0)
-        sizePolicy.setVerticalStretch(0)
-        sizePolicy.setHeightForWidth(self.files_combo.sizePolicy().hasHeightForWidth())
-        self.files_combo.setSizePolicy(sizePolicy)
-        self.files_combo.setObjectName("files_combo")
-        self.horizontalLayout_2.addWidget(self.files_combo)
-        self.verticalLayout.addLayout(self.horizontalLayout_2)
-        self.metadata_table = QtGui.QTableWidget(Form)
-        self.metadata_table.setAlternatingRowColors(True)
-        self.metadata_table.setShowGrid(False)
-        self.metadata_table.setRowCount(0)
-        self.metadata_table.setColumnCount(0)
-        self.metadata_table.setObjectName("metadata_table")
-        self.metadata_table.setColumnCount(0)
-        self.metadata_table.setRowCount(0)
-        self.verticalLayout.addWidget(self.metadata_table)
-        self.quit_button = QtGui.QPushButton(Form)
-        self.quit_button.setObjectName("quit_button")
-        self.verticalLayout.addWidget(self.quit_button)
-
-        self.retranslateUi(Form)
-        QtCore.QMetaObject.connectSlotsByName(Form)
-
-    def retranslateUi(self, Form):
-        Form.setWindowTitle(QtGui.QApplication.translate("Form", "hachoir-metadata", None, QtGui.QApplication.UnicodeUTF8))
-        self.open_button.setText(QtGui.QApplication.translate("Form", "Open", None, QtGui.QApplication.UnicodeUTF8))
-        self.quit_button.setText(QtGui.QApplication.translate("Form", "Quit", None, QtGui.QApplication.UnicodeUTF8))
-

lib/hachoir_metadata/register.py

@@ -1,17 +1,17 @@
-from lib.hachoir_core.i18n import _
-from lib.hachoir_core.tools import (
+from hachoir_core.i18n import _
+from hachoir_core.tools import (
     humanDuration, humanBitRate,
     humanFrequency, humanBitSize, humanFilesize,
     humanDatetime)
-from lib.hachoir_core.language import Language
-from lib.hachoir_metadata.filter import Filter, NumberFilter, DATETIME_FILTER
+from hachoir_core.language import Language
+from hachoir_metadata.filter import Filter, NumberFilter, DATETIME_FILTER
 from datetime import date, datetime, timedelta
-from lib.hachoir_metadata.formatter import (
+from hachoir_metadata.formatter import (
     humanAudioChannel, humanFrameRate, humanComprRate, humanAltitude,
     humanPixelSize, humanDPI)
-from lib.hachoir_metadata.setter import (
+from hachoir_metadata.setter import (
     setDatetime, setTrackNumber, setTrackTotal, setLanguage)
-from lib.hachoir_metadata.metadata_item import Data
+from hachoir_metadata.metadata_item import Data
 
 MIN_SAMPLE_RATE = 1000              # 1 kHz
 MAX_SAMPLE_RATE = 192000            # 192 kHz

lib/hachoir_metadata/riff.py

@@ -2,13 +2,13 @@
 Extract metadata from RIFF file format: AVI video and WAV sound.
 """
 
-from lib.hachoir_metadata.metadata import Metadata, MultipleMetadata, registerExtractor
-from lib.hachoir_metadata.safe import fault_tolerant, getValue
-from lib.hachoir_parser.container.riff import RiffFile
-from lib.hachoir_parser.video.fourcc import UNCOMPRESSED_AUDIO
-from lib.hachoir_core.tools import humanFilesize, makeUnicode, timedelta2seconds
-from lib.hachoir_core.i18n import _
-from lib.hachoir_metadata.audio import computeComprRate as computeAudioComprRate
+from hachoir_metadata.metadata import Metadata, MultipleMetadata, registerExtractor
+from hachoir_metadata.safe import fault_tolerant, getValue
+from hachoir_parser.container.riff import RiffFile
+from hachoir_parser.video.fourcc import UNCOMPRESSED_AUDIO
+from hachoir_core.tools import humanFilesize, makeUnicode, timedelta2seconds
+from hachoir_core.i18n import _
+from hachoir_metadata.audio import computeComprRate as computeAudioComprRate
 from datetime import timedelta
 
 class RiffMetadata(MultipleMetadata):

lib/hachoir_metadata/safe.py

@@ -1,4 +1,4 @@
-from lib.hachoir_core.error import HACHOIR_ERRORS, warning
+from hachoir_core.error import HACHOIR_ERRORS, warning
 
 def fault_tolerant(func, *args):
     def safe_func(*args, **kw):

lib/hachoir_metadata/setter.py

@@ -1,10 +1,10 @@
 from datetime import date, datetime
 import re
-from lib.hachoir_core.language import Language
+from hachoir_core.language import Language
 from locale import setlocale, LC_ALL
 from time import strptime
-from lib.hachoir_metadata.timezone import createTimezone
-from lib.hachoir_metadata import config
+from hachoir_metadata.timezone import createTimezone
+from hachoir_metadata import config
 
 NORMALIZE_REGEX = re.compile("[-/.: ]+")
 YEAR_REGEX1 = re.compile("^([0-9]{4})$")

lib/hachoir_metadata/video.py

@@ -1,14 +1,14 @@
-from lib.hachoir_core.field import MissingField
-from lib.hachoir_metadata.metadata import (registerExtractor,
+from hachoir_core.field import MissingField
+from hachoir_metadata.metadata import (registerExtractor,
     Metadata, RootMetadata, MultipleMetadata)
-from lib.hachoir_metadata.metadata_item import QUALITY_GOOD
-from lib.hachoir_metadata.safe import fault_tolerant
-from lib.hachoir_parser.video import MovFile, AsfFile, FlvFile
-from lib.hachoir_parser.video.asf import Descriptor as ASF_Descriptor
-from lib.hachoir_parser.container import MkvFile
-from lib.hachoir_parser.container.mkv import dateToDatetime
-from lib.hachoir_core.i18n import _
-from lib.hachoir_core.tools import makeUnicode, makePrintable, timedelta2seconds
+from hachoir_metadata.metadata_item import QUALITY_GOOD
+from hachoir_metadata.safe import fault_tolerant
+from hachoir_parser.video import MovFile, AsfFile, FlvFile
+from hachoir_parser.video.asf import Descriptor as ASF_Descriptor
+from hachoir_parser.container import MkvFile
+from hachoir_parser.container.mkv import dateToDatetime
+from hachoir_core.i18n import _
+from hachoir_core.tools import makeUnicode, makePrintable, timedelta2seconds
 from datetime import timedelta
 
 class MkvMetadata(MultipleMetadata):
@@ -59,9 +59,10 @@ class MkvMetadata(MultipleMetadata):
     def trackCommon(self, track, meta):
         if "Name/unicode" in track:
             meta.title = track["Name/unicode"].value
-        if "Language/string" in track \
-        and track["Language/string"].value not in ("mis", "und"):
+        if "Language/string" in track:
             meta.language = track["Language/string"].value
+        else:
+            meta.language = "eng"
 
     def processVideo(self, track):
         video = Metadata(self)
@@ -222,7 +223,7 @@ class MovMetadata(RootMetadata):
         self.last_modification = hdr["lastmod_date"].value
         self.duration = timedelta(seconds=float(hdr["duration"].value) / hdr["time_scale"].value)
         self.comment = _("Play speed: %.1f%%") % (hdr["play_speed"].value*100)
-        self.comment = _("User volume: %.1f%%") % (float(hdr["volume"].value)*100//255)
+        self.comment = _("User volume: %.1f%%") % (float(hdr["volume"].value)*100)
 
     @fault_tolerant
     def processTrackHeader(self, hdr):

lib/hachoir_parser/__init__.py

@@ -1,7 +1,7 @@
-from lib.hachoir_parser.version import __version__
-from lib.hachoir_parser.parser import ValidateError, HachoirParser, Parser
-from lib.hachoir_parser.parser_list import ParserList, HachoirParserList
-from lib.hachoir_parser.guess import (QueryParser, guessParser, createParser)
-from lib.hachoir_parser import (archive, audio, container,
+from hachoir_parser.version import __version__
+from hachoir_parser.parser import ValidateError, HachoirParser, Parser
+from hachoir_parser.parser_list import ParserList, HachoirParserList
+from hachoir_parser.guess import (QueryParser, guessParser, createParser)
+from hachoir_parser import (archive, audio, container,
     file_system, image, game, misc, network, program, video)
 

lib/hachoir_parser/archive/__init__.py

@@ -1,12 +1,13 @@
-from lib.hachoir_parser.archive.ace import AceFile
-from lib.hachoir_parser.archive.ar import ArchiveFile
-from lib.hachoir_parser.archive.bzip2_parser import Bzip2Parser
-from lib.hachoir_parser.archive.cab import CabFile
-from lib.hachoir_parser.archive.gzip_parser import GzipParser
-from lib.hachoir_parser.archive.tar import TarFile
-from lib.hachoir_parser.archive.zip import ZipFile
-from lib.hachoir_parser.archive.rar import RarFile
-from lib.hachoir_parser.archive.rpm import RpmFile
-from lib.hachoir_parser.archive.sevenzip import SevenZipParser
-from lib.hachoir_parser.archive.mar import MarFile
-
+from hachoir_parser.archive.ace import AceFile
+from hachoir_parser.archive.ar import ArchiveFile
+from hachoir_parser.archive.bzip2_parser import Bzip2Parser
+from hachoir_parser.archive.cab import CabFile
+from hachoir_parser.archive.gzip_parser import GzipParser
+from hachoir_parser.archive.tar import TarFile
+from hachoir_parser.archive.zip import ZipFile
+from hachoir_parser.archive.rar import RarFile
+from hachoir_parser.archive.rpm import RpmFile
+from hachoir_parser.archive.sevenzip import SevenZipParser
+from hachoir_parser.archive.mar import MarFile
+from hachoir_parser.archive.mozilla_ar import MozillaArchive
+from hachoir_parser.archive.zlib import ZlibData

lib/hachoir_parser/archive/ace.py

@@ -11,15 +11,15 @@ Author: Christophe Gisquet <christophe.gisquet@free.fr>
 Creation date: 19 january 2006
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (StaticFieldSet, FieldSet,
+from hachoir_parser import Parser
+from hachoir_core.field import (StaticFieldSet, FieldSet,
     Bit, Bits, NullBits, RawBytes, Enum,
     UInt8, UInt16, UInt32,
     PascalString8, PascalString16, String,
     TimeDateMSDOS32)
-from lib.hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_parser.common.msdos import MSDOSFileAttr32
+from hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_parser.common.msdos import MSDOSFileAttr32
 
 MAGIC = "**ACE**"
 

lib/hachoir_parser/archive/ar.py

@@ -2,10 +2,10 @@
 GNU ar archive : archive file (.a) and Debian (.deb) archive.
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet, ParserError,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet, ParserError,
     String, RawBytes, UnixLine)
-from lib.hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.endian import BIG_ENDIAN
 
 class ArchiveFileEntry(FieldSet):
     def createFields(self):

lib/hachoir_parser/archive/bzip2_parser.py

@@ -1,14 +1,18 @@
 """
 BZIP2 archive file
 
-Author: Victor Stinner
+Author: Victor Stinner, Robert Xiao
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (ParserError, String,
-    Bytes, Character, UInt8, UInt32, CompressedField)
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_parser import Parser
+from hachoir_core.tools import paddingSize
+from hachoir_core.field import (Field, FieldSet, GenericVector,
+    ParserError, String,
+    PaddingBits, Bit, Bits, Character,
+    UInt32, Enum, CompressedField)
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_parser.archive.zlib import build_tree, HuffmanCode
 
 try:
     from bz2 import BZ2Decompressor
@@ -27,6 +31,152 @@ try:
 except ImportError:
     has_deflate = False
 
+class ZeroTerminatedNumber(Field):
+    """Zero (bit) terminated number: e.g. 11110 is 4."""
+    def __init__(self, parent, name, description=None):
+        Field.__init__(self, parent, name, 0, description)
+
+        endian = self.parent.endian
+        stream = self.parent.stream
+        addr = self.absolute_address
+
+        value = 0
+        while True:
+            bit = stream.readBits(addr, 1, endian)
+            addr += 1
+            self._size += 1
+            if not bit:
+                break
+            value += 1
+        self._value = value
+    def createValue(self):
+        return self._value
+
+def move_to_front(l, c):
+    l[:] = l[c:c+1] + l[0:c] + l[c+1:]
+
+class Bzip2Bitmap(FieldSet):
+    def __init__(self, parent, name, nb_items, start_index, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        self.nb_items = nb_items
+        self.start_index = start_index
+
+    def createFields(self):
+        for i in xrange(self.start_index, self.start_index+self.nb_items):
+            yield Bit(self, "symbol_used[%i]"%i, "Is the symbol %i (%r) used?"%(i, chr(i)))
+
+class Bzip2Lengths(FieldSet):
+    def __init__(self, parent, name, symbols, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        self.symbols = symbols
+
+    def createFields(self):
+        yield Bits(self, "start_length", 5)
+        length = self["start_length"].value
+        lengths = []
+        for i in xrange(self.symbols):
+            while True:
+                bit = Bit(self, "change_length[%i][]"%i, "Should the length be changed for symbol %i?"%i)
+                yield bit
+                if not bit.value:
+                    break
+                else:
+                    bit = Enum(Bit(self, "length_decrement[%i][]"%i, "Decrement the value?"), {True: "Decrement", False: "Increment"})
+                    yield bit
+                    if bit.value:
+                        length -= 1
+                    else:
+                        length += 1
+            lengths.append(length)
+        self.final_length = length
+        self.tree = build_tree(lengths)
+
+class Bzip2Selectors(FieldSet):
+    def __init__(self, parent, name, ngroups, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        self.groups = range(ngroups)
+
+    def createFields(self):
+        for i in xrange(self["../selectors_used"].value):
+            field = ZeroTerminatedNumber(self, "selector_list[]")
+            move_to_front(self.groups, field.value)
+            field.realvalue = self.groups[0]
+            field._description = "MTF'ed selector index: raw value %i, real value %i"%(field.value, field.realvalue)
+            yield field
+
+class Bzip2Block(FieldSet):
+    def createFields(self):
+        yield textHandler(Bits(self, "blockheader", 48, "Block header"), hexadecimal)
+        if self["blockheader"].value != 0x314159265359: # pi
+            raise ParserError("Invalid block header!")
+        yield textHandler(UInt32(self, "crc32", "CRC32 for this block"), hexadecimal)
+        yield Bit(self, "randomized", "Is this block randomized?")
+        yield Bits(self, "orig_bwt_pointer", 24, "Starting pointer into BWT after untransform")
+        yield GenericVector(self, "huffman_used_map", 16, Bit, 'block_used', "Bitmap showing which blocks (representing 16 literals each) are in use")
+        symbols_used = []
+        for index, block_used in enumerate(self["huffman_used_map"].array('block_used')):
+            if block_used.value:
+                start_index = index*16
+                field = Bzip2Bitmap(self, "huffman_used_bitmap[%i]"%index, 16, start_index, "Bitmap for block %i (literals %i to %i) showing which symbols are in use"%(index, start_index, start_index + 15))
+                yield field
+                for i, used in enumerate(field):
+                    if used.value:
+                        symbols_used.append(start_index + i)
+        yield Bits(self, "huffman_groups", 3, "Number of different Huffman tables in use")
+        yield Bits(self, "selectors_used", 15, "Number of times the Huffman tables are switched")
+        yield Bzip2Selectors(self, "selectors_list", self["huffman_groups"].value)
+        trees = []
+        for group in xrange(self["huffman_groups"].value):
+            field = Bzip2Lengths(self, "huffman_lengths[]", len(symbols_used)+2)
+            yield field
+            trees.append(field.tree)
+        counter = 0
+        rle_run = 0
+        selector_tree = None
+        while True:
+            if counter%50 == 0:
+                select_id = self["selectors_list"].array("selector_list")[counter//50].realvalue
+                selector_tree = trees[select_id]
+            field = HuffmanCode(self, "huffman_code[]", selector_tree)
+            if field.realvalue in [0, 1]:
+                # RLE codes
+                if rle_run == 0:
+                    rle_power = 1
+                rle_run += (field.realvalue + 1) * rle_power
+                rle_power <<= 1
+                field._description = "RLE Run Code %i (for %r); Total accumulated run %i (Huffman Code %i)" % (field.realvalue, chr(symbols_used[0]), rle_run, field.value)
+            elif field.realvalue == len(symbols_used)+1:
+                field._description = "Block Terminator (%i) (Huffman Code %i)"%(field.realvalue, field.value)
+                yield field
+                break
+            else:
+                rle_run = 0
+                move_to_front(symbols_used, field.realvalue-1)
+                field._description = "Literal %r (value %i) (Huffman Code %i)"%(chr(symbols_used[0]), field.realvalue, field.value)
+            yield field
+            if field.realvalue == len(symbols_used)+1:
+                break
+            counter += 1
+
+class Bzip2Stream(FieldSet):
+    START_BLOCK = 0x314159265359 # pi
+    END_STREAM = 0x177245385090 # sqrt(pi)
+    def createFields(self):
+        end = False
+        while not end:
+            marker = self.stream.readBits(self.absolute_address + self.current_size, 48, self.endian)
+            if marker == self.START_BLOCK:
+                yield Bzip2Block(self, "block[]")
+            elif marker == self.END_STREAM:
+                yield textHandler(Bits(self, "stream_end", 48, "End-of-stream marker"), hexadecimal)
+                yield textHandler(UInt32(self, "crc32", "CRC32 for entire stream"), hexadecimal)
+                padding = paddingSize(self.current_size, 8)
+                if padding:
+                    yield PaddingBits(self, "padding[]", padding)
+                end = True
+            else:
+                raise ParserError("Invalid marker 0x%02X!"%marker)
+
 class Bzip2Parser(Parser):
     PARSER_TAGS = {
         "id": "bzip2",
@@ -37,7 +187,7 @@ class Bzip2Parser(Parser):
         "magic": (('BZh', 0),),
         "description": "bzip2 archive"
     }
-    endian = LITTLE_ENDIAN
+    endian = BIG_ENDIAN
 
     def validate(self):
         if self.stream.readBytes(0, 3) != 'BZh':
@@ -50,18 +200,6 @@ class Bzip2Parser(Parser):
         yield String(self, "id", 3, "Identifier (BZh)", charset="ASCII")
         yield Character(self, "blocksize", "Block size (KB of memory needed to uncompress)")
 
-        yield UInt8(self, "blockheader", "Block header")
-        if self["blockheader"].value == 0x17:
-            yield String(self, "id2", 4, "Identifier2 (re8P)", charset="ASCII")
-            yield UInt8(self, "id3", "Identifier3 (0x90)")
-        elif self["blockheader"].value == 0x31:
-            yield String(self, "id2", 5, "Identifier 2 (AY&SY)", charset="ASCII")
-            if self["id2"].value != "AY&SY":
-                raise ParserError("Invalid identifier 2 (AY&SY)!")
-        else:
-            raise ParserError("Invalid block header!")
-        yield textHandler(UInt32(self, "crc32", "CRC32"), hexadecimal)
-
         if self._size is None: # TODO: is it possible to handle piped input?
             raise NotImplementedError
 
@@ -73,7 +211,7 @@ class Bzip2Parser(Parser):
                     break
             else:
                 filename = None
-            data = Bytes(self, "file", size)
+            data = Bzip2Stream(self, "file", size=size*8)
             if has_deflate:
                 CompressedField(self, Bunzip2)
                 def createInputStream(**args):

lib/hachoir_parser/archive/cab.py

@@ -1,18 +1,24 @@
 """
 Microsoft Cabinet (CAB) archive.
 
-Author: Victor Stinner
+Author: Victor Stinner, Robert Xiao
 Creation date: 31 january 2007
-"""
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet, Enum,
+- Microsoft Cabinet SDK
+  http://msdn2.microsoft.com/en-us/library/ms974336.aspx
+"""
+from __future__ import absolute_import
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet, Enum,
     CString, String,
-    UInt16, UInt32, Bit, Bits, PaddingBits, NullBits,
+    UInt8, UInt16, UInt32, Bit, Bits, PaddingBits, NullBits,
     DateTimeMSDOS32, RawBytes)
-from lib.hachoir_parser.common.msdos import MSDOSFileAttr16
-from lib.hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
-from lib.hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.tools import paddingSize
+from hachoir_core.stream import StringInputStream
+from hachoir_parser.archive.lzx import LZXStream, lzx_decompress
+from hachoir_parser.archive.zlib import DeflateBlock
 
 MAX_NB_FOLDER = 30
 
@@ -26,38 +32,54 @@ COMPRESSION_NAME = {
 
 class Folder(FieldSet):
     def createFields(self):
-        yield UInt32(self, "off_data", "Offset of data")
-        yield UInt16(self, "cf_data")
+        yield UInt32(self, "offset", "Offset to data (from file start)")
+        yield UInt16(self, "data_blocks", "Number of data blocks which are in this cabinet")
         yield Enum(Bits(self, "compr_method", 4, "Compression method"), COMPRESSION_NAME)
-        yield Bits(self, "compr_level", 5, "Compression level")
-        yield PaddingBits(self, "padding", 7)
+        if self["compr_method"].value in [2, 3]: # Quantum or LZX use compression level
+            yield PaddingBits(self, "padding[]", 4)
+            yield Bits(self, "compr_level", 5, "Compression level")
+            yield PaddingBits(self, "padding[]", 3)
+        else:
+            yield PaddingBits(self, "padding[]", 12)
+        if self["../flags/has_reserved"].value and self["../reserved_folder_size"].value:
+            yield RawBytes(self, "reserved_folder", self["../reserved_folder_size"].value, "Per-folder reserved area")
 
     def createDescription(self):
         text= "Folder: compression %s" % self["compr_method"].display
-        if self["compr_method"].value != COMPRESSION_NONE:
-            text += " (level %u)" % self["compr_level"].value
+        if self["compr_method"].value in [2, 3]: # Quantum or LZX use compression level
+            text += " (level %u: window size %u)" % (self["compr_level"].value, 2**self["compr_level"].value)
         return text
 
+class CabFileAttributes(FieldSet):
+    def createFields(self):
+        yield Bit(self, "readonly")
+        yield Bit(self, "hidden")
+        yield Bit(self, "system")
+        yield Bits(self, "reserved[]", 2)
+        yield Bit(self, "archive", "Has the file been modified since the last backup?")
+        yield Bit(self, "exec", "Run file after extraction?")
+        yield Bit(self, "name_is_utf", "Is the filename using UTF-8?")
+        yield Bits(self, "reserved[]", 8)
+
 class File(FieldSet):
     def createFields(self):
         yield filesizeHandler(UInt32(self, "filesize", "Uncompressed file size"))
-        yield UInt32(self, "offset", "File offset after decompression")
-        yield UInt16(self, "iFolder", "file control id")
+        yield UInt32(self, "folder_offset", "File offset in uncompressed folder")
+        yield Enum(UInt16(self, "folder_index", "Containing folder ID (index)"), {
+            0xFFFD:"Folder continued from previous cabinet (real folder ID = 0)",
+            0xFFFE:"Folder continued to next cabinet (real folder ID = %i)" % (self["../nb_folder"].value - 1),
+            0xFFFF:"Folder spanning previous, current and next cabinets (real folder ID = 0)"})
         yield DateTimeMSDOS32(self, "timestamp")
-        yield MSDOSFileAttr16(self, "attributes")
-        yield CString(self, "filename", charset="ASCII")
+        yield CabFileAttributes(self, "attributes")
+        if self["attributes/name_is_utf"].value:
+            yield CString(self, "filename", charset="UTF-8")
+        else:
+            yield CString(self, "filename", charset="ASCII")
 
     def createDescription(self):
         return "File %s (%s)" % (
             self["filename"].display, self["filesize"].display)
 
-class Reserved(FieldSet):
-    def createFields(self):
-        yield UInt32(self, "size")
-        size = self["size"].value
-        if size:
-            yield RawBytes(self, "data", size)
-
 class Flags(FieldSet):
     static_size = 16
     def createFields(self):
@@ -66,6 +88,111 @@ class Flags(FieldSet):
         yield Bit(self, "has_reserved")
         yield NullBits(self, "padding", 13)
 
+class FragmentGroup:
+    def __init__(self, parser):
+        self.items = []
+        self.parser = parser
+        self.args = {}
+
+    def add(self, item):
+        self.items.append(item)
+
+    def createInputStream(self):
+        # FIXME: Use lazy stream creation
+        data = []
+        for item in self.items:
+            data.append( item["rawdata"].value )
+        data = "".join(data)
+
+        # FIXME: Use smarter code to send arguments
+        self.args["compr_level"] = self.items[0].parent.parent.folder["compr_level"].value
+        tags = {"class": self.parser, "args": self.args}
+        tags = tags.iteritems()
+        return StringInputStream(data, "<fragment group>", tags=tags)
+
+class CustomFragment(FieldSet):
+    def __init__(self, parent, name, size, parser, description=None, group=None):
+        FieldSet.__init__(self, parent, name, description, size=size)
+        if not group:
+            group = FragmentGroup(parser)
+        self.field_size = size
+        self.group = group
+        self.group.add(self)
+
+    def createFields(self):
+        yield RawBytes(self, "rawdata", self.field_size//8)
+
+    def _createInputStream(self, **args):
+        return self.group.createInputStream()
+
+class DataBlock(FieldSet):
+    def __init__(self, *args, **kwargs):
+        FieldSet.__init__(self, *args, **kwargs)
+        size = (self["size"].value + 8) * 8 # +8 for header values
+        if self["/flags/has_reserved"].value:
+            size += self["/reserved_data_size"].value * 8
+        self._size = size
+
+    def createFields(self):
+        yield textHandler(UInt32(self, "crc32"), hexadecimal)
+        yield UInt16(self, "size")
+        yield UInt16(self, "uncompressed_size", "If this is 0, this block is continued in a subsequent cabinet")
+        if self["/flags/has_reserved"].value and self["/reserved_data_size"].value:
+            yield RawBytes(self, "reserved_data", self["/reserved_data_size"].value, "Per-datablock reserved area")
+        compr_method = self.parent.folder["compr_method"].value
+        if compr_method == 0: # Uncompressed
+            yield RawBytes(self, "data", self["size"].value, "Folder Data")
+            self.parent.uncompressed_data += self["data"].value
+        elif compr_method == 1: # MSZIP
+            yield String(self, "mszip_signature", 2, "MSZIP Signature (CK)")
+            yield DeflateBlock(self, "deflate_block", self.parent.uncompressed_data)
+            padding = paddingSize(self.current_size, 8)
+            if padding:
+                yield PaddingBits(self, "padding[]", padding)
+            self.parent.uncompressed_data = self["deflate_block"].uncomp_data
+        elif compr_method == 2: # Quantum
+            yield RawBytes(self, "compr_data", self["size"].value, "Compressed Folder Data")
+        elif compr_method == 3: # LZX
+            group = getattr(self.parent.folder, "lzx_group", None)
+            field = CustomFragment(self, "data", self["size"].value*8, LZXStream, "LZX data fragment", group)
+            self.parent.folder.lzx_group = field.group
+            yield field
+
+class FolderParser(Parser):
+    endian = LITTLE_ENDIAN
+    def createFields(self):
+        for file in sorted(self.files, key=lambda x:x["folder_offset"].value):
+            padding = self.seekByte(file["folder_offset"].value)
+            if padding:
+                yield padding
+            yield RawBytes(self, "file[]", file["filesize"].value, file.description)
+
+class FolderData(FieldSet):
+    def __init__(self, parent, name, folder, files, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        def createInputStream(cis, source=None, **args):
+            stream = cis(source=source)
+            tags = args.setdefault("tags",[])
+            tags.extend(stream.tags)
+            tags.append(( "class", FolderParser ))
+            tags.append(( "args", {'files': files} ))
+            for unused in self:
+                pass
+            if folder["compr_method"].value == 3: # LZX
+                self.uncompressed_data = lzx_decompress(self["block[0]/data"].getSubIStream(), folder["compr_level"].value)
+            return StringInputStream(self.uncompressed_data, source=source, **args)
+        self.setSubIStream(createInputStream)
+        self.files = files
+        self.folder = folder # Folder fieldset
+
+    def createFields(self):
+        self.uncompressed_data = ""
+        for index in xrange(self.folder["data_blocks"].value):
+            block = DataBlock(self, "block[]")
+            for i in block:
+                pass
+            yield block
+
 class CabFile(Parser):
     endian = LITTLE_ENDIAN
     MAGIC = "MSCF"
@@ -82,8 +209,8 @@ class CabFile(Parser):
     def validate(self):
         if self.stream.readBytes(0, 4) != self.MAGIC:
             return "Invalid magic"
-        if self["cab_version"].value != 0x0103:
-            return "Unknown version (%s)" % self["cab_version"].display
+        if self["major_version"].value != 1 or self["minor_version"].value != 3:
+            return "Unknown version (%i.%i)" % (self["major_version"].value, self["minor_version"].value)
         if not (1 <= self["nb_folder"].value <= MAX_NB_FOLDER):
             return "Invalid number of folder (%s)" % self["nb_folder"].value
         return True
@@ -95,26 +222,54 @@ class CabFile(Parser):
         yield textHandler(UInt32(self, "fld_checksum", "Folders checksum (0 if not used)"), hexadecimal)
         yield UInt32(self, "off_file", "Offset of first file")
         yield textHandler(UInt32(self, "files_checksum", "Files checksum (0 if not used)"), hexadecimal)
-        yield textHandler(UInt16(self, "cab_version", "Cabinet version"), hexadecimal)
+        yield UInt8(self, "minor_version", "Minor version (should be 3)")
+        yield UInt8(self, "major_version", "Major version (should be 1)")
         yield UInt16(self, "nb_folder", "Number of folders")
         yield UInt16(self, "nb_files", "Number of files")
         yield Flags(self, "flags")
         yield UInt16(self, "setid")
-        yield UInt16(self, "number", "Zero-based cabinet number")
+        yield UInt16(self, "cabinet_serial", "Zero-based cabinet number")
 
-        # --- TODO: Support flags
         if self["flags/has_reserved"].value:
-            yield Reserved(self, "reserved")
-        #(3) Previous cabinet name, if CAB_HEADER.flags & CAB_FLAG_HASPREV
-        #(4) Previous disk name, if CAB_HEADER.flags & CAB_FLAG_HASPREV
-        #(5) Next cabinet name, if CAB_HEADER.flags & CAB_FLAG_HASNEXT
-        #(6) Next disk name, if CAB_HEADER.flags & CAB_FLAG_HASNEXT
-        # ----
+            yield UInt16(self, "reserved_header_size", "Size of per-cabinet reserved area")
+            yield UInt8(self, "reserved_folder_size", "Size of per-folder reserved area")
+            yield UInt8(self, "reserved_data_size", "Size of per-datablock reserved area")
+            if self["reserved_header_size"].value:
+                yield RawBytes(self, "reserved_header", self["reserved_header_size"].value, "Per-cabinet reserved area")
+        if self["flags/has_previous"].value:
+            yield CString(self, "previous_cabinet", "File name of previous cabinet", charset="ASCII")
+            yield CString(self, "previous_disk", "Description of disk/media on which previous cabinet resides", charset="ASCII")
+        if self["flags/has_next"].value:
+            yield CString(self, "next_cabinet", "File name of next cabinet", charset="ASCII")
+            yield CString(self, "next_disk", "Description of disk/media on which next cabinet resides", charset="ASCII")
 
+        folders = []
+        files = []
         for index in xrange(self["nb_folder"].value):
-            yield Folder(self, "folder[]")
+            folder = Folder(self, "folder[]")
+            yield folder
+            folders.append(folder)
         for index in xrange(self["nb_files"].value):
-            yield File(self, "file[]")
+            file = File(self, "file[]")
+            yield file
+            files.append(file)
+
+        folders = sorted(enumerate(folders), key=lambda x:x[1]["offset"].value)
+
+        for i in xrange(len(folders)):
+            index, folder = folders[i]
+            padding = self.seekByte(folder["offset"].value)
+            if padding:
+                yield padding
+            files = []
+            for file in files:
+                if file["folder_index"].value == index:
+                    files.append(file)
+            if i+1 == len(folders):
+                size = (self.size // 8) - folder["offset"].value
+            else:
+                size = (folders[i+1][1]["offset"].value) - folder["offset"].value
+            yield FolderData(self, "folder_data[%i]" % index, folder, files, size=size*8)
 
         end = self.seekBit(self.size, "endraw")
         if end:

lib/hachoir_parser/archive/gzip_parser.py

@@ -4,14 +4,14 @@ GZIP archive parser.
 Author: Victor Stinner
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (
+from hachoir_parser import Parser
+from hachoir_core.field import (
     UInt8, UInt16, UInt32, Enum, TimestampUnix32,
     Bit, CString, SubFile,
     NullBits, Bytes, RawBytes)
-from lib.hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_parser.common.deflate import Deflate
+from hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_parser.common.deflate import Deflate
 
 class GzipParser(Parser):
     endian = LITTLE_ENDIAN

lib/hachoir_parser/archive/lzx.py

@@ -0,0 +1,267 @@
+"""LZX data stream parser.
+
+Also includes a decompression function (slow!!) which can decompress
+LZX data stored in a Hachoir stream.
+
+Author: Robert Xiao
+Creation date: July 18, 2007
+"""
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet,
+    UInt32, Bit, Bits, PaddingBits,
+    RawBytes, ParserError)
+from hachoir_core.endian import MIDDLE_ENDIAN, LITTLE_ENDIAN
+from hachoir_core.tools import paddingSize, alignValue
+from hachoir_parser.archive.zlib import build_tree, HuffmanCode, extend_data
+from hachoir_core.bits import str2long
+import new # for instancemethod
+
+class LZXPreTreeEncodedTree(FieldSet):
+    def __init__(self, parent, name, num_elements, *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        self.num_elements = num_elements
+        
+    def createFields(self):
+        for i in xrange(20):
+            yield Bits(self, "pretree_lengths[]", 4)
+        pre_tree = build_tree([self['pretree_lengths[%d]'%x].value for x in xrange(20)])
+        if not hasattr(self.root, "lzx_tree_lengths_"+self.name):
+            self.lengths = [0] * self.num_elements
+            setattr(self.root, "lzx_tree_lengths_"+self.name, self.lengths)
+        else:
+            self.lengths = getattr(self.root, "lzx_tree_lengths_"+self.name)
+        i = 0
+        while i < self.num_elements:
+            field = HuffmanCode(self, "tree_code[]", pre_tree)
+            if field.realvalue <= 16:
+                self.lengths[i] = (self.lengths[i] - field.realvalue) % 17
+                field._description = "Literal tree delta length %i (new length value %i for element %i)" % (
+                        field.realvalue, self.lengths[i], i)
+                i += 1
+                yield field
+            elif field.realvalue == 17:
+                field._description = "Tree Code 17: Zeros for 4-19 elements"
+                yield field
+                extra = Bits(self, "extra[]", 4)
+                zeros = 4 + extra.value
+                extra._description = "Extra bits: zeros for %i elements (elements %i through %i)" % (zeros, i, i+zeros-1)
+                yield extra
+                self.lengths[i:i+zeros] = [0] * zeros
+                i += zeros
+            elif field.realvalue == 18:
+                field._description = "Tree Code 18: Zeros for 20-51 elements"
+                yield field
+                extra = Bits(self, "extra[]", 5)
+                zeros = 20 + extra.value
+                extra._description = "Extra bits: zeros for %i elements (elements %i through %i)" % (zeros, i, i+zeros-1)
+                yield extra
+                self.lengths[i:i+zeros] = [0] * zeros
+                i += zeros
+            elif field.realvalue == 19:
+                field._description = "Tree Code 19: Same code for 4-5 elements"
+                yield field
+                extra = Bits(self, "extra[]", 1)
+                run = 4 + extra.value
+                extra._description = "Extra bits: run for %i elements (elements %i through %i)" % (run, i, i+run-1)
+                yield extra
+                newfield = HuffmanCode(self, "tree_code[]", pre_tree)
+                assert newfield.realvalue <= 16
+                newfield._description = "Literal tree delta length %i (new length value %i for elements %i through %i)" % (
+                        newfield.realvalue, self.lengths[i], i, i+run-1)
+                self.lengths[i:i+run] = [(self.lengths[i] - newfield.realvalue) % 17] * run
+                i += run
+                yield newfield
+
+class LZXBlock(FieldSet):
+    WINDOW_SIZE = {15:30,
+                   16:32,
+                   17:34,
+                   18:36,
+                   19:38,
+                   20:42,
+                   21:50}
+    POSITION_SLOTS = {0:(0,0,0),
+                      1:(1,1,0),
+                      2:(2,2,0),
+                      3:(3,3,0),
+                      4:(4,5,1),
+                      5:(6,7,1),
+                      6:(8,11,2),
+                      7:(12,15,2),
+                      8:(16,23,3),
+                      9:(24,31,3),
+                      10:(32,47,4),
+                      11:(48,63,4),
+                      12:(64,95,5),
+                      13:(96,127,5),
+                      14:(128,191,6),
+                      15:(192,255,6),
+                      16:(256,383,7),
+                      17:(384,511,7),
+                      18:(512,767,8),
+                      19:(768,1023,8),
+                      20:(1024,1535,9),
+                      21:(1536,2047,9),
+                      22:(2048,3071,10),
+                      23:(3072,4095,10),
+                      24:(4096,6143,11),
+                      25:(6144,8191,11),
+                      26:(8192,12287,12),
+                      27:(12288,16383,12),
+                      28:(16384,24575,13),
+                      29:(24576,32767,13),
+                      30:(32768,49151,14),
+                      31:(49152,65535,14),
+                      32:(65536,98303,15),
+                      33:(98304,131071,15),
+                      34:(131072,196607,16),
+                      35:(196608,262143,16),
+                      36:(262144,393215,17),
+                      37:(393216,524287,17),
+                      38:(524288,655359,17),
+                      39:(655360,786431,17),
+                      40:(786432,917503,17),
+                      41:(917504,1048575,17),
+                      42:(1048576,1179647,17),
+                      43:(1179648,1310719,17),
+                      44:(1310720,1441791,17),
+                      45:(1441792,1572863,17),
+                      46:(1572864,1703935,17),
+                      47:(1703936,1835007,17),
+                      48:(1835008,1966079,17),
+                      49:(1966080,2097151,17),
+                      }
+    def createFields(self):
+        yield Bits(self, "block_type", 3)
+        yield Bits(self, "block_size", 24)
+        self.uncompressed_size = self["block_size"].value
+        self.compression_level = self.root.compr_level
+        self.window_size = self.WINDOW_SIZE[self.compression_level]
+        self.block_type = self["block_type"].value
+        curlen = len(self.parent.uncompressed_data)
+        if self.block_type in (1, 2): # Verbatim or aligned offset block
+            if self.block_type == 2:
+                for i in xrange(8):
+                    yield Bits(self, "aligned_len[]", 3)
+                aligned_tree = build_tree([self['aligned_len[%d]'%i].value for i in xrange(8)])
+            yield LZXPreTreeEncodedTree(self, "main_tree_start", 256)
+            yield LZXPreTreeEncodedTree(self, "main_tree_rest", self.window_size * 8)
+            main_tree = build_tree(self["main_tree_start"].lengths + self["main_tree_rest"].lengths)
+            yield LZXPreTreeEncodedTree(self, "length_tree", 249)
+            length_tree = build_tree(self["length_tree"].lengths)
+            current_decoded_size = 0
+            while current_decoded_size < self.uncompressed_size:
+                if (curlen+current_decoded_size) % 32768 == 0 and (curlen+current_decoded_size) != 0:
+                    padding = paddingSize(self.address + self.current_size, 16)
+                    if padding:
+                        yield PaddingBits(self, "padding[]", padding)
+                field = HuffmanCode(self, "main_code[]", main_tree)
+                if field.realvalue < 256:
+                    field._description = "Literal value %r" % chr(field.realvalue)
+                    current_decoded_size += 1
+                    self.parent.uncompressed_data += chr(field.realvalue)
+                    yield field
+                    continue
+                position_header, length_header = divmod(field.realvalue - 256, 8)
+                info = self.POSITION_SLOTS[position_header]
+                if info[2] == 0:
+                    if info[0] == 0:
+                        position = self.parent.r0
+                        field._description = "Position Slot %i, Position [R0] (%i)" % (position_header, position)
+                    elif info[0] == 1:
+                        position = self.parent.r1
+                        self.parent.r1 = self.parent.r0
+                        self.parent.r0 = position
+                        field._description = "Position Slot %i, Position [R1] (%i)" % (position_header, position)
+                    elif info[0] == 2:
+                        position = self.parent.r2
+                        self.parent.r2 = self.parent.r0
+                        self.parent.r0 = position
+                        field._description = "Position Slot %i, Position [R2] (%i)" % (position_header, position)
+                    else:
+                        position = info[0] - 2
+                        self.parent.r2 = self.parent.r1
+                        self.parent.r1 = self.parent.r0
+                        self.parent.r0 = position
+                        field._description = "Position Slot %i, Position %i" % (position_header, position)
+                else:
+                    field._description = "Position Slot %i, Positions %i to %i" % (position_header, info[0] - 2, info[1] - 2)
+                if length_header == 7:
+                    field._description += ", Length Values 9 and up"
+                    yield field
+                    length_field = HuffmanCode(self, "length_code[]", length_tree)
+                    length = length_field.realvalue + 9
+                    length_field._description = "Length Code %i, total length %i" % (length_field.realvalue, length)
+                    yield length_field
+                else:
+                    field._description += ", Length Value %i (Huffman Code %i)"%(length_header + 2, field.value)
+                    yield field
+                    length = length_header + 2
+                if info[2]:
+                    if self.block_type == 1 or info[2] < 3: # verbatim
+                        extrafield = Bits(self, "position_extra[%s" % field.name.split('[')[1], info[2])
+                        position = extrafield.value + info[0] - 2
+                        extrafield._description = "Position Extra Bits (%i), total position %i"%(extrafield.value, position)
+                        yield extrafield
+                    else: # aligned offset
+                        position = info[0] - 2
+                        if info[2] > 3:
+                            extrafield = Bits(self, "position_verbatim[%s" % field.name.split('[')[1], info[2]-3)
+                            position += extrafield.value*8
+                            extrafield._description = "Position Verbatim Bits (%i), added position %i"%(extrafield.value, extrafield.value*8)
+                            yield extrafield
+                        if info[2] >= 3:
+                            extrafield = HuffmanCode(self, "position_aligned[%s" % field.name.split('[')[1], aligned_tree)
+                            position += extrafield.realvalue
+                            extrafield._description = "Position Aligned Bits (%i), total position %i"%(extrafield.realvalue, position)
+                            yield extrafield
+                    self.parent.r2 = self.parent.r1
+                    self.parent.r1 = self.parent.r0
+                    self.parent.r0 = position
+                self.parent.uncompressed_data = extend_data(self.parent.uncompressed_data, length, position)
+                current_decoded_size += length
+        elif self.block_type == 3: # Uncompressed block
+            padding = paddingSize(self.address + self.current_size, 16)
+            if padding:
+                yield PaddingBits(self, "padding[]", padding)
+            else:
+                yield PaddingBits(self, "padding[]", 16)
+            self.endian = LITTLE_ENDIAN
+            yield UInt32(self, "r[]", "New value of R0")
+            yield UInt32(self, "r[]", "New value of R1")
+            yield UInt32(self, "r[]", "New value of R2")
+            self.parent.r0 = self["r[0]"].value
+            self.parent.r1 = self["r[1]"].value
+            self.parent.r2 = self["r[2]"].value
+            yield RawBytes(self, "data", self.uncompressed_size)
+            self.parent.uncompressed_data+=self["data"].value
+            if self["block_size"].value % 2:
+                yield PaddingBits(self, "padding", 8)
+        else:
+            raise ParserError("Unknown block type %d!"%self.block_type)
+
+class LZXStream(Parser):
+    endian = MIDDLE_ENDIAN
+    def createFields(self):
+        self.uncompressed_data = ""
+        self.r0 = 1
+        self.r1 = 1
+        self.r2 = 1
+        yield Bit(self, "filesize_indicator")
+        if self["filesize_indicator"].value:
+            yield UInt32(self, "filesize")
+        while self.current_size < self.size:
+            block = LZXBlock(self, "block[]")
+            yield block
+            if self.size - self.current_size < 16:
+                padding = paddingSize(self.address + self.current_size, 16)
+                if padding:
+                    yield PaddingBits(self, "padding[]", padding)
+                break
+
+def lzx_decompress(stream, window_bits):
+    data = LZXStream(stream)
+    data.compr_level = window_bits
+    for unused in data:
+        pass
+    return data.uncompressed_data

lib/hachoir_parser/archive/mar.py

@@ -7,10 +7,10 @@ Creation date: 2007-03-04
 
 MAX_NB_FILE = 100000
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import FieldSet, String, UInt32, SubFile
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
+from hachoir_parser import Parser
+from hachoir_core.field import FieldSet, String, UInt32, SubFile
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
 
 class FileIndex(FieldSet):
     static_size = 68*8

lib/hachoir_parser/archive/mozilla_ar.py

@@ -0,0 +1,60 @@
+"""MAR (Mozilla ARchive) parser
+
+Author: Robert Xiao
+Creation date: July 10, 2007
+
+"""
+
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.field import (RootSeekableFieldSet, FieldSet,
+    String, CString, UInt32, RawBytes)
+from hachoir_core.text_handler import displayHandler, filesizeHandler
+from hachoir_core.tools import humanUnixAttributes
+from hachoir_parser import HachoirParser
+
+class IndexEntry(FieldSet):
+    def createFields(self):
+        yield UInt32(self, "offset", "Offset in bytes relative to start of archive")
+        yield filesizeHandler(UInt32(self, "length", "Length in bytes"))
+        yield displayHandler(UInt32(self, "flags"), humanUnixAttributes)
+        yield CString(self, "name", "Filename (byte array)")
+
+    def createDescription(self):
+        return 'File %s, Size %s, Mode %s'%(
+            self["name"].display, self["length"].display, self["flags"].display)
+
+class MozillaArchive(HachoirParser, RootSeekableFieldSet):
+    MAGIC = "MAR1"
+    PARSER_TAGS = {
+        "id": "mozilla_ar",
+        "category": "archive",
+        "file_ext": ("mar",),
+        "min_size": (8+4+13)*8,  # Header, Index Header, 1 Index Entry
+        "magic": ((MAGIC, 0),),
+        "description": "Mozilla Archive",
+    }
+    endian = BIG_ENDIAN
+    
+    def __init__(self, stream, **args):
+        RootSeekableFieldSet.__init__(self, None, "root", stream, None, stream.askSize(self))
+        HachoirParser.__init__(self, stream, **args)
+        
+    def validate(self):
+        if self.stream.readBytes(0, 4) != self.MAGIC:
+            return "Invalid magic"
+        return True
+
+    def createFields(self):
+        yield String(self, "magic", 4, "File signature (MAR1)", charset="ASCII")
+        yield UInt32(self, "index_offset", "Offset to index relative to file start")
+        self.seekByte(self["index_offset"].value, False)
+        yield UInt32(self, "index_size", "size of index in bytes")
+        current_index_size = 0 # bytes
+        while current_index_size < self["index_size"].value:
+            # plus 4 compensates for index_size
+            self.seekByte(self["index_offset"].value + current_index_size + 4, False)
+            entry = IndexEntry(self, "index_entry[]")
+            yield entry
+            current_index_size += entry.size // 8
+            self.seekByte(entry["offset"].value, False)
+            yield RawBytes(self, "file[]", entry["length"].value)

lib/hachoir_parser/archive/rar.py

@@ -5,15 +5,15 @@ Status: can only read higher-level attructures
 Author: Christophe Gisquet
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (StaticFieldSet, FieldSet,
+from hachoir_parser import Parser
+from hachoir_core.field import (StaticFieldSet, FieldSet,
     Bit, Bits, Enum,
     UInt8, UInt16, UInt32, UInt64,
     String, TimeDateMSDOS32,
     NullBytes, NullBits, RawBytes)
-from lib.hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_parser.common.msdos import MSDOSFileAttr32
+from hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_parser.common.msdos import MSDOSFileAttr32
 
 MAX_FILESIZE = 1000 * 1024 * 1024
 

lib/hachoir_parser/archive/rpm.py

@@ -4,14 +4,14 @@ RPM archive parser.
 Author: Victor Stinner, 1st December 2005.
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet, ParserError,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet, ParserError,
     UInt8, UInt16, UInt32, UInt64, Enum,
     NullBytes, Bytes, RawBytes, SubFile,
     Character, CString, String)
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_parser.archive.gzip_parser import GzipParser
-from lib.hachoir_parser.archive.bzip2_parser import Bzip2Parser
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_parser.archive.gzip_parser import GzipParser
+from hachoir_parser.archive.bzip2_parser import Bzip2Parser
 
 class ItemContent(FieldSet):
     format_type = {

lib/hachoir_parser/archive/sevenzip.py

@@ -9,13 +9,13 @@ Author: Olivier SCHWAB
 Creation date: 6 december 2006
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (Field, FieldSet, ParserError,
+from hachoir_parser import Parser
+from hachoir_core.field import (Field, FieldSet, ParserError,
     GenericVector,
     Enum, UInt8, UInt32, UInt64,
     Bytes, RawBytes)
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.text_handler import textHandler, hexadecimal, filesizeHandler
 
 class SZUInt64(Field):
     """

lib/hachoir_parser/archive/tar.py

@@ -4,11 +4,11 @@ Tar archive parser.
 Author: Victor Stinner
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet,
     Enum, UInt8, SubFile, String, NullBytes)
-from lib.hachoir_core.tools import humanFilesize, paddingSize, timestampUNIX
-from lib.hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.tools import humanFilesize, paddingSize, timestampUNIX
+from hachoir_core.endian import BIG_ENDIAN
 import re
 
 class FileEntry(FieldSet):

lib/hachoir_parser/archive/zip.py

@@ -5,18 +5,18 @@ Status: can read most important headers
 Authors: Christophe Gisquet and Victor Stinner
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet, ParserError,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet, ParserError,
     Bit, Bits, Enum,
     TimeDateMSDOS32, SubFile,
     UInt8, UInt16, UInt32, UInt64,
     String, PascalString16,
     RawBytes)
-from lib.hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
-from lib.hachoir_core.error import HACHOIR_ERRORS
-from lib.hachoir_core.tools import makeUnicode
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_parser.common.deflate import Deflate
+from hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
+from hachoir_core.error import HACHOIR_ERRORS
+from hachoir_core.tools import makeUnicode
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_parser.common.deflate import Deflate
 
 MAX_FILESIZE = 1000 * 1024 * 1024
 
@@ -80,16 +80,7 @@ class ZipGeneralFlags(FieldSet):
         # Need the compression info from the parent, and that is the byte following
         method = self.stream.readBits(self.absolute_address+16, 16, LITTLE_ENDIAN)
 
-        yield Bits(self, "unused[]", 2, "Unused")
-        yield Bit(self, "encrypted_central_dir", "Selected data values in the Local Header are masked")
-        yield Bit(self, "incomplete", "Reserved by PKWARE for enhanced compression.")
-        yield Bit(self, "uses_unicode", "Filename and comments are in UTF-8")
-        yield Bits(self, "unused[]", 4, "Unused")
-        yield Bit(self, "strong_encrypt", "Strong encryption (version >= 50)")
-        yield Bit(self, "is_patched", "File is compressed with patched data?")
-        yield Bit(self, "enhanced_deflate", "Reserved for use with method 8")
-        yield Bit(self, "has_descriptor",
-                  "Compressed data followed by descriptor?")
+        yield Bit(self, "is_encrypted", "File is encrypted?")
         if method == 6:
             yield Bit(self, "use_8k_sliding", "Use 8K sliding dictionary (instead of 4K)")
             yield Bit(self, "use_3shannon", "Use a 3 Shannon-Fano tree (instead of 2 Shannon-Fano)")
@@ -106,7 +97,16 @@ class ZipGeneralFlags(FieldSet):
             yield Bit(self, "unused[]")
         else:
             yield Bits(self, "compression_info", 2)
-        yield Bit(self, "is_encrypted", "File is encrypted?")
+        yield Bit(self, "has_descriptor",
+                  "Compressed data followed by descriptor?")
+        yield Bit(self, "enhanced_deflate", "Reserved for use with method 8")
+        yield Bit(self, "is_patched", "File is compressed with patched data?")
+        yield Bit(self, "strong_encrypt", "Strong encryption (version >= 50)")
+        yield Bits(self, "unused[]", 4, "Unused")
+        yield Bit(self, "uses_unicode", "Filename and comments are in UTF-8")
+        yield Bit(self, "incomplete", "Reserved by PKWARE for enhanced compression.")
+        yield Bit(self, "encrypted_central_dir", "Selected data values in the Local Header are masked")
+        yield Bits(self, "unused[]", 2, "Unused")
 
 class ExtraField(FieldSet):
     EXTRA_FIELD_ID = {
@@ -141,7 +141,12 @@ class ExtraField(FieldSet):
         size = UInt16(self, "field_data_size", "Extra field data size")
         yield size
         if size.value > 0:
-            yield RawBytes(self, "field_data", size, "Unknown field data")
+            yield RawBytes(self, "field_data", size.value, "Unknown field data")
+
+class ExtraFields(FieldSet):
+    def createFields(self):
+        while self.current_size < self.size:
+            yield ExtraField(self, "extra[]")
 
 def ZipStartCommonFields(self):
     yield ZipVersion(self, "version_needed", "Version needed")
@@ -179,8 +184,8 @@ class ZipCentralDirectory(FieldSet):
         yield String(self, "filename", self["filename_length"].value,
                      "Filename", charset=charset)
         if 0 < self["extra_length"].value:
-            yield RawBytes(self, "extra", self["extra_length"].value,
-                           "Extra fields")
+            yield ExtraFields(self, "extra", size=self["extra_length"].value*8,
+                           description="Extra fields")
         if 0 < self["comment_length"].value:
             yield String(self, "comment", self["comment_length"].value,
                          "Comment", charset=charset)
@@ -278,14 +283,15 @@ class FileEntry(FieldSet):
             yield filename
             self.filename = filename.value
         if self["extra_length"].value:
-            yield RawBytes(self, "extra", self["extra_length"].value, "Extra")
+            yield ExtraFields(self, "extra", size=self["extra_length"].value*8,
+                           description="Extra fields")
         size = self["compressed_size"].value
         if size > 0:
             yield self.data(size)
         elif self["flags/incomplete"].value:
             for field in self.resync():
                 yield field
-        if self["flags/has_descriptor"].value:
+        if self["flags/has_descriptor"].value and self['crc32'].value == 0:
             yield ZipDataDescriptor(self, "data_desc", "Data descriptor")
 
     def createDescription(self):

lib/hachoir_parser/archive/zlib.py

@@ -0,0 +1,301 @@
+"""Detailed ZLIB parser
+
+Author: Robert Xiao
+Creation date: July 9 2007
+
+"""
+
+from hachoir_parser import Parser
+from hachoir_core.field import (Bit, Bits, Field, Int16, UInt32,
+    Enum, FieldSet, GenericFieldSet,
+    PaddingBits, ParserError, RawBytes)
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_core.tools import paddingSize, alignValue
+
+def extend_data(data, length, offset):
+    """Extend data using a length and an offset."""
+    if length >= offset:
+        new_data = data[-offset:] * (alignValue(length, offset) // offset)
+        return data + new_data[:length]
+    else:
+        return data + data[-offset:-offset+length]
+
+def build_tree(lengths):
+    """Build a Huffman tree from a list of lengths.
+       The ith entry of the input list is the length of the Huffman code corresponding to
+       integer i, or 0 if the integer i is unused."""
+    max_length = max(lengths) + 1
+    bit_counts = [0]*max_length
+    next_code = [0]*max_length
+    tree = {}
+    for i in lengths:
+        if i:
+            bit_counts[i] += 1
+    code = 0
+    for i in xrange(1, len(bit_counts)):
+        next_code[i] = code = (code + bit_counts[i-1]) << 1
+    for i, ln in enumerate(lengths):
+        if ln:
+            tree[(ln, next_code[ln])] = i
+            next_code[ln] += 1
+    return tree
+
+class HuffmanCode(Field):
+    """Huffman code. Uses tree parameter as the Huffman tree."""
+    def __init__(self, parent, name, tree, description=None):
+        Field.__init__(self, parent, name, 0, description)
+
+        endian = self.parent.endian
+        stream = self.parent.stream
+        addr = self.absolute_address
+
+        value = 0
+        while (self.size, value) not in tree:
+            if self.size > 256:
+                raise ParserError("Huffman code too long!")
+            bit = stream.readBits(addr, 1, endian)
+            value <<= 1
+            value += bit
+            self._size += 1
+            addr += 1
+        self.huffvalue = value
+        self.realvalue = tree[(self.size, value)]
+    def createValue(self):
+        return self.huffvalue
+
+class DeflateBlock(FieldSet):
+    # code: (min, max, extrabits)
+    LENGTH_SYMBOLS = {257:(3,3,0),
+                      258:(4,4,0),
+                      259:(5,5,0),
+                      260:(6,6,0),
+                      261:(7,7,0),
+                      262:(8,8,0),
+                      263:(9,9,0),
+                      264:(10,10,0),
+                      265:(11,12,1),
+                      266:(13,14,1),
+                      267:(15,16,1),
+                      268:(17,18,1),
+                      269:(19,22,2),
+                      270:(23,26,2),
+                      271:(27,30,2),
+                      272:(31,34,2),
+                      273:(35,42,3),
+                      274:(43,50,3),
+                      275:(51,58,3),
+                      276:(59,66,3),
+                      277:(67,82,4),
+                      278:(83,98,4),
+                      279:(99,114,4),
+                      280:(115,130,4),
+                      281:(131,162,5),
+                      282:(163,194,5),
+                      283:(195,226,5),
+                      284:(227,257,5),
+                      285:(258,258,0)
+                      }
+    DISTANCE_SYMBOLS = {0:(1,1,0),
+                        1:(2,2,0),
+                        2:(3,3,0),
+                        3:(4,4,0),
+                        4:(5,6,1),
+                        5:(7,8,1),
+                        6:(9,12,2),
+                        7:(13,16,2),
+                        8:(17,24,3),
+                        9:(25,32,3),
+                        10:(33,48,4),
+                        11:(49,64,4),
+                        12:(65,96,5),
+                        13:(97,128,5),
+                        14:(129,192,6),
+                        15:(193,256,6),
+                        16:(257,384,7),
+                        17:(385,512,7),
+                        18:(513,768,8),
+                        19:(769,1024,8),
+                        20:(1025,1536,9),
+                        21:(1537,2048,9),
+                        22:(2049,3072,10),
+                        23:(3073,4096,10),
+                        24:(4097,6144,11),
+                        25:(6145,8192,11),
+                        26:(8193,12288,12),
+                        27:(12289,16384,12),
+                        28:(16385,24576,13),
+                        29:(24577,32768,13),
+                        }
+    CODE_LENGTH_ORDER = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]
+    def __init__(self, parent, name, uncomp_data="", *args, **kwargs):
+        FieldSet.__init__(self, parent, name, *args, **kwargs)
+        self.uncomp_data = uncomp_data
+        
+    def createFields(self):
+        yield Bit(self, "final", "Is this the final block?") # BFINAL
+        yield Enum(Bits(self, "compression_type", 2), # BTYPE
+                   {0:"None", 1:"Fixed Huffman", 2:"Dynamic Huffman", 3:"Reserved"})
+        if self["compression_type"].value == 0: # no compression
+            padding = paddingSize(self.current_size + self.absolute_address, 8) # align on byte boundary
+            if padding:
+                yield PaddingBits(self, "padding[]", padding)
+            yield Int16(self, "len")
+            yield Int16(self, "nlen", "One's complement of len")
+            if self["len"].value != ~self["nlen"].value:
+                raise ParserError("len must be equal to the one's complement of nlen!")
+            if self["len"].value: # null stored blocks produced by some encoders (e.g. PIL)
+                yield RawBytes(self, "data", self["len"].value, "Uncompressed data")
+            return
+        elif self["compression_type"].value == 1: # Fixed Huffman
+            length_tree = {} # (size, huffman code): value
+            distance_tree = {}
+            for i in xrange(144):
+                length_tree[(8, i+48)] = i
+            for i in xrange(144, 256):
+                length_tree[(9, i+256)] = i
+            for i in xrange(256, 280):
+                length_tree[(7, i-256)] = i
+            for i in xrange(280, 288):
+                length_tree[(8, i-88)] = i
+            for i in xrange(32):
+                distance_tree[(5, i)] = i
+        elif self["compression_type"].value == 2: # Dynamic Huffman
+            yield Bits(self, "huff_num_length_codes", 5, "Number of Literal/Length Codes, minus 257")
+            yield Bits(self, "huff_num_distance_codes", 5, "Number of Distance Codes, minus 1")
+            yield Bits(self, "huff_num_code_length_codes", 4, "Number of Code Length Codes, minus 4")
+            code_length_code_lengths = [0]*19 # confusing variable name...
+            for i in self.CODE_LENGTH_ORDER[:self["huff_num_code_length_codes"].value+4]:
+                field = Bits(self, "huff_code_length_code[%i]" % i, 3, "Code lengths for the code length alphabet")
+                yield field
+                code_length_code_lengths[i] = field.value
+            code_length_tree = build_tree(code_length_code_lengths)
+            length_code_lengths = []
+            distance_code_lengths = []
+            for numcodes, name, lengths in (
+                (self["huff_num_length_codes"].value + 257, "length", length_code_lengths),
+                (self["huff_num_distance_codes"].value + 1, "distance", distance_code_lengths)):
+                while len(lengths) < numcodes:
+                    field = HuffmanCode(self, "huff_%s_code[]" % name, code_length_tree)
+                    value = field.realvalue
+                    if value < 16:
+                        prev_value = value
+                        field._description = "Literal Code Length %i (Huffman Code %i)" % (value, field.value)
+                        yield field
+                        lengths.append(value)
+                    else:
+                        info = {16: (3,6,2),
+                                17: (3,10,3),
+                                18: (11,138,7)}[value]
+                        if value == 16:
+                            repvalue = prev_value
+                        else:
+                            repvalue = 0
+                        field._description = "Repeat Code %i, Repeating value (%i) %i to %i times (Huffman Code %i)" % (value, repvalue, info[0], info[1], field.value)
+                        yield field
+                        extrafield = Bits(self, "huff_%s_code_extra[%s" % (name, field.name.split('[')[1]), info[2])
+                        num_repeats = extrafield.value+info[0]
+                        extrafield._description = "Repeat Extra Bits (%i), total repeats %i"%(extrafield.value, num_repeats)
+                        yield extrafield
+                        lengths += [repvalue]*num_repeats
+            length_tree = build_tree(length_code_lengths)
+            distance_tree = build_tree(distance_code_lengths)
+        else:
+            raise ParserError("Unsupported compression type 3!")
+        while True:
+            field = HuffmanCode(self, "length_code[]", length_tree)
+            value = field.realvalue
+            if value < 256:
+                field._description = "Literal Code %r (Huffman Code %i)" % (chr(value), field.value)
+                yield field
+                self.uncomp_data += chr(value)
+            if value == 256:
+                field._description = "Block Terminator Code (256) (Huffman Code %i)" % field.value
+                yield field
+                break
+            elif value > 256:
+                info = self.LENGTH_SYMBOLS[value]
+                if info[2] == 0:
+                    field._description = "Length Code %i, Value %i (Huffman Code %i)" % (value, info[0], field.value)
+                    length = info[0]
+                    yield field
+                else:
+                    field._description = "Length Code %i, Values %i to %i (Huffman Code %i)" % (value, info[0], info[1], field.value)
+                    yield field
+                    extrafield = Bits(self, "length_extra[%s" % field.name.split('[')[1], info[2])
+                    length = extrafield.value + info[0]
+                    extrafield._description = "Length Extra Bits (%i), total length %i"%(extrafield.value, length)
+                    yield extrafield
+                field = HuffmanCode(self, "distance_code[]", distance_tree)
+                value = field.realvalue
+                info = self.DISTANCE_SYMBOLS[value]
+                if info[2] == 0:
+                    field._description = "Distance Code %i, Value %i (Huffman Code %i)" % (value, info[0], field.value)
+                    distance = info[0]
+                    yield field
+                else:
+                    field._description = "Distance Code %i, Values %i to %i (Huffman Code %i)" % (value, info[0], info[1], field.value)
+                    yield field
+                    extrafield = Bits(self, "distance_extra[%s" % field.name.split('[')[1], info[2])
+                    distance = extrafield.value + info[0]
+                    extrafield._description = "Distance Extra Bits (%i), total length %i"%(extrafield.value, distance)
+                    yield extrafield
+                self.uncomp_data = extend_data(self.uncomp_data, length, distance)
+
+class DeflateData(GenericFieldSet):
+    endian = LITTLE_ENDIAN
+    def createFields(self):
+        uncomp_data = ""
+        blk=DeflateBlock(self, "compressed_block[]", uncomp_data)
+        yield blk
+        uncomp_data = blk.uncomp_data
+        while not blk["final"].value:
+            blk=DeflateBlock(self, "compressed_block[]", uncomp_data)
+            yield blk
+            uncomp_data = blk.uncomp_data
+        padding = paddingSize(self.current_size + self.absolute_address, 8) # align on byte boundary
+        if padding:
+            yield PaddingBits(self, "padding[]", padding)
+        self.uncompressed_data = uncomp_data
+
+class ZlibData(Parser):
+    PARSER_TAGS = {
+        "id": "zlib",
+        "category": "archive",
+        "file_ext": ("zlib",),
+        "min_size": 8*8,
+        "description": "ZLIB Data",
+    }
+    endian = LITTLE_ENDIAN
+
+    def validate(self):
+        if self["compression_method"].value != 8:
+            return "Incorrect compression method"
+        if ((self["compression_info"].value << 12) +
+            (self["compression_method"].value << 8) +
+            (self["flag_compression_level"].value << 6) +
+            (self["flag_dictionary_present"].value << 5) +
+            (self["flag_check_bits"].value)) % 31 != 0:
+            return "Invalid flag check value"
+        return True
+
+    def createFields(self):
+        yield Enum(Bits(self, "compression_method", 4), {8:"deflate", 15:"reserved"}) # CM
+        yield Bits(self, "compression_info", 4, "base-2 log of the window size") # CINFO
+        yield Bits(self, "flag_check_bits", 5) # FCHECK
+        yield Bit(self, "flag_dictionary_present") # FDICT
+        yield Enum(Bits(self, "flag_compression_level", 2), # FLEVEL
+                   {0:"Fastest", 1:"Fast", 2:"Default", 3:"Maximum, Slowest"})
+        if self["flag_dictionary_present"].value:
+            yield textHandler(UInt32(self, "dict_checksum", "ADLER32 checksum of dictionary information"), hexadecimal)
+        yield DeflateData(self, "data", self.stream, description = "Compressed Data")
+        yield textHandler(UInt32(self, "data_checksum", "ADLER32 checksum of compressed data"), hexadecimal)
+
+def zlib_inflate(stream, wbits=None, prevdata=""):
+    if wbits is None or wbits >= 0:
+        return ZlibData(stream)["data"].uncompressed_data
+    else:
+        data = DeflateData(None, "root", stream, "", stream.askSize(None))
+        for unused in data:
+            pass
+        return data.uncompressed_data

lib/hachoir_parser/audio/8svx.py

@@ -1,126 +0,0 @@
-"""
-Audio Interchange File Format (AIFF) parser.
-
-Author: Victor Stinner
-Creation: 27 december 2006
-"""
-
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet,
-    UInt16, UInt32, Float80, TimestampMac32,
-    RawBytes, NullBytes,
-    String, Enum, PascalString32)
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.text_handler import filesizeHandler
-from lib.hachoir_core.tools import alignValue
-from lib.hachoir_parser.audio.id3 import ID3v2
-
-CODEC_NAME = {
-    'ACE2': u"ACE 2-to-1",
-    'ACE8': u"ACE 8-to-3",
-    'MAC3': u"MAC 3-to-1",
-    'MAC6': u"MAC 6-to-1",
-    'NONE': u"None",
-    'sowt': u"Little-endian, no compression",
-}
-
-class Comment(FieldSet):
-    def createFields(self):
-        yield TimestampMac32(self, "timestamp")
-        yield PascalString32(self, "text")
-
-def parseText(self):
-    yield String(self, "text", self["size"].value)
-
-def parseID3(self):
-    yield ID3v2(self, "id3v2", size=self["size"].value*8)
-
-def parseComment(self):
-    yield UInt16(self, "nb_comment")
-    for index in xrange(self["nb_comment"].value):
-        yield Comment(self, "comment[]")
-
-def parseCommon(self):
-    yield UInt16(self, "nb_channel")
-    yield UInt32(self, "nb_sample")
-    yield UInt16(self, "sample_size")
-    yield Float80(self, "sample_rate")
-    yield Enum(String(self, "codec", 4, strip="\0", charset="ASCII"), CODEC_NAME)
-
-def parseVersion(self):
-    yield TimestampMac32(self, "timestamp")
-
-def parseSound(self):
-    yield UInt32(self, "offset")
-    yield UInt32(self, "block_size")
-    size = (self.size - self.current_size) // 8
-    if size:
-        yield RawBytes(self, "data", size)
-
-class Chunk(FieldSet):
-    TAG_INFO = {
-        'COMM': ('common', "Common chunk", parseCommon),
-        'COMT': ('comment', "Comment", parseComment),
-        'NAME': ('name', "Name", parseText),
-        'AUTH': ('author', "Author", parseText),
-        'FVER': ('version', "Version", parseVersion),
-        'SSND': ('sound', "Sound data", parseSound),
-        'ID3 ': ('id3', "ID3", parseID3),
-    }
-
-    def __init__(self, *args):
-        FieldSet.__init__(self, *args)
-        self._size = (8 + alignValue(self["size"].value, 2)) * 8
-        tag = self["type"].value
-        if tag in self.TAG_INFO:
-            self._name, self._description, self._parser = self.TAG_INFO[tag]
-        else:
-            self._parser = None
-
-    def createFields(self):
-        yield String(self, "type", 4, "Signature (FORM)", charset="ASCII")
-        yield filesizeHandler(UInt32(self, "size"))
-        size = self["size"].value
-        if size:
-            if self._parser:
-                for field in self._parser(self):
-                    yield field
-                if size % 2:
-                    yield NullBytes(self, "padding", 1)
-            else:
-                yield RawBytes(self, "data", size)
-
-class HeightSVX(Parser):
-    PARSER_TAGS = {
-        "id": "8svx",
-        "category": "audio",
-        "file_ext": ("8svx",),
-        "mime": (u"audio/x-aiff",),
-        "min_size": 12*8,
-        "description": "8SVX (audio) format"
-    }
-    endian = BIG_ENDIAN
-
-    def validate(self):
-        if self.stream.readBytes(0, 4) != "FORM":
-            return "Invalid signature"
-        if self.stream.readBytes(8*8, 4) != "8SVX":
-            return "Invalid type"
-        return True
-
-    def createFields(self):
-        yield String(self, "signature", 4, "Signature (FORM)", charset="ASCII")
-        yield filesizeHandler(UInt32(self, "filesize"))
-        yield String(self, "type", 4, "Form type (AIFF or AIFC)", charset="ASCII")
-        while not self.eof:
-            yield Chunk(self, "chunk[]")
-
-    def createDescription(self):
-        if self["type"].value == "AIFC":
-            return "Audio Interchange File Format Compressed (AIFC)"
-        else:
-            return "Audio Interchange File Format (AIFF)"
-
-    def createContentSize(self):
-        return self["filesize"].value * 8
-

lib/hachoir_parser/audio/__init__.py

@@ -1,12 +1,12 @@
-from lib.hachoir_parser.audio.aiff import AiffFile
-from lib.hachoir_parser.audio.au import AuFile
-from lib.hachoir_parser.audio.itunesdb import ITunesDBFile
-from lib.hachoir_parser.audio.midi import MidiFile
-from lib.hachoir_parser.audio.mpeg_audio import MpegAudioFile
-from lib.hachoir_parser.audio.real_audio import RealAudioFile
-from lib.hachoir_parser.audio.xm import XMModule
-from lib.hachoir_parser.audio.s3m import S3MModule
-from lib.hachoir_parser.audio.s3m import PTMModule
-from lib.hachoir_parser.audio.mod import AmigaModule
-from lib.hachoir_parser.audio.flac import FlacParser
+from hachoir_parser.audio.aiff import AiffFile
+from hachoir_parser.audio.au import AuFile
+from hachoir_parser.audio.itunesdb import ITunesDBFile
+from hachoir_parser.audio.midi import MidiFile
+from hachoir_parser.audio.mpeg_audio import MpegAudioFile
+from hachoir_parser.audio.real_audio import RealAudioFile
+from hachoir_parser.audio.xm import XMModule
+from hachoir_parser.audio.s3m import S3MModule
+from hachoir_parser.audio.s3m import PTMModule
+from hachoir_parser.audio.mod import AmigaModule
+from hachoir_parser.audio.flac import FlacParser
 

lib/hachoir_parser/audio/aiff.py

@@ -5,15 +5,15 @@ Author: Victor Stinner
 Creation: 27 december 2006
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet,
     UInt16, UInt32, Float80, TimestampMac32,
     RawBytes, NullBytes,
     String, Enum, PascalString32)
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.text_handler import filesizeHandler
-from lib.hachoir_core.tools import alignValue
-from lib.hachoir_parser.audio.id3 import ID3v2
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.text_handler import filesizeHandler
+from hachoir_core.tools import alignValue
+from hachoir_parser.audio.id3 import ID3v2
 
 CODEC_NAME = {
     'ACE2': u"ACE 2-to-1",

lib/hachoir_parser/audio/au.py

@@ -5,11 +5,11 @@ Author: Victor Stinner
 Creation: 12 july 2006
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import UInt32, Enum, String, RawBytes
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.text_handler import displayHandler, filesizeHandler
-from lib.hachoir_core.tools import createDict, humanFrequency
+from hachoir_parser import Parser
+from hachoir_core.field import UInt32, Enum, String, RawBytes
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.text_handler import displayHandler, filesizeHandler
+from hachoir_core.tools import createDict, humanFrequency
 
 class AuFile(Parser):
     PARSER_TAGS = {

lib/hachoir_parser/audio/flac.py

@@ -9,11 +9,11 @@ Author: Esteban Loiseau <baal AT tuxfamily.org>
 Creation date: 2008-04-09
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import FieldSet, String, Bit, Bits, UInt16, UInt24, RawBytes, Enum, NullBytes
-from lib.hachoir_core.stream import BIG_ENDIAN, LITTLE_ENDIAN
-from lib.hachoir_core.tools import createDict
-from lib.hachoir_parser.container.ogg import parseVorbisComment
+from hachoir_parser import Parser
+from hachoir_core.field import FieldSet, String, Bit, Bits, UInt16, UInt24, RawBytes, Enum, NullBytes
+from hachoir_core.stream import BIG_ENDIAN, LITTLE_ENDIAN
+from hachoir_core.tools import createDict
+from hachoir_parser.container.ogg import parseVorbisComment
 
 class VorbisComment(FieldSet):
     endian = LITTLE_ENDIAN

lib/hachoir_parser/audio/id3.py

@@ -6,13 +6,13 @@ Informations: http://www.id3.org/
 Author: Victor Stinner
 """
 
-from lib.hachoir_core.field import (FieldSet, MatchError, ParserError,
+from hachoir_core.field import (FieldSet, MatchError, ParserError,
     Enum, UInt8, UInt24, UInt32,
     CString, String, RawBytes,
     Bit, Bits, NullBytes, NullBits)
-from lib.hachoir_core.text_handler import textHandler
-from lib.hachoir_core.tools import humanDuration
-from lib.hachoir_core.endian import NETWORK_ENDIAN
+from hachoir_core.text_handler import textHandler
+from hachoir_core.tools import humanDuration
+from hachoir_core.endian import NETWORK_ENDIAN
 
 class ID3v1(FieldSet):
     static_size = 128 * 8

lib/hachoir_parser/audio/itunesdb.py

@@ -8,13 +8,13 @@ Author: Romain HERAULT
 Creation date: 19 august 2006
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet,
-    UInt8, UInt16, UInt32, UInt64, TimestampMac32,
-    String, Float32, NullBytes, Enum)
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_core.tools import humanDuration
-from lib.hachoir_core.text_handler import displayHandler, filesizeHandler
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet,
+    UInt8, UInt16, UInt32, Int32, UInt64, TimestampMac32,
+    String, Float32, NullBytes, Enum, RawBytes)
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.tools import humanDuration
+from hachoir_core.text_handler import displayHandler, filesizeHandler
 
 list_order={
         1 : "playlist order (manual sort order)",
@@ -75,6 +75,9 @@ class DataObject(FieldSet):
         51:"Smart Playlist Rules",
         52:"Library Playlist Index",
         100:"Column info",
+        200:"Album name (for album descriptions)",
+        201:"Album artist (for album descriptions)",
+        202:"Album sort artist (for album descriptions)"
     }
 
     mhod52_sort_index_type_name={
@@ -94,7 +97,7 @@ class DataObject(FieldSet):
         yield UInt32(self, "header_length", "Header Length")
         yield UInt32(self, "entry_length", "Entry Length")
         yield Enum(UInt32(self, "type", "type"),self.type_name)
-        if(self["type"].value<15):
+        if(self["type"].value<15) or (self["type"].value >= 200):
             yield UInt32(self, "unknown[]")
             yield UInt32(self, "unknown[]")
             yield UInt32(self, "position", "Position")
@@ -162,7 +165,7 @@ class TrackItem(FieldSet):
         yield Enum(UInt8(self, "x2_type", "Extended type 2"),self.x2_type_name)
         yield UInt8(self, "compilation_flag", "Compilation Flag")
         yield UInt8(self, "rating", "Rating")
-        yield TimestampMac32(self, "added_date", "Date when the item was added")
+        yield TimestampMac32(self, "last_modified", "Time of the last modification of the track")
         yield filesizeHandler(UInt32(self, "size", "Track size in bytes"))
         yield displayHandler(UInt32(self, "length", "Track length in milliseconds"), humanDuration)
         yield UInt32(self, "track_number", "Number of this track")
@@ -180,23 +183,24 @@ class TrackItem(FieldSet):
         yield UInt32(self, "disc_number", "disc number in multi disc sets")
         yield UInt32(self, "total_discs", "Total number of discs in the disc set")
         yield UInt32(self, "userid", "User ID in the DRM scheme")
-        yield TimestampMac32(self, "last_modified", "Time of the last modification of the track")
+        yield TimestampMac32(self, "added_date", "Date when the item was added")
         yield UInt32(self, "bookmark_time", "Bookmark time for AudioBook")
         yield UInt64(self, "dbid", "Unique DataBase ID for the song (identical in mhit and in mhii)")
         yield UInt8(self, "checked", "song is checked")
         yield UInt8(self, "application_rating", "Last Rating before change")
         yield UInt16(self, "BPM", "BPM of the track")
-        yield UInt16(self, "artwork_count", "number of artworks fo this item")
+        yield UInt16(self, "artwork_count", "number of artworks for this item")
         yield UInt16(self, "unknown[]")
         yield UInt32(self, "artwork_size", "Total size of artworks in bytes")
         yield UInt32(self, "unknown[]")
         yield Float32(self, "sample_rate_2", "Sample Rate express in float")
         yield UInt32(self, "released_date", "Date of release in Music Store or in Podcast")
+        yield UInt16(self, "unknown[]")
+        yield UInt16(self, "explicit_flag[]", "Explicit flag")
         yield UInt32(self, "unknown[]")
         yield UInt32(self, "unknown[]")
-        yield UInt32(self, "unknown[]")
-        yield UInt32(self, "unknown[]")
-        yield UInt32(self, "unknown[]")
+        yield UInt32(self, "skip_count[]", "Skip Count")
+        yield TimestampMac32(self, "last_skipped", "Date when the item was last skipped")
         yield UInt8(self, "has_artwork", "0x01 for track with artwork, 0x02 otherwise")
         yield UInt8(self, "skip_wen_shuffling", "Skip that track when shuffling")
         yield UInt8(self, "remember_playback_position", "Remember playback position")
@@ -207,11 +211,10 @@ class TrackItem(FieldSet):
         yield UInt8(self, "played_mark", "Track has been played")
         yield UInt8(self, "unknown[]")
         yield UInt32(self, "unknown[]")
+        yield UInt32(self, "pregap[]", "Number of samples of silence before the song starts")
+        yield UInt64(self, "sample_count", "Number of samples in the song (only for WAV and AAC files)")
         yield UInt32(self, "unknown[]")
-        yield UInt32(self, "sample_count", "Number of samples in the song (only for WAV and AAC files)")
-        yield UInt32(self, "unknown[]")
-        yield UInt32(self, "unknown[]")
-        yield UInt32(self, "unknown[]")
+        yield UInt32(self, "postgap[]", "Number of samples of silence at the end of the song")
         yield UInt32(self, "unknown[]")
         yield Enum(UInt32(self, "media_type", "Media Type for video iPod"),self.media_type_name)
         yield UInt32(self, "season_number", "Season Number")
@@ -222,6 +225,20 @@ class TrackItem(FieldSet):
         yield UInt32(self, "unknown[]")
         yield UInt32(self, "unknown[]")
         yield UInt32(self, "unknown[]")
+        yield UInt32(self, "unknown[]")
+        yield UInt32(self, "gapless_data[]","The size in bytes from first Sync Frame until the 8th before the last frame." )
+        yield UInt32(self, "unknown[]")
+        yield UInt16(self, "gaplessTrackFlag[]", "1 if track has gapless data")
+        yield UInt16(self, "gaplessAlbumFlag[]", "1 if track uses crossfading in iTunes")
+        yield RawBytes(self, "unknown[]", 20)
+        yield UInt32(self, "unknown[]")
+        yield UInt32(self, "unknown[]")
+        yield UInt32(self, "unknown[]")
+        yield UInt32(self, "unknown[]")
+        yield UInt16(self, "unknown[]")
+        yield UInt16(self, "album_id[]", "Album ID (used to link tracks with MHIAs)")
+        yield RawBytes(self, "unknown[]", 52)
+        yield UInt32(self, "mhii_link[]", "Artwork ID (used to link tracks with MHIIs)")
         padding = self.seekByte(self["header_length"].value, "header padding")
         if padding:
             yield padding
@@ -319,7 +336,7 @@ class Playlist(FieldSet):
         self._size = self["entry_length"].value *8
 
     def createFields(self):
-        yield String(self, "header_id", 4, "Playlist List Header Markup (\"mhyp\")", charset="ISO-8859-1")
+        yield String(self, "header_id", 4, "Playlist Header Markup (\"mhyp\")", charset="ISO-8859-1")
         yield UInt32(self, "header_length", "Header Length")
         yield UInt32(self, "entry_length", "Entry Length")
         yield UInt32(self, "data_object_child_count", "Number of Child Data Objects")
@@ -360,11 +377,48 @@ class PlaylistList(FieldSet):
         for i in xrange(self["playlist_number"].value):
             yield Playlist(self, "playlist[]")
 
+class Album(FieldSet):
+    def __init__(self, *args, **kw):
+        FieldSet.__init__(self, *args, **kw)
+        self._size = self["entry_length"].value *8
+
+    def createFields(self):
+        yield String(self, "header_id", 4, "Album Item Header Markup (\"mhia\")", charset="ISO-8859-1")
+        yield UInt32(self, "header_length", "Header Length")
+        yield UInt32(self, "entry_length", "Entry Length")
+        yield UInt32(self, "data_object_child_count", "Number of Child Data Objects")
+        yield UInt16(self, "unknow[]")
+        yield UInt16(self, "album_id[]", "Album ID")
+        yield UInt32(self, "unknow[]")
+        yield UInt32(self, "unknow[]")
+        yield UInt32(self, "unknow[]")
+
+        padding = self.seekByte(self["header_length"].value, "entry padding")
+        if padding:
+            yield padding
+
+        for i in xrange(self["data_object_child_count"].value):
+            yield DataObject(self, "mhod[]")
+
+class AlbumList(FieldSet):
+    def createFields(self):
+        yield String(self, "header_id", 4, "Album List Header Markup (\"mhla\")", charset="ISO-8859-1")
+        yield UInt32(self, "header_length", "Header Length")
+        yield UInt32(self, "album_number", "Number of Albums")
+
+        padding = self.seekByte(self["header_length"].value, "header padding")
+        if padding:
+            yield padding
+
+        for i in xrange(self["album_number"].value):
+            yield Album(self, "album[]")
+
 class DataSet(FieldSet):
     type_name={
         1:"Track List",
         2:"Play List",
-        3:"Podcast List"
+        3:"Podcast List",
+        4:"Album List"
         }
     def __init__(self, *args, **kw):
         FieldSet.__init__(self, *args, **kw)
@@ -384,6 +438,8 @@ class DataSet(FieldSet):
             yield PlaylistList(self, "playlist_list[]");
         if self["type"].value == 3:
             yield PlaylistList(self, "podcast_list[]");
+        if self["type"].value == 4:
+            yield AlbumList(self, "album_list[]");
         padding = self.seekBit(self._size, "entry padding")
         if padding:
             yield padding
@@ -417,8 +473,20 @@ class ITunesDBFile(Parser):
         yield UInt32(self, "version_number", "Version Number")
         yield UInt32(self, "child_number", "Number of Children")
         yield UInt64(self, "id", "ID for this database")
+        yield UInt16(self, "unknown[]")
+        yield UInt32(self, "unknown[]")
+        yield UInt64(self, "unknown[]")
+        yield UInt16(self, "unknown[]")
+        yield UInt16(self, "hashing_scheme[]", "Algorithm used to calculate the database hash")
+        yield NullBytes(self, "unknown[]", 20)
+        yield String(self, "language_id", 2, "Language ID")
+        yield UInt64(self, "persistent_id", "Library Persistent ID")
         yield UInt32(self, "unknown[]")
-        yield UInt64(self, "initial_dbid", "Initial DBID")
+        yield UInt32(self, "unknown[]")
+        yield RawBytes(self, "hash[]", 20)
+        yield Int32(self, "timezone_offset[]", "Timezone offset in seconds")
+        yield UInt16(self, "unknown[]")
+        yield RawBytes(self, "iphone_hash[]", 45)
         size = self["header_length"].value-self.current_size/ 8
         if size>0:
             yield NullBytes(self, "padding", size)

lib/hachoir_parser/audio/midi.py

@@ -8,13 +8,13 @@ Author: Victor Stinner
 Creation: 27 december 2006
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet, Bits, ParserError,
-    String, UInt32, UInt24, UInt16, UInt8, Enum, RawBytes)
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.text_handler import textHandler, hexadecimal
-from lib.hachoir_core.tools import createDict, humanDurationNanosec
-from lib.hachoir_parser.common.tracker import NOTE_NAME
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet, Bits, ParserError,
+    String, UInt32, UInt24, UInt16, UInt8, Enum, RawBits, RawBytes)
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_core.tools import createDict, humanDurationNanosec
+from hachoir_parser.common.tracker import NOTE_NAME
 
 MAX_FILESIZE = 10 * 1024 * 1024
 
@@ -46,7 +46,7 @@ def parseControl(parser):
 def parsePatch(parser):
     yield UInt8(parser, "program", "New program number")
 
-def parseChannel(parser):
+def parseChannel(parser, size=1):
     yield UInt8(parser, "channel", "Channel number")
 
 def parsePitch(parser):
@@ -56,6 +56,16 @@ def parsePitch(parser):
 def parseText(parser, size):
     yield String(parser, "text", size)
 
+def parseSMPTEOffset(parser, size):
+    yield RawBits(parser, "padding", 1)
+    yield Enum(Bits(parser, "frame_rate", 2),
+        {0:"24 fps", 1:"25 fps", 2:"30 fps (drop frame)", 3:"30 fps"})
+    yield Bits(parser, "hour", 5)
+    yield UInt8(parser, "minute")
+    yield UInt8(parser, "second")
+    yield UInt8(parser, "frame")
+    yield UInt8(parser, "subframe", "100 subframes per frame")
+
 def formatTempo(field):
     return humanDurationNanosec(field.value*1000)
 
@@ -92,8 +102,10 @@ class Command(FieldSet):
         0x05: ("Lyric", parseText),
         0x06: ("Marker", parseText),
         0x07: ("Cue point", parseText),
+        0x20: ("MIDI Channel Prefix", parseChannel),
         0x2F: ("End of the track", None),
         0x51: ("Set tempo", parseTempo),
+        0x54: ("SMPTE offset", parseSMPTEOffset),
         0x58: ("Time Signature", parseTimeSignature),
         0x59: ("Key signature", None),
         0x7F: ("Sequencer specific information", None),
@@ -101,11 +113,27 @@ class Command(FieldSet):
     META_COMMAND_DESC = createDict(META_COMMAND, 0)
     META_COMMAND_PARSER = createDict(META_COMMAND, 1)
 
+    def __init__(self, *args, **kwargs):
+        if 'prev_command' in kwargs:
+            self.prev_command = kwargs['prev_command']
+            del kwargs['prev_command']
+        else:
+            self.prev_command = None
+        self.command = None
+        FieldSet.__init__(self, *args, **kwargs)
+
     def createFields(self):
         yield Integer(self, "time", "Delta time in ticks")
-        yield Enum(textHandler(UInt8(self, "command"), hexadecimal), self.COMMAND_DESC)
-        command = self["command"].value
-        if command == 0xFF:
+        next = self.stream.readBits(self.absolute_address+self.current_size, 8, self.root.endian)
+        if next & 0x80 == 0:
+            # "Running Status" command
+            if self.prev_command is None:
+                raise ParserError("Running Status command not preceded by another command.")
+            self.command = self.prev_command.command
+        else:
+            yield Enum(textHandler(UInt8(self, "command"), hexadecimal), self.COMMAND_DESC)
+            self.command = self["command"].value
+        if self.command == 0xFF:
             yield Enum(textHandler(UInt8(self, "meta_command"), hexadecimal), self.META_COMMAND_DESC)
             yield UInt8(self, "data_len")
             size = self["data_len"].value
@@ -121,9 +149,9 @@ class Command(FieldSet):
                 else:
                     yield RawBytes(self, "data", size)
         else:
-            if command not in self.COMMAND_PARSER:
+            if self.command not in self.COMMAND_PARSER:
                 raise ParserError("Unknown command: %s" % self["command"].display)
-            parser = self.COMMAND_PARSER[command]
+            parser = self.COMMAND_PARSER[self.command]
             for field in parser(self):
                 yield field
 
@@ -131,7 +159,7 @@ class Command(FieldSet):
         if "meta_command" in self:
             return self["meta_command"].display
         else:
-            return self["command"].display
+            return self.COMMAND_DESC[self.command]
 
 class Track(FieldSet):
     def __init__(self, *args):
@@ -141,9 +169,11 @@ class Track(FieldSet):
     def createFields(self):
         yield String(self, "marker", 4, "Track marker (MTrk)", charset="ASCII")
         yield UInt32(self, "size")
+        cur = None
         if True:
             while not self.eof:
-                yield Command(self, "command[]")
+                cur = Command(self, "command[]", prev_command=cur)
+                yield cur
         else:
             size = self["size"].value
             if size:

lib/hachoir_parser/audio/mod.py

@@ -18,12 +18,12 @@ Creation: 18th February 2007
 """
 
 from math import log10
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet,
     Bits, UInt16, UInt8,
     RawBytes, String, GenericVector)
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.text_handler import textHandler
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.text_handler import textHandler
 
 # Old NoiseTracker 15-samples modules can have anything here.
 MODULE_TYPE = {

lib/hachoir_parser/audio/modplug.py

@@ -8,11 +8,11 @@ Author: Christophe GISQUET <christophe.gisquet@free.fr>
 Creation: 10th February 2007
 """
 
-from lib.hachoir_core.field import (FieldSet,
+from hachoir_core.field import (FieldSet,
     UInt32, UInt16, UInt8, Int8, Float32,
     RawBytes, String, GenericVector, ParserError)
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.text_handler import textHandler, hexadecimal
 
 MAX_ENVPOINTS = 32
 

lib/hachoir_parser/audio/mpeg_audio.py

@@ -5,18 +5,18 @@ Creation: 12 decembre 2005
 Author: Victor Stinner
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet,
     MissingField, ParserError, createOrphanField,
     Bit, Bits, Enum,
     PaddingBits, PaddingBytes,
     RawBytes)
-from lib.hachoir_parser.audio.id3 import ID3v1, ID3v2
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.tools import humanFrequency, humanBitSize
-from lib.hachoir_core.bits import long2raw
-from lib.hachoir_core.error import HACHOIR_ERRORS
-from lib.hachoir_core.stream import InputStreamError
+from hachoir_parser.audio.id3 import ID3v1, ID3v2
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.tools import humanFrequency, humanBitSize
+from hachoir_core.bits import long2raw
+from hachoir_core.error import HACHOIR_ERRORS
+from hachoir_core.stream import InputStreamError
 
 # Max MP3 filesize: 200 MB
 MAX_FILESIZE = 200*1024*1024*8

lib/hachoir_parser/audio/real_audio.py

@@ -8,14 +8,14 @@ Samples:
   http://samples.mplayerhq.hu/real/RA/
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet,
     UInt8, UInt16, UInt32,
     Bytes, RawBytes, String,
     PascalString8)
-from lib.hachoir_core.tools import humanFrequency
-from lib.hachoir_core.text_handler import displayHandler
-from lib.hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.tools import humanFrequency
+from hachoir_core.text_handler import displayHandler
+from hachoir_core.endian import BIG_ENDIAN
 
 class Metadata(FieldSet):
     def createFields(self):

lib/hachoir_parser/audio/s3m.py

@@ -9,15 +9,15 @@ Author: Christophe GISQUET <christophe.gisquet@free.fr>
 Creation: 11th February 2007
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (StaticFieldSet, FieldSet, Field,
+from hachoir_parser import Parser
+from hachoir_core.field import (StaticFieldSet, FieldSet, Field,
     Bit, Bits,
     UInt32, UInt16, UInt8, Enum,
     PaddingBytes, RawBytes, NullBytes,
     String, GenericVector, ParserError)
-from lib.hachoir_core.endian import LITTLE_ENDIAN
-from lib.hachoir_core.text_handler import textHandler, hexadecimal
-from lib.hachoir_core.tools import alignValue
+from hachoir_core.endian import LITTLE_ENDIAN
+from hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_core.tools import alignValue
 
 class Chunk:
     def __init__(self, cls, name, offset, size, *args):
@@ -326,7 +326,7 @@ class PTMHeader(Header):
     # static_size should prime over _size, right?
     static_size = 8*608
 
-    def getTrackerVersion(val):
+    def getTrackerVersion(self, val):
         val = val.value
         return "ProTracker x%04X" % val
 

lib/hachoir_parser/audio/xm.py

@@ -13,15 +13,15 @@ Author: Christophe GISQUET <christophe.gisquet@free.fr>
 Creation: 8th February 2007
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (StaticFieldSet, FieldSet,
+from hachoir_parser import Parser
+from hachoir_core.field import (StaticFieldSet, FieldSet,
     Bit, RawBits, Bits,
     UInt32, UInt16, UInt8, Int8, Enum,
     RawBytes, String, GenericVector)
-from lib.hachoir_core.endian import LITTLE_ENDIAN, BIG_ENDIAN
-from lib.hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
-from lib.hachoir_parser.audio.modplug import ParseModplugMetadata
-from lib.hachoir_parser.common.tracker import NOTE_NAME
+from hachoir_core.endian import LITTLE_ENDIAN, BIG_ENDIAN
+from hachoir_core.text_handler import textHandler, filesizeHandler, hexadecimal
+from hachoir_parser.audio.modplug import ParseModplugMetadata
+from hachoir_parser.common.tracker import NOTE_NAME
 
 def parseSigned(val):
     return "%i" % (val.value-128)

lib/hachoir_parser/common/deflate.py

@@ -1,4 +1,4 @@
-from lib.hachoir_core.field import CompressedField
+from hachoir_core.field import CompressedField
 
 try:
     from zlib import decompressobj, MAX_WBITS
@@ -12,8 +12,8 @@ try:
 
         def __call__(self, size, data=None):
             if data is None:
-                data = self.gzip.unconsumed_tail
-            return self.gzip.decompress(data, size)
+                data = ''
+            return self.gzip.decompress(self.gzip.unconsumed_tail+data, size)
 
     class DeflateStreamWbits(DeflateStream):
         def __init__(self, stream):

lib/hachoir_parser/common/msdos.py

@@ -6,8 +6,8 @@ Documentation:
   http://www.cs.colorado.edu/~main/cs1300/include/ddk/winddk.h
 """
 
-from lib.hachoir_core.field import StaticFieldSet
-from lib.hachoir_core.field import Bit, NullBits
+from hachoir_core.field import StaticFieldSet
+from hachoir_core.field import Bit, NullBits
 
 _FIELDS = (
     (Bit, "read_only"),

lib/hachoir_parser/common/win32.py

@@ -1,9 +1,9 @@
-from lib.hachoir_core.field import (FieldSet,
+from hachoir_core.field import (FieldSet,
     UInt16, UInt32, Enum, String, Bytes, Bits, TimestampUUID60)
-from lib.hachoir_parser.video.fourcc import video_fourcc_name
-from lib.hachoir_core.bits import str2hex
-from lib.hachoir_core.text_handler import textHandler, hexadecimal
-from lib.hachoir_parser.network.common import MAC48_Address
+from hachoir_parser.video.fourcc import video_fourcc_name
+from hachoir_core.bits import str2hex
+from hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_parser.network.common import MAC48_Address
 
 # Dictionary: Windows codepage => Python charset name
 CODEPAGE_CHARSET = {
@@ -24,6 +24,26 @@ CODEPAGE_CHARSET = {
     65001: "UTF-8",
 }
 
+class PascalStringWin16(FieldSet):
+    def __init__(self, parent, name, description=None, strip=None, charset="UTF-16-LE"):
+        FieldSet.__init__(self, parent, name, description)
+        length = self["length"].value
+        self._size = 16 + length * 16
+        self.strip = strip
+        self.charset = charset
+
+    def createFields(self):
+        yield UInt16(self, "length", "Length in widechar characters")
+        size = self["length"].value
+        if size:
+            yield String(self, "text", size*2, charset=self.charset, strip=self.strip)
+
+    def createValue(self):
+        if "text" in self:
+            return self["text"].value
+        else:
+            return None
+
 class PascalStringWin32(FieldSet):
     def __init__(self, parent, name, description=None, strip=None, charset="UTF-16-LE"):
         FieldSet.__init__(self, parent, name, description)

lib/hachoir_parser/container/__init__.py

@@ -1,7 +1,7 @@
-from lib.hachoir_parser.container.asn1 import ASN1File
-from lib.hachoir_parser.container.mkv import MkvFile
-from lib.hachoir_parser.container.ogg import OggFile, OggStream
-from lib.hachoir_parser.container.riff import RiffFile
-from lib.hachoir_parser.container.swf import SwfFile
-from lib.hachoir_parser.container.realmedia import RealMediaFile
+from hachoir_parser.container.asn1 import ASN1File
+from hachoir_parser.container.mkv import MkvFile
+from hachoir_parser.container.ogg import OggFile, OggStream
+from hachoir_parser.container.riff import RiffFile
+from hachoir_parser.container.swf import SwfFile
+from hachoir_parser.container.realmedia import RealMediaFile
 

lib/hachoir_parser/container/action_script.py

@@ -5,29 +5,64 @@ Documentation:
 
  - Alexis' SWF Reference:
    http://www.m2osw.com/swf_alexref.html
+ - Tamarin ABC format:
+   http://www.m2osw.com/abc_format.html
 
-Author: Sebastien Ponce
+Authors: Sebastien Ponce, Robert Xiao
 Creation date: 26 April 2008
 """
 
-from lib.hachoir_core.field import (FieldSet, ParserError,
-    Bit, Bits, UInt8, UInt32, Int16, UInt16, Float32, CString,
-    RawBytes)
-#from lib.hachoir_core.field import Field
-from lib.hachoir_core.field.float import FloatExponent
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet, ParserError,
+    Bit, Bits, UInt8, UInt32, Int16, UInt16, Float32, Float64, CString, Enum,
+    Bytes, RawBytes, NullBits, String, SubFile, Field)
+from hachoir_core.endian import LITTLE_ENDIAN, BIG_ENDIAN
+from hachoir_core.field.float import FloatExponent
 from struct import unpack
 
+class FlashPackedInteger(Bits):
+    def __init__(self, parent, name, signed=False, nbits=30, description=None):
+        Bits.__init__(self, parent, name, 8, description)
+        stream = self._parent.stream
+        addr = self.absolute_address
+        size = 0
+        value = 0
+        mult = 1
+        while True:
+            byte = stream.readBits(addr+size, 8, LITTLE_ENDIAN)
+            value += mult * (byte & 0x7f)
+            size += 8
+            mult <<= 7
+            if byte < 128:
+                break
+        self._size = size
+        if signed and (1 << (nbits-1)) <= value:
+            value -= (1 << nbits)
+        self.createValue = lambda: value
+
+class FlashU30(FlashPackedInteger):
+    def __init__(self, parent, name, description=None):
+        FlashPackedInteger.__init__(self, parent, name, signed=False, nbits=30, description=description)
+
+class FlashS32(FlashPackedInteger):
+    def __init__(self, parent, name, description=None):
+        FlashPackedInteger.__init__(self, parent, name, signed=True, nbits=32, description=description)
+
+class FlashU32(FlashPackedInteger):
+    def __init__(self, parent, name, description=None):
+        FlashPackedInteger.__init__(self, parent, name, signed=False, nbits=32, description=description)
+
 class FlashFloat64(FieldSet):
     def createFields(self):
-        yield Bits(self, "mantisa_high", 20)
+        yield Bits(self, "mantissa_high", 20)
         yield FloatExponent(self, "exponent", 11)
         yield Bit(self, "negative")
-        yield Bits(self, "mantisa_low", 32)
+        yield Bits(self, "mantissa_low", 32)
 
     def createValue(self):
         # Manual computation:
-        # mantisa = mantisa_high * 2^32 + mantisa_low
-        # float = 2^exponent + (1 + mantisa / 2^52)
+        # mantissa = mantissa_high * 2^32 + mantissa_low
+        # float = 2^exponent + (1 + mantissa / 2^52)
         # (and float is negative if negative=True)
         bytes = self.parent.stream.readBytes(
             self.absolute_address, self.size//8)
@@ -44,8 +79,8 @@ TYPE_INFO = {
     0x05: (UInt8, "Boolean[]"),
     0x06: (FlashFloat64, "Double[]"),
     0x07: (UInt32, "Integer[]"),
-    0x08: (UInt8, "Dictionnary_Lookup_Index[]"),
-    0x09: (UInt16, "Large_Dictionnary_Lookup_Index[]"),
+    0x08: (UInt8, "Dictionary_Lookup_Index[]"),
+    0x09: (UInt16, "Large_Dictionary_Lookup_Index[]"),
 }
 
 def parseBranch(parent, size):
@@ -135,7 +170,7 @@ def parseWaitForFrame(parent, size):
 def parseWaitForFrameDyn(parent, size):
     yield UInt8(parent, "skip")
 
-def parseDeclareDictionnary(parent, size):
+def parseDeclareDictionary(parent, size):
     count = UInt16(parent, "count")
     yield count
     for i in range(count.value):
@@ -231,7 +266,7 @@ class Instruction(FieldSet):
         # Objects
         0x2B: ("Cast_Object[]", "Cast Object", None),
         0x42: ("Declare_Array[]", "Declare Array", None),
-        0x88: ("Declare_Dictionary[]", "Declare Dictionary", parseDeclareDictionnary),
+        0x88: ("Declare_Dictionary[]", "Declare Dictionary", parseDeclareDictionary),
         0x43: ("Declare_Object[]", "Declare Object", None),
         0x3A: ("Delete[]", "Delete", None),
         0x3B: ("Delete_All[]", "Delete All", None),
@@ -314,3 +349,313 @@ class ActionScript(FieldSet):
 def parseActionScript(parent, size):
     yield ActionScript(parent, "action", size=size*8)
 
+def FindABC(field):
+    while not getattr(field, "isABC", False):
+        field = field.parent
+        if field is None:
+            return None
+    return field
+
+def GetConstant(field, pool, index):
+    if index == 0:
+        return None
+    return FindABC(field)["constant_%s_pool/constant[%i]"%(pool, index)]
+
+def GetMultiname(field, index):
+    fld = GetConstant(field, "multiname", index)
+    if fld is None:
+        return "*"
+    if "name_index" not in fld:
+        return "?"
+    fld2 = GetConstant(fld, "string", fld["name_index"].value)
+    if fld2 is None:
+        return "*"
+    return fld2.value
+
+class ABCStringIndex(FlashU30):
+    def createDisplay(self):
+        fld = GetConstant(self, "string", self.value)
+        if fld is None:
+            return "*"
+        return fld.value
+
+class ABCNSIndex(FlashU30):
+    def createDisplay(self):
+        fld = GetConstant(self, "namespace", self.value)
+        if fld is None:
+            return "*"
+        return fld.display
+
+class ABCMethodIndex(FlashU30):
+    def createDisplay(self):
+        fld = FindABC(self)["method_array/method[%i]"%self.value]
+        if fld is None:
+            return "*"
+        return fld.description
+
+class ABCMultinameIndex(FlashU30):
+    def createDisplay(self):
+        return GetMultiname(self, self.value)
+
+class ABCConstantPool(FieldSet):
+    def __init__(self, parent, name, klass):
+        FieldSet.__init__(self, parent, 'constant_%s_pool'%name)
+        self.klass = klass
+    def createFields(self):
+        ctr = FlashU30(self, "count")
+        yield ctr
+        for i in xrange(ctr.value-1):
+            yield self.klass(self, "constant[%i]"%(i+1))
+
+class ABCObjectArray(FieldSet):
+    def __init__(self, parent, name, klass):
+        self.arrname = name
+        FieldSet.__init__(self, parent, name+'_array')
+        self.klass = klass
+    def createFields(self):
+        ctr = FlashU30(self, "count")
+        yield ctr
+        for i in xrange(ctr.value):
+            yield self.klass(self, self.arrname+"[]")
+
+class ABCClassArray(FieldSet):
+    def __init__(self, parent, name):
+        FieldSet.__init__(self, parent, name+'_array')
+    def createFields(self):
+        ctr = FlashU30(self, "count")
+        yield ctr
+        for i in xrange(ctr.value):
+            yield ABCInstanceInfo(self, "instance[]")
+        for i in xrange(ctr.value):
+            yield ABCClassInfo(self, "class[]")
+
+class ABCConstantString(FieldSet):
+    def createFields(self):
+        yield FlashU30(self, "length")
+        size = self["length"].value
+        if size:
+            yield String(self, "data", size, charset="UTF-8")
+
+    def createDisplay(self):
+        if "data" in self:
+            return self["data"].display
+        else:
+            return "<empty>"
+
+    def createValue(self):
+        if "data" in self:
+            return self["data"].value
+        else:
+            return ""
+
+class ABCConstantNamespace(FieldSet):
+    NAMESPACE_KIND = {8: "Namespace",
+                      5: "PrivateNamespace",
+                      22: "PackageNamespace",
+                      23: "PacakgeInternalNamespace",
+                      24: "ProtectedNamespace",
+                      25: "ExplicitNamespace",
+                      26: "MultinameL"}
+    def createFields(self):
+        yield Enum(UInt8(self, "kind"), self.NAMESPACE_KIND)
+        yield ABCStringIndex(self, "name_index")
+
+    def createDisplay(self):
+        return "%s %s"%(self["kind"].display, self["name_index"].display)
+
+    def createValue(self):
+        return self["name_index"].value
+
+class ABCConstantNamespaceSet(FieldSet):
+    def createFields(self):
+        ctr = FlashU30(self, "namespace_count")
+        yield ctr
+        for i in xrange(ctr.value):
+            yield ABCNSIndex(self, "namespace_index[]")
+
+    def createDescription(self):
+        ret = [fld.display for fld in self.array("namespace_index")]
+        return ', '.join(ret)
+
+class ABCConstantMultiname(FieldSet):
+    MULTINAME_KIND = {7: "Qname",
+                      13: "QnameA",
+                      9: "Multiname",
+                      14: "MultinameA",
+                      15: "RTQname",
+                      16: "RTQnameA",
+                      27: "MultinameL",
+                      17: "RTQnameL",
+                      18: "RTQnameLA"}
+    def createFields(self):
+        yield Enum(UInt8(self, "kind"), self.MULTINAME_KIND)
+        kind = self["kind"].value
+        if kind in (7,13): # Qname
+            yield FlashU30(self, "namespace_index")
+            yield ABCStringIndex(self, "name_index")
+        elif kind in (9,14): # Multiname
+            yield ABCStringIndex(self, "name_index")
+            yield FlashU30(self, "namespace_set_index")
+        elif kind in (15,16): # RTQname
+            yield ABCStringIndex(self, "name_index")
+        elif kind == 27: # MultinameL
+            yield FlashU30(self, "namespace_set_index")
+        elif kind in (17,18): # RTQnameL
+            pass
+
+    def createDisplay(self):
+        kind = self["kind"].display
+        if "name_index" in self:
+            return kind + " " + self["name_index"].display
+        return kind
+
+    def createValue(self):
+        return self["kind"].value
+
+class ABCTrait(FieldSet):
+    TRAIT_KIND = {0: "slot",
+                  1: "method",
+                  2: "getter",
+                  3: "setter",
+                  4: "class",
+                  5: "function",
+                  6: "const",}
+    def createFields(self):
+        yield ABCMultinameIndex(self, "name_index")
+        yield Enum(Bits(self, "kind", 4), self.TRAIT_KIND)
+        yield Enum(Bit(self, "is_final"), {True:'final',False:'virtual'})
+        yield Enum(Bit(self, "is_override"), {True:'override',False:'new'})
+        yield Bit(self, "has_metadata")
+        yield Bits(self, "unused", 1)
+        kind = self["kind"].value
+        if kind in (0,6): # slot, const
+            yield FlashU30(self, "slot_id")
+            yield ABCMultinameIndex(self, "type_index")
+            ### TODO reference appropriate constant pool using value_kind
+            yield FlashU30(self, "value_index")
+            if self['value_index'].value != 0:
+                yield UInt8(self, "value_kind")
+        elif kind in (1,2,3): # method, getter, setter
+            yield FlashU30(self, "disp_id")
+            yield ABCMethodIndex(self, "method_info")
+        elif kind == 4: # class
+            yield FlashU30(self, "disp_id")
+            yield FlashU30(self, "class_info")
+        elif kind == 5: # function
+            yield FlashU30(self, "disp_id")
+            yield ABCMethodIndex(self, "method_info")
+        if self['has_metadata'].value:
+            yield ABCObjectArray(self, "metadata", FlashU30)
+
+class ABCValueKind(FieldSet):
+    def createFields(self):
+        yield FlashU30(self, "value_index")
+        yield UInt8(self, "value_kind")
+
+class ABCMethodInfo(FieldSet):
+    def createFields(self):
+        yield FlashU30(self, "param_count")
+        yield ABCMultinameIndex(self, "ret_type")
+        for i in xrange(self["param_count"].value):
+            yield ABCMultinameIndex(self, "param_type[]")
+        yield ABCStringIndex(self, "name_index")
+        yield Bit(self, "need_arguments")
+        yield Bit(self, "need_activation")
+        yield Bit(self, "need_rest")
+        yield Bit(self, "has_optional")
+        yield Bit(self, "ignore_rest")
+        yield Bit(self, "explicit")
+        yield Bit(self, "setsdxns")
+        yield Bit(self, "has_paramnames")
+        if self["has_optional"].value:
+            yield ABCObjectArray(self, "optional", ABCValueKind)
+        if self["has_paramnames"].value:
+            for i in xrange(self["param_count"].value):
+                yield FlashU30(self, "param_name[]")
+
+    def createDescription(self):
+        ret = GetMultiname(self, self["ret_type"].value)
+        ret += " " + self["name_index"].display
+        ret += "(" + ", ".join(GetMultiname(self, fld.value) for fld in self.array("param_type")) + ")"
+        return ret
+
+class ABCMetadataInfo(FieldSet):
+    def createFields(self):
+        yield ABCStringIndex(self, "name_index")
+        yield FlashU30(self, "values_count")
+        count = self["values_count"].value
+        for i in xrange(count):
+            yield FlashU30(self, "key[]")
+        for i in xrange(count):
+            yield FlashU30(self, "value[]")
+
+class ABCInstanceInfo(FieldSet):
+    def createFields(self):
+        yield ABCMultinameIndex(self, "name_index")
+        yield ABCMultinameIndex(self, "super_index")
+        yield Bit(self, "is_sealed")
+        yield Bit(self, "is_final")
+        yield Bit(self, "is_interface")
+        yield Bit(self, "is_protected")
+        yield Bits(self, "unused", 4)
+        if self['is_protected'].value:
+            yield ABCNSIndex(self, "protectedNS")
+        yield FlashU30(self, "interfaces_count")
+        for i in xrange(self["interfaces_count"].value):
+            yield ABCMultinameIndex(self, "interface[]")
+        yield ABCMethodIndex(self, "iinit_index")
+        yield ABCObjectArray(self, "trait", ABCTrait)
+
+class ABCClassInfo(FieldSet):
+    def createFields(self):
+        yield ABCMethodIndex(self, "cinit_index")
+        yield ABCObjectArray(self, "trait", ABCTrait)
+
+class ABCScriptInfo(FieldSet):
+    def createFields(self):
+        yield ABCMethodIndex(self, "init_index")
+        yield ABCObjectArray(self, "trait", ABCTrait)
+
+class ABCException(FieldSet):
+    def createFields(self):
+        yield FlashU30(self, "start")
+        yield FlashU30(self, "end")
+        yield FlashU30(self, "target")
+        yield FlashU30(self, "type_index")
+        yield FlashU30(self, "name_index")
+
+class ABCMethodBody(FieldSet):
+    def createFields(self):
+        yield ABCMethodIndex(self, "method_info")
+        yield FlashU30(self, "max_stack")
+        yield FlashU30(self, "max_regs")
+        yield FlashU30(self, "scope_depth")
+        yield FlashU30(self, "max_scope")
+        yield FlashU30(self, "code_length")
+        yield RawBytes(self, "code", self['code_length'].value)
+        yield ABCObjectArray(self, "exception", ABCException)
+        yield ABCObjectArray(self, "trait", ABCTrait)
+
+def parseABC(parent, size):
+    code = parent["code"].value
+    if code == parent.TAG_DO_ABC_DEFINE:
+        yield UInt32(parent, "action_flags")
+        yield CString(parent, "action_name")
+    yield UInt16(parent, "minor_version")
+    yield UInt16(parent, "major_version")
+    parent.isABC = True
+
+    yield ABCConstantPool(parent, "int", FlashS32)
+    yield ABCConstantPool(parent, "uint", FlashU32)
+    yield ABCConstantPool(parent, "double", Float64)
+    yield ABCConstantPool(parent, "string", ABCConstantString)
+    yield ABCConstantPool(parent, "namespace", ABCConstantNamespace)
+    yield ABCConstantPool(parent, "namespace_set", ABCConstantNamespaceSet)
+    yield ABCConstantPool(parent, "multiname", ABCConstantMultiname)
+
+    yield ABCObjectArray(parent, "method", ABCMethodInfo)
+    yield ABCObjectArray(parent, "metadata", ABCMetadataInfo)
+    yield ABCClassArray(parent, "class")
+    yield ABCObjectArray(parent, "script", ABCScriptInfo)
+    yield ABCObjectArray(parent, "body", ABCMethodBody)
+

lib/hachoir_parser/container/asn1.py

@@ -39,15 +39,15 @@ Author: Victor Stinner
 Creation date: 24 september 2006
 """
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet,
     FieldError, ParserError,
     Bit, Bits, Bytes, UInt8, GenericInteger, String,
     Field, Enum, RawBytes)
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.tools import createDict, humanDatetime
-from lib.hachoir_core.stream import InputStreamError
-from lib.hachoir_core.text_handler import textHandler
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.tools import createDict, humanDatetime
+from hachoir_core.stream import InputStreamError
+from hachoir_core.text_handler import textHandler
 
 # --- Field parser ---
 

lib/hachoir_parser/container/mkv.py

@@ -4,18 +4,18 @@
 # Created: 8 june 2006
 #
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (FieldSet, Link,
+from hachoir_parser import Parser
+from hachoir_core.field import (FieldSet, Link,
     MissingField, ParserError,
     Enum as _Enum, String as _String,
     Float32, Float64,
     NullBits, Bits, Bit, RawBytes, Bytes,
     Int16, GenericInteger)
-from lib.hachoir_core.endian import BIG_ENDIAN
-from lib.hachoir_core.iso639 import ISO639_2
-from lib.hachoir_core.tools import humanDatetime
-from lib.hachoir_core.text_handler import textHandler, hexadecimal
-from lib.hachoir_parser.container.ogg import XiphInt
+from hachoir_core.endian import BIG_ENDIAN
+from hachoir_core.iso639 import ISO639_2
+from hachoir_core.tools import humanDatetime
+from hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_parser.container.ogg import XiphInt
 from datetime import datetime, timedelta
 
 class RawInt(GenericInteger):
@@ -66,7 +66,7 @@ def SInt(parent):
     return GenericInteger(parent, 'signed', True, parent['size'].value*8)
 
 def String(parent):
-    return _String(parent, 'string', parent['size'].value, charset="ASCII")
+    return _String(parent, 'string', parent['size'].value, charset="ASCII", strip="\0")
 
 def EnumString(parent, enum):
     return _Enum(String(parent), enum)
@@ -206,7 +206,7 @@ class Block(FieldSet):
             yield Bit(self, 'invisible')
             yield self.lacing()
             yield NullBits(self, 'reserved[]', 1)
-        elif self.parent._name == 'SimpleBlock[]':
+        elif self.parent._name.startswith('SimpleBlock'):
             yield Bit(self, 'keyframe')
             yield NullBits(self, 'reserved', 3)
             yield Bit(self, 'invisible')

lib/hachoir_parser/container/ogg.py

@@ -4,15 +4,15 @@
 # Created: 10 june 2006
 #
 
-from lib.hachoir_parser import Parser
-from lib.hachoir_core.field import (Field, FieldSet, createOrphanField,
+from hachoir_parser import Parser
+from hachoir_core.field import (Field, FieldSet, createOrphanField,
     NullBits, Bit, Bits, Enum, Fragment, MissingField, ParserError,
     UInt8, UInt16, UInt24, UInt32, UInt64,
     RawBytes, String, PascalString32, NullBytes)
-from lib.hachoir_core.stream import FragmentedStream, InputStreamError
-from lib.hachoir_core.endian import LITTLE_ENDIAN, BIG_ENDIAN
-from lib.hachoir_core.tools import humanDurationNanosec
-from lib.hachoir_core.text_handler import textHandler, hexadecimal
+from hachoir_core.stream import FragmentedStream, InputStreamError
+from hachoir_core.endian import LITTLE_ENDIAN, BIG_ENDIAN
+from hachoir_core.tools import humanDurationNanosec
+from hachoir_core.text_handler import textHandler, hexadecimal
 
 MAX_FILESIZE = 1000 * 1024 * 1024