Merge branch 'long-str-unconvert-fix' of https://github.com/wabu/pandas into wabu-long-str-unconvert-fix

Conflicts:
	doc/source/release.rst

Author: jreback

doc/source/release.rst

@@ -90,6 +90,7 @@ Bug Fixes
 - ``HDFStore.remove`` now handles start and stop (:issue:`6177`)
 - ``HDFStore.select_as_multiple`` handles start and stop the same way as ``select`` (:issue:`6177`)
 - ``HDFStore.select_as_coordinates`` and ``select_column`` works where clauses that result in filters (:issue:`6177`)
+- Bug with numpy < 1.7.2 when reading long strings from ``HDFStore`` (:issue:`6166`)
 
 pandas 0.13.1
 -------------

pandas/io/pytables.py

@@ -40,13 +40,15 @@
 import pandas.tslib as tslib
 
 from contextlib import contextmanager
+from distutils.version import LooseVersion
 
 # versioning attribute
 _version = '0.10.1'
 
 # PY3 encoding if we don't specify
 _default_encoding = 'UTF-8'
 
+_np_version_under_172 = LooseVersion(np.__version__) < '1.7.2'
 
 def _ensure_decoded(s):
     """ if we have bytes, decode them to unicde """
@@ -776,8 +778,8 @@ def func(_start, _stop):
                 c = s.read_coordinates(where=where, start=_start, stop=_stop, **kwargs)
             else:
                 c = None
-                
-            objs = [t.read(where=c, start=_start, stop=_stop, 
+
+            objs = [t.read(where=c, start=_start, stop=_stop,
                            columns=columns, **kwargs) for t in tbls]
 
             # concat and return
@@ -4165,7 +4167,6 @@ def _convert_string_array(data, encoding, itemsize=None):
     data = np.array(data, dtype="S%d" % itemsize)
     return data
 
-
 def _unconvert_string_array(data, nan_rep=None, encoding=None):
     """ deserialize a string array, possibly decoding """
     shape = data.shape
@@ -4175,8 +4176,14 @@ def _unconvert_string_array(data, nan_rep=None, encoding=None):
     # where the passed encoding is actually None)
     encoding = _ensure_encoding(encoding)
     if encoding is not None and len(data):
+        if _np_version_under_172:
+            itemsize = lib.max_len_string_array(data)
+            dtype = "S{0}".format(itemsize)
+        else:
+            dtype = string_types[0]
+
         try:
-            data = data.astype(string_types).astype(object)
+            data = data.astype(dtype).astype(object)
         except:
             f = np.vectorize(lambda x: x.decode(encoding), otypes=[np.object])
             data = f(data)

pandas/io/tests/test_pytables.py

@@ -176,6 +176,24 @@ def roundtrip(key, obj,**kwargs):
         finally:
             safe_remove(self.path)
 
+    def test_long_strings(self):
+        df = DataFrame({'a': [tm.rands(100) for _ in range(10)]},
+                index=[tm.rands(100) for _ in range(10)])
+
+        with ensure_clean_store(self.path) as store:
+            store.append('df', df, data_columns=['a'])
+            assert_frame_equal(store['df'], df)
+
+        # test with an encoding
+        if LooseVersion(tables.__version__) < '3.0.0':
+            raise nose.SkipTest('tables version does not support proper encoding')
+        if sys.byteorder != 'little':
+            raise nose.SkipTest('system byteorder is not little')
+
+        with ensure_clean_store(self.path) as store:
+            store.append('df', df, data_columns=['a'], encoding='ascii')
+            assert_frame_equal(store['df'], df)
+
     def test_api(self):
 
         # GH4584
@@ -2199,7 +2217,7 @@ def test_remove_startstop(self):
         # GH #4835 and #6177
 
         with ensure_clean_store(self.path) as store:
-            
+
             wp = tm.makePanel()
 
             # start
@@ -2246,7 +2264,7 @@ def test_remove_startstop(self):
             result = store.select('wp6')
             expected = wp.reindex(major_axis=wp.major_axis)
             assert_panel_equal(result, expected)
-            
+
             # with where
             date = wp.major_axis.take(np.arange(0,30,3))
             crit = Term('major_axis=date')
@@ -2256,7 +2274,7 @@ def test_remove_startstop(self):
             result = store.select('wp7')
             expected = wp.reindex(major_axis=wp.major_axis-wp.major_axis[np.arange(0,20,3)])
             assert_panel_equal(result, expected)
-            
+
 
     def test_remove_crit(self):