Oftentimes I find myself converting pandas.DataFrame objects to lists of formatted row strings, so I can print the rows into, e.g. a tkinter.Listbox. To do this, I have been utilizing pandas.DataFrame.to_string. There is a lot of nice functionality built into the method, but when the number of dataframe rows/columns gets relatively large, to_string starts to tank.
Below I implement a custom pandas.DataFrame class with a few added methods for returning formatted row lines. I am looking to improve upon the get_lines_fast_struct method.
import pandas
np = pandas.np
class DataFrame2(pandas.DataFrame):
def __init__( self, *args, **kwargs ):
pandas.DataFrame.__init__(self, *args, **kwargs)
def get_lines_standard(self):
"""standard way to convert pandas dataframe
to lines with fomrmatted column spacing"""
lines = self.to_string(index=False).split('\n')
return lines
def get_lines_fast_unstruct(self):
""" lighter version of pandas.DataFrame.to_string()
with no special spacing format"""
df_recs = self.to_records(index=False)
col_titles = [' '.join(list(self))]
col_data = map(lambda rec:' '.join( map(str,rec) ),
df_recs.tolist())
lines = col_titles + col_data
return lines
def get_lines_fast_struct(self,col_space=1):
""" lighter version of pandas.DataFrame.to_string()
with special spacing format"""
df_recs = self.to_records(index=False) # convert dataframe to array of records
str_data = map(lambda rec: map(str,rec), df_recs ) # map each element to string
self.space = map(lambda x:len(max(x,key=len))+col_space, # returns the max string length in each column as a list
zip(*str_data))
col_titles = [self._format_line(list(self))]
col_data = [self._format_line(row ) for row in str_data ]
lines = col_titles + col_data
return lines
def _format_line(self, row_vals):
"""row_vals: list of strings.
Adds variable amount of white space to each
list entry and returns a single string"""
line_val_gen = ( ('{0: >%d}'%self.space[i]).format(entry) for i,entry in enumerate(row_vals) ) # takes dataframe row entries and adds white spaces based on a format
line = ''.join(line_val_gen)
return line
#SOME TEST DATA
df = DataFrame2({'A':np.random.randint(0,1000,1000),
'B':np.random.random(1000),
'C':[random.choice(['EYE', '<3', 'PANDAS', '0.16'])
for _ in range(1000)]})
Method outputs
df.get_lines_standard()[:5] # first five rows in dataframe #[u' A B C', # u' 504 0.924385 <3', # u' 388 0.285854 0.16', # u' 984 0.254156 0.16', # u' 446 0.472621 PANDAS'] df.get_lines_fast_struct()[:5] #[' A B C', # ' 504 0.9243853594 <3', # ' 388 0.285854082778 0.16', # ' 984 0.254155910401 0.16', # ' 446 0.472621088021 PANDAS'] df.get_lines_fast_unstruct()[:5] #['A B C', # '504 0.9243853594 <3', # '388 0.285854082778 0.16', # '984 0.254155910401 0.16', # '446 0.472621088021 PANDAS']
Timing results
In [262]: %timeit df.get_lines_standard() 10 loops, best of 3: 70.3 ms per loop In [263]: %timeit df.get_lines_fast_struct() 100 loops, best of 3: 15.4 ms per loop In [264]: %timeit df.get_lines_fast_unstruct() 100 loops, best of 3: 2.3 ms per loop
