Data Preparation in Pandas

Data Preparation in Pandas

Data cleaning

import pandas as pd

import numpy as np

string_data=pd.Series(['aardvark','artichoke',np.nan,'avocado']);string_data

0     aardvark
1    artichoke
2          NaN
3      avocado
dtype: object

string_data.isnull()

0    False
1    False
2     True
3    False
dtype: bool

string_data[2]

nan

from numpy import nan as NA

data=pd.Series([1,NA,3.5,NA,7])

data.dropna()

0    1.0
2    3.5
4    7.0
dtype: float64

data[[False,True,True,False,False]]

1    NaN
2    3.5
dtype: float64

data[data.notnull()]

0    1.0
2    3.5
4    7.0
dtype: float64

data=pd.DataFrame([[1,6.5,3],[1,NA,NA],[NA,NA,NA],[NA,6.5,3]]);data
0 1 2
0 1.0 6.5 3.0
1 1.0 NaN NaN
2 NaN NaN NaN
3 NaN 6.5 3.0 
data.dropna()
0 1 2
0 1.0 6.5 3.0 
data.dropna(how='all')
0 1 2
0 1.0 6.5 3.0
1 1.0 NaN NaN
3 NaN 6.5 3.0 
data[4]=NA;data
0 1 2 4
0 1.0 6.5 3.0 NaN
1 1.0 NaN NaN NaN
2 NaN NaN NaN NaN
3 NaN 6.5 3.0 NaN 
data.dropna(how='all',axis='columns')
0 1 2
0 1.0 6.5 3.0
1 1.0 NaN NaN
2 NaN NaN NaN
3 NaN 6.5 3.0 
df=pd.DataFrame(np.random.randn(7,3))

df
0 1 2
0 -1.744196 -0.281787 -0.963212
1 -1.114174 0.024707 0.095524
2 0.879205 -1.272202 -0.317218
3 0.227725 -0.067809 0.609824
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
help(np.random.randn)

Help on built-in function randn:

randn(...) method of mtrand.RandomState instance
    randn(d0, d1, ..., dn)
    
    Return a sample (or samples) from the "standard normal" distribution.
    
    If positive, int_like or int-convertible arguments are provided,
    `randn` generates an array of shape ``(d0, d1, ..., dn)``, filled
    with random floats sampled from a univariate "normal" (Gaussian)
    distribution of mean 0 and variance 1 (if any of the :math:`d_i` are
    floats, they are first converted to integers by truncation). A single
    float randomly sampled from the distribution is returned if no
    argument is provided.
    
    This is a convenience function.  If you want an interface that takes a
    tuple as the first argument, use `numpy.random.standard_normal` instead.
    
    Parameters
    ----------
    d0, d1, ..., dn : int, optional
        The dimensions of the returned array, should be all positive.
        If no argument is given a single Python float is returned.
    
    Returns
    -------
    Z : ndarray or float
        A ``(d0, d1, ..., dn)``-shaped array of floating-point samples from
        the standard normal distribution, or a single such float if
        no parameters were supplied.
    
    See Also
    --------
    random.standard_normal : Similar, but takes a tuple as its argument.
    
    Notes
    -----
    For random samples from :math:`N(\mu, \sigma^2)`, use:
    
    ``sigma * np.random.randn(...) + mu``
    
    Examples
    --------
    >>> np.random.randn()
    2.1923875335537315 #random
    
    Two-by-four array of samples from N(3, 6.25):
    
    >>> 2.5 * np.random.randn(2, 4) + 3
    array([[-4.49401501,  4.00950034, -1.81814867,  7.29718677],  #random
           [ 0.39924804,  4.68456316,  4.99394529,  4.84057254]]) #random

df
0 1 2
0 -1.744196 -0.281787 -0.963212
1 -1.114174 0.024707 0.095524
2 0.879205 -1.272202 -0.317218
3 0.227725 -0.067809 0.609824
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
df.iloc[:4,1]=NA;df
0 1 2
0 -1.744196 NaN -0.963212
1 -1.114174 NaN 0.095524
2 0.879205 NaN -0.317218
3 0.227725 NaN 0.609824
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
df.iloc[:2,2]=NA;df
0 1 2
0 -1.744196 NaN NaN
1 -1.114174 NaN NaN
2 0.879205 NaN -0.317218
3 0.227725 NaN 0.609824
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
df.dropna()
0 1 2
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
df.dropna(thresh=2)
0 1 2
2 0.879205 NaN -0.317218
3 0.227725 NaN 0.609824
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
df.fillna(0)
0 1 2
0 -1.744196 0.000000 0.000000
1 -1.114174 0.000000 0.000000
2 0.879205 0.000000 -0.317218
3 0.227725 0.000000 0.609824
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
df.fillna({1:0.5,2:0})
0 1 2
0 -1.744196 0.500000 0.000000
1 -1.114174 0.500000 0.000000
2 0.879205 0.500000 -0.317218
3 0.227725 0.500000 0.609824
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
df
0 1 2
0 -1.744196 NaN NaN
1 -1.114174 NaN NaN
2 0.879205 NaN -0.317218
3 0.227725 NaN 0.609824
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
df.fillna(0,inplace=True)

df
0 1 2
0 -1.744196 0.000000 0.000000
1 -1.114174 0.000000 0.000000
2 0.879205 0.000000 -0.317218
3 0.227725 0.000000 0.609824
4 -1.082470 -1.230476 -1.616135
5 -1.218976 0.018245 -0.155761
6 -0.607157 -0.641986 -0.406378 
df=pd.DataFrame(np.random.randn(6,3))

df.iloc[2:,1]=NA

df.iloc[4:,2]=NA

df
0 1 2
0 -0.970921 -1.311345 0.779965
1 -0.352837 0.290834 -0.440396
2 0.574406 NaN 2.034865
3 0.088611 NaN -0.004141
4 0.792289 NaN NaN
5 0.668345 NaN NaN 
df.fillna(method='ffill')
0 1 2
0 -0.970921 -1.311345 0.779965
1 -0.352837 0.290834 -0.440396
2 0.574406 0.290834 2.034865
3 0.088611 0.290834 -0.004141
4 0.792289 0.290834 -0.004141
5 0.668345 0.290834 -0.004141 
df
0 1 2
0 -0.970921 -1.311345 0.779965
1 -0.352837 0.290834 -0.440396
2 0.574406 NaN 2.034865
3 0.088611 NaN -0.004141
4 0.792289 NaN NaN
5 0.668345 NaN NaN 
df.dropna()
0 1 2
0 -0.970921 -1.311345 0.779965
1 -0.352837 0.290834 -0.440396 
df.dropna(thresh=2)
0 1 2
0 -0.970921 -1.311345 0.779965
1 -0.352837 0.290834 -0.440396
2 0.574406 NaN 2.034865
3 0.088611 NaN -0.004141 
df.dropna(thresh=2,inplace=True)

df
0 1 2
0 -0.970921 -1.311345 0.779965
1 -0.352837 0.290834 -0.440396
2 0.574406 NaN 2.034865
3 0.088611 NaN -0.004141 
data=pd.DataFrame({'K1':['one','two']*3+['two'],'K2':[1,1,2,3,3,4,4]});data
K1 K2
0 one 1
1 two 1
2 one 2
3 two 3
4 one 3
5 two 4
6 two 4 
data.duplicated()

0    False
1    False
2    False
3    False
4    False
5    False
6     True
dtype: bool

data.drop_duplicates()
K1 K2
0 one 1
1 two 1
2 one 2
3 two 3
4 one 3
5 two 4 
data['v1']=range(7)

data
K1 K2 v1
0 one 1 0
1 two 1 1
2 one 2 2
3 two 3 3
4 one 3 4
5 two 4 5
6 two 4 6 
data.drop_duplicates(['K1','K2'])
K1 K2 v1
0 one 1 0
1 two 1 1
2 one 2 2
3 two 3 3
4 one 3 4
5 two 4 5 
df
0 1 2
0 -0.970921 -1.311345 0.779965
1 -0.352837 0.290834 -0.440396
2 0.574406 NaN 2.034865
3 0.088611 NaN -0.004141 
data
K1 K2 v1
0 one 1 0
1 two 1 1
2 one 2 2
3 two 3 3
4 one 3 4
5 two 4 5
6 two 4 6 
data.drop_duplicates(['K1','K2'])
K1 K2 v1
0 one 1 0
1 two 1 1
2 one 2 2
3 two 3 3
4 one 3 4
5 two 4 5

Transforming Data Using a Function or Mapping

import pandas as pd

import numpy as np

data=pd.DataFrame({'food':['bacon','pulled pork','bacon','pastrami','corned beef','Bacon','Pastrami','honey ham','nova lox'],
                  'ounces':[4,3,12,6,7.5,8,3,5,6]});data
food ounces
0 bacon 4.0
1 pulled pork 3.0
2 bacon 12.0
3 pastrami 6.0
4 corned beef 7.5
5 Bacon 8.0
6 Pastrami 3.0
7 honey ham 5.0
8 nova lox 6.0 
meat_to_animal={'bacon':'pig',
               'pulled pork':'pig',
               'pastrami':'cow',
               'corned beef':'cow',
               'honey ham':'pig',
               'nova lox':'salmon'}

pd.Series.str.lower

<function pandas.core.strings._noarg_wrapper.<locals>.wrapper>
  • str.lower above is a Series method.
lowercased=data['food'].str.lower()

data['animal']=lowercased

data
food ounces animal
0 bacon 4.0 bacon
1 pulled pork 3.0 pulled pork
2 bacon 12.0 bacon
3 pastrami 6.0 pastrami
4 corned beef 7.5 corned beef
5 Bacon 8.0 bacon
6 Pastrami 3.0 pastrami
7 honey ham 5.0 honey ham
8 nova lox 6.0 nova lox

The map() method on a Series accepts a function or dict-like object containing a mapping.Using map() is a convenient way to perform element-wise transformations and other data cleaning related operations.

data['animal']=lowercased.map(meat_to_animal);data
food ounces animal
0 bacon 4.0 pig
1 pulled pork 3.0 pig
2 bacon 12.0 pig
3 pastrami 6.0 cow
4 corned beef 7.5 cow
5 Bacon 8.0 pig
6 Pastrami 3.0 cow
7 honey ham 5.0 pig
8 nova lox 6.0 salmon

We could also have passed a function that does all the work.Such as the following:

data['food'].map(lambda x:meat_to_animal[x.lower()])

0       pig
1       pig
2       pig
3       cow
4       cow
5       pig
6       cow
7       pig
8    salmon
Name: food, dtype: object

Replacing values

data=pd.Series([1,-999,2,-999,-1000,3]);data

0       1
1    -999
2       2
3    -999
4   -1000
5       3
dtype: int64

data.replace(-999,np.nan) # Replcace one value with one value

0       1.0
1       NaN
2       2.0
3       NaN
4   -1000.0
5       3.0
dtype: float64

data.replace([-999,-1000],np.nan) # Replace multi-values with one value

0    1.0
1    NaN
2    2.0
3    NaN
4    NaN
5    3.0
dtype: float64

data.replace([-999,-1000],[np.nan,0])# Replace multi-values with multi-values

0    1.0
1    NaN
2    2.0
3    NaN
4    0.0
5    3.0
dtype: float64

data.replace({-999:np.nan,0-1000:0}) # dict can also be passed into replace method

0    1.0
1    NaN
2    2.0
3    NaN
4    0.0
5    3.0
dtype: float64

data1=pd.Series(['A','B','c',12])

help(data1.str.replace)

Help on method replace in module pandas.core.strings:

replace(pat, repl, n=-1, case=True, flags=0) method of pandas.core.strings.StringMethods instance
    Replace occurrences of pattern/regex in the Series/Index with
    some other string. Equivalent to :meth:`str.replace` or
    :func:`re.sub`.
    
    Parameters
    ----------
    pat : string
        Character sequence or regular expression
    repl : string
        Replacement sequence
    n : int, default -1 (all)
        Number of replacements to make from start
    case : boolean, default True
        If True, case sensitive
    flags : int, default 0 (no flags)
        re module flags, e.g. re.IGNORECASE
    
    Returns
    -------
    replaced : Series/Index of objects

Renaming Axis indexes

data=pd.DataFrame(np.arange(12).reshape((3,4)),index=['Ohio','Colorado','New York'],columns=['One','Two','three','Four']);data
One Two three Four
Ohio 0 1 2 3
Colorado 4 5 6 7
New York 8 9 10 11 
data.index.map(lambda x:x[:4].upper())

array(['OHIO', 'COLO', 'NEW '], dtype=object)

data
One Two three Four
Ohio 0 1 2 3
Colorado 4 5 6 7
New York 8 9 10 11 
data.index=data.index.map(lambda x:x[:4].upper());data # Modify DataFrame in-place
One Two three Four
OHIO 0 1 2 3
COLO 4 5 6 7
NEW 8 9 10 11

If you want to create a transformed version of a dataset without modifying the original,a useful method is rename().

data
One Two three Four
OHIO 0 1 2 3
COLO 4 5 6 7
NEW 8 9 10 11 
data.rename(index=str.title,columns=str.upper)
ONE TWO THREE FOUR
Ohio 0 1 2 3
Colo 4 5 6 7
New 8 9 10 11 
data
One Two three Four
OHIO 0 1 2 3
COLO 4 5 6 7
NEW 8 9 10 11

To modify dataset in-place,pass inplace=True.

data.rename(index={'OHIO':'INDIANA'},inplace=True)

data
One Two three Four
INDIANA 0 1 2 3
COLO 4 5 6 7
NEW 8 9 10 11

Discretization and Binning

help(pd.cut)

Help on function cut in module pandas.tools.tile:

cut(x, bins, right=True, labels=None, retbins=False, precision=3, include_lowest=False)
    Return indices of half-open bins to which each value of `x` belongs.
    
    Parameters
    ----------
    x : array-like
        Input array to be binned. It has to be 1-dimensional.
    bins : int or sequence of scalars
        If `bins` is an int, it defines the number of equal-width bins in the
        range of `x`. However, in this case, the range of `x` is extended
        by .1% on each side to include the min or max values of `x`. If
        `bins` is a sequence it defines the bin edges allowing for
        non-uniform bin width. No extension of the range of `x` is done in
        this case.
    right : bool, optional
        Indicates whether the bins include the rightmost edge or not. If
        right == True (the default), then the bins [1,2,3,4] indicate
        (1,2], (2,3], (3,4].
    labels : array or boolean, default None
        Used as labels for the resulting bins. Must be of the same length as
        the resulting bins. If False, return only integer indicators of the
        bins.
    retbins : bool, optional
        Whether to return the bins or not. Can be useful if bins is given
        as a scalar.
    precision : int
        The precision at which to store and display the bins labels
    include_lowest : bool
        Whether the first interval should be left-inclusive or not.
    
    Returns
    -------
    out : Categorical or Series or array of integers if labels is False
        The return type (Categorical or Series) depends on the input: a Series
        of type category if input is a Series else Categorical. Bins are
        represented as categories when categorical data is returned.
    bins : ndarray of floats
        Returned only if `retbins` is True.
    
    Notes
    -----
    The `cut` function can be useful for going from a continuous variable to
    a categorical variable. For example, `cut` could convert ages to groups
    of age ranges.
    
    Any NA values will be NA in the result.  Out of bounds values will be NA in
    the resulting Categorical object
    
    
    Examples
    --------
    >>> pd.cut(np.array([.2, 1.4, 2.5, 6.2, 9.7, 2.1]), 3, retbins=True)
    ([(0.191, 3.367], (0.191, 3.367], (0.191, 3.367], (3.367, 6.533],
      (6.533, 9.7], (0.191, 3.367]]
    Categories (3, object): [(0.191, 3.367] < (3.367, 6.533] < (6.533, 9.7]],
    array([ 0.1905    ,  3.36666667,  6.53333333,  9.7       ]))
    >>> pd.cut(np.array([.2, 1.4, 2.5, 6.2, 9.7, 2.1]), 3,
               labels=["good","medium","bad"])
    [good, good, good, medium, bad, good]
    Categories (3, object): [good < medium < bad]
    >>> pd.cut(np.ones(5), 4, labels=False)
    array([1, 1, 1, 1, 1], dtype=int64)

ages=[20,22,25,27,21,23,37,31,61,45,41,32]

bins=[18,25,35,60,100]

cats=pd.cut(ages,bins)

cats

[(18, 25], (18, 25], (18, 25], (25, 35], (18, 25], ..., (25, 35], (60, 100], (35, 60], (35, 60], (25, 35]]
Length: 12
Categories (4, object): [(18, 25] < (25, 35] < (35, 60] < (60, 100]]

len(ages)

12

type(cats)

pandas.core.categorical.Categorical

  cats.codes

array([0, 0, 0, 1, 0, 0, 2, 1, 3, 2, 2, 1], dtype=int8)

cats.categories

Index(['(18, 25]', '(25, 35]', '(35, 60]', '(60, 100]'], dtype='object')

type(pd.value_counts(cats))

pandas.core.series.Series

help(pd.value_counts)

Help on function value_counts in module pandas.core.algorithms:

value_counts(values, sort=True, ascending=False, normalize=False, bins=None, dropna=True)
    Compute a histogram of the counts of non-null values.
    
    Parameters
    ----------
    values : ndarray (1-d)
    sort : boolean, default True
        Sort by values
    ascending : boolean, default False
        Sort in ascending order
    normalize: boolean, default False
        If True then compute a relative histogram
    bins : integer, optional
        Rather than count values, group them into half-open bins,
        convenience for pd.cut, only works with numeric data
    dropna : boolean, default True
        Don't include counts of NaN
    
    Returns
    -------
    value_counts : Series

pd.value_counts([1,1,2,3,4,45,5])

1     2
5     1
45    1
4     1
3     1
2     1
dtype: int64

pd.value_counts(cats)

(18, 25]     5
(35, 60]     3
(25, 35]     3
(60, 100]    1
dtype: int64

You can also pass your bin names by passing a list or array to the labels option.

group_names=['Youth','YoungAdult','MiddleAged','Senior']

pd.cut(ages,bins,labels=group_names) # bin is a reserved key.

[Youth, Youth, Youth, YoungAdult, Youth, ..., YoungAdult, Senior, MiddleAged, MiddleAged, YoungAdult]
Length: 12
Categories (4, object): [Youth < YoungAdult < MiddleAged < Senior]

help(bin)

Help on built-in function bin in module builtins:

bin(number, /)
    Return the binary representation of an integer.
    
    >>> bin(2796202)
    '0b1010101010101010101010'

bin(2)

'0b10'
  • bins can also be an integer, and in that case, the category will be equal-space.
data=np.random.rand(20)

pd.cut(data,4,precision=2)# precision limits the decimal precision to two digits.

[(0.25, 0.5], (0.25, 0.5], (0.25, 0.5], (0.75, 1], (0.5, 0.75], ..., (0.0024, 0.25], (0.25, 0.5], (0.25, 0.5], (0.25, 0.5], (0.0024, 0.25]]
Length: 20
Categories (4, object): [(0.0024, 0.25] < (0.25, 0.5] < (0.5, 0.75] < (0.75, 1]]
  • A closely related function,qcut,bins the data based on sample quantiles.Using cut will not usually result in each bin having the same number of data points.
data=np.random.randn(1000)

cats=pd.qcut(data,4);cats

[(0.0211, 0.689], (0.689, 3.225], (-0.62, 0.0211], (0.689, 3.225], (0.689, 3.225], ..., (0.689, 3.225], [-3.401, -0.62], (-0.62, 0.0211], (-0.62, 0.0211], (-0.62, 0.0211]]
Length: 1000
Categories (4, object): [[-3.401, -0.62] < (-0.62, 0.0211] < (0.0211, 0.689] < (0.689, 3.225]]

pd.value_counts(cats)

(0.689, 3.225]     250
(0.0211, 0.689]    250
(-0.62, 0.0211]    250
[-3.401, -0.62]    250
dtype: int64

cats1=pd.qcut(data,[0,0.1,0.5,0.9,1])

pd.value_counts(cats1)

(0.0211, 1.33]      400
(-1.201, 0.0211]    400
(1.33, 3.225]       100
[-3.401, -1.201]    100
dtype: int64

Detecting and filtering Outliers

data=pd.DataFrame(np.random.randn(1000,4))

data.describe()
0 1 2 3
count 1000.000000 1000.000000 1000.000000 1000.000000
mean 0.002634 -0.038263 0.001432 -0.040628
std 0.981600 0.996856 1.021248 1.030675
min -3.400618 -3.427137 -4.309211 -4.375632
25% -0.656369 -0.713371 -0.681777 -0.754702
50% -0.005199 -0.026878 -0.019116 0.005450
75% 0.649159 0.613807 0.690614 0.625859
max 3.408137 3.171119 3.784272 2.992607 
col=data[2]

col[np.abs(col)>3]

322    3.059163
431   -3.089013
648   -4.309211
653    3.784272
834    3.007481
Name: 2, dtype: float64

help(pd.DataFrame.any)

Help on function any in module pandas.core.frame:

any(self, axis=None, bool_only=None, skipna=None, level=None, **kwargs)
    Return whether any element is True over requested axis
    
    Parameters
    ----------
    axis : {index (0), columns (1)}
    skipna : boolean, default True
        Exclude NA/null values. If an entire row/column is NA, the result
        will be NA
    level : int or level name, default None
        If the axis is a MultiIndex (hierarchical), count along a
        particular level, collapsing into a Series
    bool_only : boolean, default None
        Include only boolean columns. If None, will attempt to use everything,
        then use only boolean data. Not implemented for Series.
    
    Returns
    -------
    any : Series or DataFrame (if level specified)

(abs(data)>3) ==(np.abs(data)>3)
0 1 2 3
0 True True True True
1 True True True True
2 True True True True
3 True True True True
4 True True True True
5 True True True True
6 True True True True
7 True True True True
8 True True True True
9 True True True True
10 True True True True
11 True True True True
12 True True True True
13 True True True True
14 True True True True
15 True True True True
16 True True True True
17 True True True True
18 True True True True
19 True True True True
20 True True True True
21 True True True True
22 True True True True
23 True True True True
24 True True True True
25 True True True True
26 True True True True
27 True True True True
28 True True True True
29 True True True True
... ... ... ... ...
970 True True True True
971 True True True True
972 True True True True
973 True True True True
974 True True True True
975 True True True True
976 True True True True
977 True True True True
978 True True True True
979 True True True True
980 True True True True
981 True True True True
982 True True True True
983 True True True True
984 True True True True
985 True True True True
986 True True True True
987 True True True True
988 True True True True
989 True True True True
990 True True True True
991 True True True True
992 True True True True
993 True True True True
994 True True True True
995 True True True True
996 True True True True
997 True True True True
998 True True True True
999 True True True True

1000 rows × 4 columns

data[(np.abs(data)>3).any(1)]
0 1 2 3
59 -3.400618 0.342563 0.649758 -2.629268
274 1.264869 -3.427137 0.991494 -0.906788
322 2.714233 -1.239436 3.059163 0.318054
431 -0.376058 -0.713530 -3.089013 -0.791221
460 0.411801 -0.323974 0.301139 -3.051362
465 0.054043 -1.046532 2.054820 -4.375632
587 0.857067 -3.162763 0.137409 -1.327873
648 -0.323629 0.325867 -4.309211 -0.477572
653 0.171840 0.148702 3.784272 0.269508
678 0.303109 3.171119 0.854269 0.489537
834 1.651314 1.303992 3.007481 0.494971
841 3.408137 0.869413 -0.111245 1.306775
960 -0.302520 -3.118445 2.116509 0.003669 
np.sign([0,0.3,-0.3,20,-90])

array([ 0.,  1., -1.,  1., -1.])

data[np.abs(data)>3]=np.sign(data)*3

np.sign(data)*3
0 1 2 3
0 -3.0 -3.0 -3.0 3.0
1 -3.0 3.0 3.0 -3.0
2 3.0 3.0 3.0 3.0
3 -3.0 -3.0 -3.0 -3.0
4 3.0 -3.0 3.0 -3.0
5 -3.0 -3.0 -3.0 3.0
6 -3.0 -3.0 3.0 3.0
7 3.0 3.0 3.0 3.0
8 -3.0 3.0 -3.0 3.0
9 3.0 -3.0 3.0 3.0
10 -3.0 3.0 -3.0 -3.0
11 -3.0 3.0 3.0 3.0
12 3.0 3.0 3.0 3.0
13 3.0 -3.0 3.0 3.0
14 3.0 3.0 3.0 3.0
15 3.0 3.0 3.0 -3.0
16 3.0 -3.0 3.0 3.0
17 3.0 -3.0 -3.0 3.0
18 -3.0 3.0 3.0 3.0
19 3.0 3.0 3.0 -3.0
20 -3.0 3.0 3.0 3.0
21 3.0 3.0 -3.0 3.0
22 -3.0 3.0 -3.0 -3.0
23 3.0 3.0 -3.0 -3.0
24 3.0 -3.0 3.0 3.0
25 -3.0 -3.0 -3.0 3.0
26 3.0 3.0 -3.0 -3.0
27 3.0 -3.0 -3.0 -3.0
28 3.0 -3.0 -3.0 3.0
29 3.0 3.0 -3.0 -3.0
... ... ... ... ...
970 -3.0 -3.0 3.0 -3.0
971 -3.0 3.0 -3.0 -3.0
972 -3.0 3.0 -3.0 3.0
973 3.0 3.0 3.0 3.0
974 3.0 -3.0 -3.0 3.0
975 -3.0 3.0 -3.0 3.0
976 -3.0 3.0 3.0 3.0
977 -3.0 -3.0 3.0 -3.0
978 3.0 -3.0 -3.0 -3.0
979 -3.0 3.0 -3.0 3.0
980 -3.0 -3.0 -3.0 3.0
981 3.0 3.0 3.0 -3.0
982 -3.0 3.0 -3.0 -3.0
983 -3.0 3.0 -3.0 -3.0
984 3.0 3.0 -3.0 -3.0
985 3.0 3.0 -3.0 3.0
986 -3.0 -3.0 -3.0 3.0
987 -3.0 3.0 -3.0 -3.0
988 3.0 3.0 -3.0 -3.0
989 3.0 -3.0 -3.0 3.0
990 3.0 -3.0 3.0 -3.0
991 3.0 -3.0 3.0 3.0
992 -3.0 3.0 -3.0 -3.0
993 -3.0 3.0 -3.0 3.0
994 3.0 -3.0 -3.0 -3.0
995 3.0 -3.0 -3.0 -3.0
996 3.0 -3.0 3.0 -3.0
997 -3.0 -3.0 -3.0 -3.0
998 3.0 3.0 -3.0 -3.0
999 3.0 3.0 3.0 -3.0

1000 rows × 4 columns

data
0 1 2 3
0 -0.564062 -0.887969 -0.854782 0.107613
1 -1.364165 1.337851 1.671698 -0.814129
2 0.765877 1.916774 0.441002 2.128419
3 -0.581957 -1.024641 -1.983024 -2.757392
4 0.778034 -1.375845 0.044277 -1.037062
5 -0.796683 -0.540663 -0.120198 0.003503
6 -0.708554 -0.105414 1.037527 0.826310
7 1.233856 1.217529 1.097430 0.842746
8 -0.201433 0.249823 -1.620147 0.436595
9 1.328493 -0.396323 1.927629 1.615656
10 -0.560207 0.252996 -0.151543 -0.667813
11 -1.729057 1.144087 1.087689 0.520086
12 0.704758 1.707940 0.720834 0.447245
13 1.024834 -0.217376 1.340304 0.176801
14 0.075745 1.430761 0.193627 0.191701
15 0.536566 0.047559 1.715175 -1.115074
16 2.803965 -0.465377 1.127140 1.417856
17 0.677525 -1.091631 -0.572231 0.241533
18 -1.172228 1.049830 0.266288 0.836902
19 0.930699 0.379891 1.637741 -1.770379
20 -0.749769 0.711326 1.591292 1.099071
21 1.550585 1.276488 -0.214484 0.195340
22 -0.289236 1.882439 -0.275263 -0.247316
23 0.688167 0.357913 -1.675828 -0.305840
24 1.255532 -1.802804 0.889900 0.864982
25 -1.391447 -0.291022 -0.190022 0.540653
26 0.435101 2.444416 -1.235937 -0.428450
27 0.165456 -1.091942 -1.560662 -0.739435
28 1.469728 -0.123806 -2.071746 2.574603
29 1.287949 1.278130 -0.825906 -1.852465
... ... ... ... ...
970 -0.379102 -0.778606 2.213794 -0.062573
971 -1.108557 0.723650 -2.436704 -0.068733
972 -0.518995 0.455508 -0.217321 1.363977
973 0.444636 1.625221 0.222103 1.236397
974 0.699354 -2.076747 -0.454499 0.383902
975 -1.759718 0.717117 -0.077413 1.698893
976 -1.230778 0.222673 0.151731 0.174875
977 -0.575290 -0.316810 0.380077 -0.048428
978 1.906133 -0.861802 -0.026937 -2.865641
979 -0.134489 0.607949 -0.821089 0.831827
980 -0.058894 -0.707492 -0.273980 0.129724
981 2.288519 0.149683 0.580679 -0.055218
982 -0.280748 0.861358 -0.254339 -0.596723
983 -1.322965 0.323534 -0.585862 -1.316894
984 0.793711 0.165646 -0.212855 -1.752453
985 0.310908 0.758156 -0.040923 0.538293
986 -0.589173 -1.688947 -0.501485 0.019880
987 -0.111807 1.007026 -0.853133 -0.249211
988 0.601993 0.690953 -1.168277 -0.516737
989 1.319895 -0.046141 -0.680194 1.443361
990 1.839785 -0.480675 0.056481 -0.097993
991 2.590916 -0.367057 1.110105 0.130826
992 -0.108846 1.717209 -0.580895 -0.985869
993 -1.152810 0.390732 -0.104866 1.553947
994 1.721177 -0.088994 -0.565308 -1.602808
995 0.922409 -0.027923 -1.258001 -1.933848
996 0.647699 -0.089378 1.455509 -0.598519
997 -1.590236 -0.544202 -0.764923 -0.329425
998 0.969542 0.106538 -0.188919 -1.474017
999 0.235337 0.232514 0.113181 -1.403455

1000 rows × 4 columns

np.sign(data).head(10) # return the first 10 rows.
0 1 2 3
0 -1.0 -1.0 -1.0 1.0
1 -1.0 1.0 1.0 -1.0
2 1.0 1.0 1.0 1.0
3 -1.0 -1.0 -1.0 -1.0
4 1.0 -1.0 1.0 -1.0
5 -1.0 -1.0 -1.0 1.0
6 -1.0 -1.0 1.0 1.0
7 1.0 1.0 1.0 1.0
8 -1.0 1.0 -1.0 1.0
9 1.0 -1.0 1.0 1.0

Permutation and random sample

df=pd.DataFrame(np.arange(20).reshape((5,4)))

sampler=np.random.permutation(5);sampler

array([4, 3, 1, 2, 0])

df.take(sampler)
0 1 2 3
4 16 17 18 19
3 12 13 14 15
1 4 5 6 7
2 8 9 10 11
0 0 1 2 3 
df.sample(n=4)
0 1 2 3
2 8 9 10 11
0 0 1 2 3
1 4 5 6 7
4 16 17 18 19 
df.sample(n=10,replace=True) # replace allows repeat choices.
0 1 2 3
1 4 5 6 7
2 8 9 10 11
1 4 5 6 7
2 8 9 10 11
0 0 1 2 3
3 12 13 14 15
3 12 13 14 15
2 8 9 10 11
0 0 1 2 3
4 16 17 18 19 
choices=pd.Series([5,7,-1,6,4])

choices.sample(n=10,replace=True)

2   -1
4    4
1    7
0    5
0    5
3    6
4    4
2   -1
1    7
1    7
dtype: int64

Computing indicator/Dummy variables

df=pd.DataFrame({'Key':['b','b','a','c','a','b'],'data1':range(6)});df
Key data1
0 b 0
1 b 1
2 a 2
3 c 3
4 a 4
5 b 5 
pd.get_dummies(df['Key'])
a b c
0 0 1 0
1 0 1 0
2 1 0 0
3 0 0 1
4 1 0 0
5 0 1 0 
pd.get_dummies(df['Key'],prefix='key')
key_a key_b key_c
0 0 1 0
1 0 1 0
2 1 0 0
3 0 0 1
4 1 0 0
5 0 1 0 
df[['data1']]
data1
0 0
1 1
2 2
3 3
4 4
5 5 
df['data1']

0    0
1    1
2    2
3    3
4    4
5    5
Name: data1, dtype: int32
  • so the difference between df[['data1']] and df['data1'] is apparent, the former one returns DataFrame,the latter one returns Series.
posted @ 2020-04-16 13:32  JohnYang819  阅读(175)  评论(0)  编辑  收藏  举报