# %load ipython_log.py
# IPython log file
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
path='./usagov_bitly_data2012-03-16-1331923249.txt'
open(path).readline()
import json
record=[json.loads(line) for line in open(path)] #Read in json format
record[0] #Since the content of record is long, let's look at only one element for the time being
record[0]['tz'] #Of which tz is the key
time_zone=[rec['tz'] for rec in record if 'tz' in rec] #See only tz in record. But only when there was tz
time_zone[:10] #See only the 10th line from the top
def get_counts(seq):
'''
Count how many are the same as the character string in seq, and dictionary{'String':Quantity,...}Return as
def get_counts(seq):#Same meaning as this, but easy with defaultdict
count=defaultdict(int) #count={}
for x in seq:
if x in counts:
counts[x]+=1
else:
count[x]=1
return counts
'''
from collections import defaultdict
counts=defaultdict(int) #returns `defaultdict(<class 'int'>, {})`
for x in seq:
counts[x]+=1
return counts
counts=get_counts(time_zone)
counts['America/New_York']
len(time_zone)
What we are doing is different in shape, but everyone is together
def top_counts(count_dict,n=10):
value_key_pairs=[(count,tz) for tz, count in count_dict.items()]
value_key_pairs.sort()
return value_key_pairs[-n:]
top_counts(counts)
from collections import Counter
counts= Counter(time_zone)
counts.most_common(10)
from pandas import DataFrame,Series
import pandas as pd
frame=DataFrame(record)
frame['tz'][:10]
tz_counts=frame['tz'].value_counts()
tz_counts[:10]
clean_tz=frame['tz'].fillna('Missing')
clean_tz[clean_tz=='']='UNknown'
clean_tz
tz_counts=clean_tz.value_counts()
tz_counts[:10]
tz_counts[:10].plot(kind='barh',rot=0)
import matplotlib.pyplot as plt
# plt.show()
frame['a'][1]
frame['a'][50]
frame['a'][51]
results=Series([x.split()[0] for x in frame.a.dropna()]) #.dropna()pandas method Delete blank line Specify line to delete with argument
#str.split(x)Divide str into a list with x as the delimiter
#List the strings separated by spaces(In-list notation), Make pandas dataframe in Series class
results[:5]
results.value_counts()[:8] #value_counts()Count the number of the same element with
cframe=frame[frame.a.notnull()] #Collected only non-null guys in column a of frame(cframe['a']==frame.a.dropna())
bool(map(list,[cframe['a'],frame.a.dropna()])) #cframe the list function['a']And frame.a.dropna()Apply to and see if they are the same
'Windows' or Not?
import numpy as np
operating_system=np.where(cframe['a'].str.contains('Windows'),'Windows','Not Windows') #cframe['a']But'Windows'In True, including the characters'Windows'with false'Not Windows'return it
#` ['Windows' if 'Windows' in x else 'Not Windows' for x in cframe['a']]`Same as
operating_system[:5]
operating_system Another Way
operating_system2=['Windows' if 'Windows' in x else 'Not Windows' for x in cframe['a']]
bool(list(operating_system)==operating_system2) #True
by_tz_os=cframe.groupby(['tz',operating_system])
agg_counts=by_tz_os.size().unstack().fillna(0)
agg_counts[:10]
#2016/07/28 22:56:30__
indexer=agg_counts.sum(1).argsort() #argsort()Np sorted index.Returns in array format
#np.sum()Basically, return the one with all the contents of the array added
'''
# ABOUT np.sum()
>>> np.sum([[0, 1], [0, 5]], axis=0)
array([0, 6]) #return array([0+0],[1+5])
>>> np.sum([[0, 1], [0, 5]], axis=1)
array([1, 5]) #return array([0+1],[0+5])
'''
indexer[:10]
count_subset=agg_counts.take(indexer)[-10:] #Only 10 minutes from the end of the indexer agg_Returns counts(take=get)
count_subset.plot(kind='barh', stacked=True)
# plt.show()
normed_subset=count_subset.div(count_subset.sum(1),axis=0)
normed_subset.plot(kind='barh',stacked=True)
# plt.show()
Recommended Posts