#!/usr/bin/env python
# coding: utf-8
# # 推荐系统
#
# - 音乐数据处理
#
# - 基于商品相似性的推荐
#
# - 基于SVD矩阵分解的推荐
# In[1]:
import os
os.getcwd()
# ## 数据读取
# In[2]:
import pandas as pd
import numpy as np
import time
import sqlite3
data_home = './'
# 在数据中只需要用户，歌曲，播放量
# In[4]:
get_ipython().run_cell_magic('time', '', "triplet_dataset = pd.read_csv(filepath_or_buffer=data_home+'train_triplets.txt', \n sep='\\t', header=None, \n names=['user','song','play_count'])")
# In[5]:
get_ipython().run_cell_magic('time', '', 'triplet_dataset.shape')
# In[6]:
get_ipython().run_cell_magic('time', '', 'triplet_dataset.info()')
# In[7]:
get_ipython().run_cell_magic('time', '', 'triplet_dataset.head(n=10)')
# In[8]:
get_ipython().run_cell_magic('time', '', "triplet_dataset['play_count'] = triplet_dataset['play_count'].astype('int32')")
# In[9]:
triplet_dataset.info()
# In[10]:
triplet_dataset.head(n=10)
# In[11]:
get_ipython().run_cell_magic('time', '', 'triplet_dataset.count()')
# ## 对每一个用户，分别统计他的播放总量
# In[12]:
get_ipython().run_cell_magic('time', '', "output_dict = {}\n# b80344d063b5ccb3212f76538f3d9e43d87dca9e \tSOBXHDL12A81C204C0 \t1\nwith open(data_home+'train_triplets.txt') as f:\n for line_number, line in enumerate(f):\n user = line.split('\\t')[0] # b80344d063b5ccb3212f76538f3d9e43d87dca9e\n play_count = int(line.split('\\t')[2]) # 1\n if user in output_dict: # 如果这个用在字典中 \n play_count +=output_dict[user] # 将play_count播放量累加\n output_dict.update({user:play_count}) # 更新该用户的累计播放量\n output_dict.update({user:play_count}) # 给该用户初始赋值\noutput_list = [{'user':k,'play_count':v} for k,v in output_dict.items()] # 将字典转换成一个列表\nplay_count_df = pd.DataFrame(output_list) # 将列表转换成dataframe\nplay_count_df = play_count_df.sort_values(by = 'play_count', ascending = False)\nplay_count_df.head(10)")
# In[16]:
play_count_df.count()
# In[13]:
play_count_df.head(10)
# In[14]:
get_ipython().run_cell_magic('time', '', "play_count_df.to_csv(path_or_buf='user_playcount_df.csv', index = False)")
# ## 对于每一首歌，分别统计它的播放总量
# In[15]:
get_ipython().run_cell_magic('time', '', "output_dict = {}\nwith open(data_home+'train_triplets.txt') as f:\n for line_number, line in enumerate(f):\n song = line.split('\\t')[1]\n play_count = int(line.split('\\t')[2])\n if song in output_dict:\n play_count +=output_dict[song]\n output_dict.update({song:play_count})\n output_dict.update({song:play_count})\noutput_list = [{'song':k,'play_count':v} for k,v in output_dict.items()]\nsong_count_df = pd.DataFrame(output_list)\nsong_count_df = song_count_df.sort_values(by = 'play_count', ascending = False)")
# In[18]:
song_count_df.head(10)
# song_count_df.tail(10)
# In[19]:
song_count_df.count()
# In[20]:
get_ipython().run_cell_magic('time', '', "song_count_df.to_csv(path_or_buf='song_playcount_df.csv', index = False)")
# ## 看看目前的排行情况
# In[21]:
play_count_df = pd.read_csv(filepath_or_buffer='user_playcount_df.csv')
play_count_df.head(n =10)
# In[22]:
song_count_df = pd.read_csv(filepath_or_buffer='song_playcount_df.csv')
song_count_df.head(10)
# ## 取其中一部分数（按大小排好序的了，这些应该是比较重要的数据），作为我们的实验数据
# In[23]:
get_ipython().run_cell_magic('time', '', 'total_play_count = sum(song_count_df.play_count) # 统计所有歌曲的播放总量\nprint ((float(play_count_df.head(n=100000).play_count.sum())/total_play_count)*100) # 播放量前10w的用户播放总量/所有歌曲播放总量\nplay_count_subset = play_count_df.head(n=100000) # 将前10w的用户播放量保存成一个新的df')
# In[24]:
get_ipython().run_cell_magic('time', '', '(float(song_count_df.head(n=30000).play_count.sum())/total_play_count)*100 # 前3w首最热门的歌曲的播放总量/所有歌曲的播放总量')
# In[25]:
song_count_subset = song_count_df.head(n=30000)
# 取10W个用户，3W首歌
# In[26]:
user_subset = list(play_count_subset.user)
song_subset = list(song_count_subset.song)
# 过滤掉其他用户数据
# In[ ]:
get_ipython().run_cell_magic('time', '', "triplet_dataset = pd.read_csv(filepath_or_buffer=data_home+'train_triplets.txt',sep='\\t', \n header=None, names=['user','song','play_count'])\ntriplet_dataset_sub = triplet_dataset[triplet_dataset.user.isin(user_subset) ]\ndel(triplet_dataset)\ntriplet_dataset_sub_song = triplet_dataset_sub[triplet_dataset_sub.song.isin(song_subset)]\ndel(triplet_dataset_sub)")
# In[18]:
triplet_dataset_sub_song.to_csv(path_or_buf=data_home+'triplet_dataset_sub_song.csv', index=False)
# 我们的数据量
# In[19]:
triplet_dataset_sub_song.shape
# In[20]:
triplet_dataset_sub_song.head(n=10)
# ## 加入音乐详细信息
# .db文件需要稍微处理下 转换成csv
# In[21]:
conn = sqlite3.connect(data_home+'track_metadata.db')
cur = conn.cursor()
cur.execute("SELECT name FROM sqlite_master WHERE type='table'")
cur.fetchall()
# In[22]:
track_metadata_df = pd.read_sql(con=conn, sql='select * from songs')
track_metadata_df_sub = track_metadata_df[track_metadata_df.song_id.isin(song_subset)]
# In[23]:
track_metadata_df_sub.to_csv(path_or_buf=data_home+'track_metadata_df_sub.csv', index=False)
# In[24]:
track_metadata_df_sub.shape
# ## 我们现有的数据
# In[25]:
# 用户 音乐 播放量 10w用户 3w歌曲
triplet_dataset_sub_song = pd.read_csv(filepath_or_buffer=data_home+'triplet_dataset_sub_song.csv',encoding = "ISO-8859-1")
# 3w歌曲的信息
track_metadata_df_sub = pd.read_csv(filepath_or_buffer=data_home+'track_metadata_df_sub.csv',encoding = "ISO-8859-1")
# In[26]:
triplet_dataset_sub_song.head()
# In[27]:
# 音乐元数据的信息
track_metadata_df_sub.head()
# ## 清洗数据集
# 去除掉无用的和重复的
# In[28]:
del(track_metadata_df_sub['track_id'])
del(track_metadata_df_sub['artist_mbid']) # 删除字段
track_metadata_df_sub = track_metadata_df_sub.drop_duplicates(['song_id']) # 去重
# merge合并 triplet_dataset_sub_song用户id音乐id播放量 track_metadata_df_sub音乐id+各种信息
triplet_dataset_sub_song_merged = pd.merge(triplet_dataset_sub_song, track_metadata_df_sub, how='left', left_on='song', right_on='song_id')
triplet_dataset_sub_song_merged.rename(columns={'play_count':'listen_count'},inplace=True)
# In[29]:
del(triplet_dataset_sub_song_merged['song_id'])
del(triplet_dataset_sub_song_merged['artist_id'])
del(triplet_dataset_sub_song_merged['duration'])
del(triplet_dataset_sub_song_merged['artist_familiarity'])
del(triplet_dataset_sub_song_merged['artist_hotttnesss'])
del(triplet_dataset_sub_song_merged['track_7digitalid'])
del(triplet_dataset_sub_song_merged['shs_perf'])
del(triplet_dataset_sub_song_merged['shs_work'])
# 搞定数据
# In[31]:
triplet_dataset_sub_song_merged.head(n=10)
# ## 瞅瞅音乐数据集的情况
# ### 最流行的歌曲
# In[32]:
# 返回歌曲名 以及歌曲的播放量
popular_songs = triplet_dataset_sub_song_merged[['title','listen_count']].groupby('title').sum().reset_index()
# 按照播放量排序 倒序 取前20
popular_songs_top_20 = popular_songs.sort_values('listen_count', ascending=False).head(n=20)
import matplotlib.pyplot as plt; plt.rcdefaults()
import numpy as np
import matplotlib.pyplot as plt
objects = (list(popular_songs_top_20['title'])) # 20首歌的名字
y_pos = np.arange(len(objects)) # 0-19
performance = list(popular_songs_top_20['listen_count']) # 20首歌的播放量
plt.bar(y_pos, performance, align='center', alpha=0.5)
plt.xticks(y_pos, objects, rotation='vertical')
plt.ylabel('Item count')
plt.title('Most popular songs')
plt.show()
# ### 最受欢迎的releases
# In[33]:
popular_release = triplet_dataset_sub_song_merged[['release','listen_count']].groupby('release').sum().reset_index()
popular_release_top_20 = popular_release.sort_values('listen_count', ascending=False).head(n=20)
objects = (list(popular_release_top_20['release']))
y_pos = np.arange(len(objects))
performance = list(popular_release_top_20['listen_count'])
plt.bar(y_pos, performance, align='center', alpha=0.5)
plt.xticks(y_pos, objects, rotation='vertical')
plt.ylabel('Item count')
plt.title('Most popular Release')
plt.show()
# ## 最受欢迎的歌手
# In[34]:
popular_artist = triplet_dataset_sub_song_merged[['artist_name','listen_count']].groupby('artist_name').sum().reset_index()
popular_artist_top_20 = popular_artist.sort_values('listen_count', ascending=False).head(n=20)
objects = (list(popular_artist_top_20['artist_name']))
y_pos = np.arange(len(objects))
performance = list(popular_artist_top_20['listen_count'])
plt.bar(y_pos, performance, align='center', alpha=0.5)
plt.xticks(y_pos, objects, rotation='vertical')
plt.ylabel('Item count')
plt.title('Most popular Artists')
plt.show()
# ## 用户播放量的分布
# In[35]:
user_song_count_distribution = triplet_dataset_sub_song_merged[['user','title']].groupby('user').count().reset_index().sort_values(
by='title',ascending = False)
user_song_count_distribution.title.describe()
# In[36]:
x = user_song_count_distribution.title
n, bins, patches = plt.hist(x, 50, facecolor='green', alpha=0.75)
plt.xlabel('Play Counts')
plt.ylabel('Num of Users')
plt.title(r'$\mathrm{Histogram\ of\ User\ Play\ Count\ Distribution}\ $')
plt.grid(True)
plt.show()
# # 推荐系统
# In[38]:
import Recommenders as Recommenders
from sklearn.model_selection import train_test_split
# ## 简单暴力，排行榜单推荐
# In[39]:
triplet_dataset_sub_song_merged_set = triplet_dataset_sub_song_merged
train_data, test_data = train_test_split(triplet_dataset_sub_song_merged_set, test_size = 0.40, random_state=0)
# In[40]:
train_data.head()
# In[41]:
def create_popularity_recommendation(train_data, user_id, item_id):
#Get a count of user_ids for each unique song as recommendation score
# #根据指定的特征来统计其播放情况，可以选择歌曲名，专辑名，歌手名
train_data_grouped = train_data.groupby([item_id]).agg({user_id: 'count'}).reset_index() # 按照歌曲名称分组，统计每组的数量
# #为了直观展示，我们用得分来表示其结果
train_data_grouped.rename(columns = {user_id: 'score'},inplace=True) # 将每个歌曲的用户量重命名为歌曲的得分
#Sort the songs based upon recommendation score
# #排行榜单需要排序
train_data_sort = train_data_grouped.sort_values(['score', item_id], ascending = [0,1])
#Generate a recommendation rank based upon score
#加入一项排行等级，表示其推荐的优先级
train_data_sort['Rank'] = train_data_sort['score'].rank(ascending=0, method='first')
#Get the top 10 recommendations
popularity_recommendations = train_data_sort.head(20)
return popularity_recommendations
# In[42]:
recommendations = create_popularity_recommendation(triplet_dataset_sub_song_merged,'user','title')
# In[43]:
recommendations
# ## 基于歌曲相似度的推荐
# 选择一小部分歌曲来实验
# In[44]:
song_count_subset = song_count_df.head(n=5000) # 获取全部歌曲的前5000条
user_subset = list(play_count_subset.user) # 获取10w用户
song_subset = list(song_count_subset.song) # 获取5000歌曲id
# triplet_dataset_sub_song_merged为用户 歌曲 播放量 + 歌曲信息
triplet_dataset_sub_song_merged_sub = triplet_dataset_sub_song_merged[triplet_dataset_sub_song_merged.song.isin(song_subset)]
# In[45]:
triplet_dataset_sub_song_merged_sub.head()
# In[46]:
train_data, test_data = train_test_split(triplet_dataset_sub_song_merged_sub, test_size = 0.30, random_state=0)
is_model = Recommenders.item_similarity_recommender_py()
is_model.create(train_data, 'user', 'title')
user_id = list(train_data.user)[7]
user_items = is_model.get_user_items(user_id)
# In[47]:
#Recommend songs for the user using personalized model
is_model.recommend(user_id)
# ## 基于矩阵分解（SVD）的推荐
# <img src="1.png" style="width:550px;height:280px;float:left">
# <img src="5.png" style="width:350px;height:280px;float:left">
# 对矩阵进行SVD分解，将得到USV
# <img src="2.png" style="width:500px;height:380px;float:left"><img src="3.png" style="width:400px;height:200px;float:left">
# 重新计算 U*S*V的结果得到A2 来比较下A2和A的差异，看起来差异是有的，但是并不大，所以我们可以近似来代替
# <img src="4.png" style="width:330px;height:220px;float:left">
# <img src="5.png" style="width:330px;height:220px;float:left">
# <img src="6.png" style="width:650px;height:480px;float:left">
# <img src="7.png" style="width:650px;height:480px;float:left">
# 先计算歌曲被当前用户播放量 / 用户播放总量 当做分值
# In[48]:
triplet_dataset_sub_song_merged_sum_df = triplet_dataset_sub_song_merged[['user','listen_count']].groupby('user').sum().reset_index()
triplet_dataset_sub_song_merged_sum_df.rename(columns={'listen_count':'total_listen_count'},inplace=True)
triplet_dataset_sub_song_merged = pd.merge(triplet_dataset_sub_song_merged,triplet_dataset_sub_song_merged_sum_df)
triplet_dataset_sub_song_merged.head()
# In[49]:
triplet_dataset_sub_song_merged['fractional_play_count'] = triplet_dataset_sub_song_merged['listen_count']/triplet_dataset_sub_song_merged['total_listen_count']
# 大概是这样
# In[50]:
triplet_dataset_sub_song_merged[triplet_dataset_sub_song_merged.user =='d6589314c0a9bcbca4fee0c93b14bc402363afea'][['user','song','listen_count','fractional_play_count']].head()
# In[51]:
from scipy.sparse import coo_matrix
small_set = triplet_dataset_sub_song_merged
user_codes = small_set.user.drop_duplicates().reset_index()
song_codes = small_set.song.drop_duplicates().reset_index()
user_codes.rename(columns={'index':'user_index'}, inplace=True)
song_codes.rename(columns={'index':'song_index'}, inplace=True)
song_codes['so_index_value'] = list(song_codes.index)
user_codes['us_index_value'] = list(user_codes.index)
small_set = pd.merge(small_set,song_codes,how='left')
small_set = pd.merge(small_set,user_codes,how='left')
mat_candidate = small_set[['us_index_value','so_index_value','fractional_play_count']]
data_array = mat_candidate.fractional_play_count.values
row_array = mat_candidate.us_index_value.values
col_array = mat_candidate.so_index_value.values
data_sparse = coo_matrix((data_array, (row_array, col_array)),dtype=float)
# In[52]:
data_sparse
# In[53]:
user_codes[user_codes.user =='2a2f776cbac6df64d6cb505e7e834e01684673b6']
# In[54]:
import math as mt
from scipy.sparse.linalg import * #used for matrix multiplication
from scipy.sparse.linalg import svds
from scipy.sparse import csc_matrix
# In[55]:
def compute_svd(urm, K):
U, s, Vt = svds(urm, K)
dim = (len(s), len(s))
S = np.zeros(dim, dtype=np.float32)
for i in range(0, len(s)):
S[i,i] = mt.sqrt(s[i])
U = csc_matrix(U, dtype=np.float32)
S = csc_matrix(S, dtype=np.float32)
Vt = csc_matrix(Vt, dtype=np.float32)
return U, S, Vt
def compute_estimated_matrix(urm, U, S, Vt, uTest, K, test):
rightTerm = S*Vt
max_recommendation = 250
estimatedRatings = np.zeros(shape=(MAX_UID, MAX_PID), dtype=np.float16)
recomendRatings = np.zeros(shape=(MAX_UID,max_recommendation ), dtype=np.float16)
for userTest in uTest:
prod = U[userTest, :]*rightTerm
estimatedRatings[userTest, :] = prod.todense()
recomendRatings[userTest, :] = (-estimatedRatings[userTest, :]).argsort()[:max_recommendation]
return recomendRatings
# In[56]:
K=50
urm = data_sparse
MAX_PID = urm.shape[1]
MAX_UID = urm.shape[0]
U, S, Vt = compute_svd(urm, K)
# In[57]:
uTest = [4,5,6,7,8,873,23]
uTest_recommended_items = compute_estimated_matrix(urm, U, S, Vt, uTest, K, True)
# In[58]:
for user in uTest:
print("Recommendation for user with user id {}". format(user))
rank_value = 1
for i in uTest_recommended_items[user,0:10]:
song_details = small_set[small_set.so_index_value == i].drop_duplicates('so_index_value')[['title','artist_name']]
print("The number {} recommended song is {} BY {}".format(rank_value, list(song_details['title'])[0],list(song_details['artist_name'])[0]))
rank_value+=1
# In[59]:
uTest = [27513]
#Get estimated rating for test user
print("Predictied ratings:")
uTest_recommended_items = compute_estimated_matrix(urm, U, S, Vt, uTest, K, True)
# In[60]:
for user in uTest:
print("Recommendation for user with user id {}". format(user))
rank_value = 1
for i in uTest_recommended_items[user,0:10]:
song_details = small_set[small_set.so_index_value == i].drop_duplicates('so_index_value')[['title','artist_name']]
print("The number {} recommended song is {} BY {}".format(rank_value, list(song_details['title'])[0],list(song_details['artist_name'])[0]))
rank_value+=1
浙公网安备 33010602011771号