surpriseでレコメンドを試す

 user_id		movie_id	rating
196	242	3
186	302	3

# install surprise
!pip install surprise

# import library
import surprise as sur
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
#import networkx as nx
plt.style.use('ggplot')
# %matplotlib inline


# load datasets
url = 'http://files.grouplens.org/datasets/movielens/ml-100k/'

col_rat = ['user_id','movie_id','rating','unix_timestamp']
df_ratings = pd.read_csv(url+'u.data',
                       sep='\t', names=col_rat)

col_items = ['movie_id', 'movie_title' ,'release_date','video_release_date', 'IMDb_URL', 'unknown', 'Action', 'Adventure',
'Animation', 'Childrens', 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy',
'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci_Fi', 'Thriller', 'War', 'Western']
df_items = pd.read_csv(url+'u.item',
                   sep='|', names=col_items, encoding='latin-1')

# データ確認
df_ratings['rating'].value_counts(dropna=False)

# load_from_dfに必要、ratingの範囲を指定する
reader = sur.Reader(rating_scale=(1,5))

sur_data = sur.Dataset.load_from_df(df_ratings[['user_id','movie_id','rating']], reader)

sur_data.df.head(2)

# アルゴリズムとデータの定義

sur_svd = sur.SVD(random_state = 1,
              reg_all=0.2, # using some regularization
              n_epochs=20,
              n_factors=100) # number of latent factors to retain

trainset_full = sur_data.build_full_trainset()

# 学習
sur_svd.fit(trainset_full)

# 各手法で評価
trainsetfull_build = trainset_full.build_testset()
predictions_full = sur_svd.test(trainsetfull_build)

# 評価用（ここでは学習データを使う）
rmse = sur.accuracy.rmse(predictions_full)
mae = sur.accuracy.mae(predictions_full)

predictions_full[0:2]

# あとで比較するために値と、評価結果を格納したデータフレーム作成

df_pred = pd.DataFrame([(x.r_ui, x.est) for x in predictions_full], 
                       columns=['Rating','Predicted'])

# 学習データとテストデータを分けて学習させる
import random
raw_ratings = sur_data.raw_ratings
np.random.seed(0)

random.shuffle(raw_ratings)

# 9:1で分ける

threshold = int(0.9 * len(raw_ratings))
train_raw_ratings = raw_ratings[:threshold]
test_raw_ratings = raw_ratings[threshold:]

sur_data.raw_ratings = train_raw_ratings

len(sur_data.raw_ratings)

# cross valudationを行う
sur_svd = sur.SVD(random_state = 1)
cv_results = sur.model_selection.cross_validate(sur_svd, sur_data, measures=['RMSE','MAE'], cv=5)
pd.DataFrame(cv_results)

# gird searchをやってみる
param_grid = {'n_epochs': [15,20,25],
            'lr_all': [0.002, 0.02, 0.2, 2]}

grid_search = sur.model_selection.GridSearchCV(sur.SVD,
                   param_grid,
                   measures=['rmse'],
                   cv = 3,
                   refit = True)

grid_search.fit(sur_data)

# bestモデルを取り出す
sur_svd = grid_search.best_estimator['rmse']

# 学習用データで学習させる
trainset = sur_data.build_full_trainset()
sur_svd.fit(trainset)

# 評価
trainset_build = trainset.build_testset()
predictions_train = sur_svd.test(trainset_build)
print('Training score (rated items):')
sur.accuracy.rmse(predictions_train)

# test用データで評価
testset = sur_data.construct_testset(test_raw_ratings)
predictions_test = sur_svd.test(testset)
print('Test score (rated items)')
sur.accuracy.rmse(predictions_test)

# trainにてratingされていないデータを使って、予測
no_ratings = trainset.build_anti_testset()
predictions_no_ratings = sur_svd.test(no_ratings)
print('Test score (unrated items):')
sur.accuracy.rmse(predictions_no_ratings)

# 全結果について、サンプルを比較
print(predictions_train[0])
print(predictions_train[1])
print("")
print(predictions_test[0])
print(predictions_test[1])
print("")
print(predictions_no_ratings[0])
print(predictions_no_ratings[1])

# 評価
trainset_build = trainset.build_testset()
predictions_train = sur_svd.test(trainset_build)
print('Training score (rated items):')
sur.accuracy.rmse(predictions_train)

# test用データで評価
testset = sur_data.construct_testset(test_raw_ratings)
predictions_test = sur_svd.test(testset)
print('Test score (rated items)')
sur.accuracy.rmse(predictions_test)

# trainにてratingされていないデータを使って、予測
no_ratings = trainset.build_anti_testset()
predictions_no_ratings = sur_svd.test(no_ratings)
print('Test score (unrated items):')
sur.accuracy.rmse(predictions_no_ratings)


Training score (rated items):
RMSE: 0.8367
Test score (rated items)
RMSE: 0.9428
Test score (unrated items):
RMSE: 0.5167

# 全結果について、サンプルを比較
print(predictions_train[0])
print(predictions_train[1])
print("")
print(predictions_test[0])
print(predictions_test[1])
print("")
print(predictions_no_ratings[0])
print(predictions_no_ratings[1])

user: 903        item: 46         r_ui = 4.00   est = 3.66   {'was_impossible': False}
user: 903        item: 333        r_ui = 4.00   est = 3.56   {'was_impossible': False}

user: 663        item: 259        r_ui = 2.00   est = 2.83   {'was_impossible': False}
user: 405        item: 1119       r_ui = 3.00   est = 1.75   {'was_impossible': False}

user: 903        item: 205        r_ui = 3.53   est = 4.21   {'was_impossible': False}
user: 903        item: 219        r_ui = 3.53   est = 3.50   {'was_impossible': False}