This document compares several machine learning algorithms for a binary classification problem using a census dataset: 1. It builds logistic regression, decision tree, random forest, and boosted tree models on a 80% training set and evaluates their performance on a 10% test set. 2. Tuning is performed on decision tree and random forest models which improves their AUC. 3. The best performing models are boosted trees with an AUC of 0.922 and logistic regression with an AUC of 0.91, as evaluated on the held-out test set.

1. library(ROCR)
library(rpart)
library(rpart.plot)
#census.csvdataanalysis
#http://archive.ics.uci.edu/ml/datasets/Adult
cdata=read.csv("census.csv")
head(cdata,10)
summary(cdata)
str(cdata)
pairs(cdata[1:200,],col=cdata$over50k)
hist((cdata$capitalgain))
hist(log(cdata$capitalgain))
#prepare train80% test10% and validation10% sets
set.seed(2017)
spl = sample.split(cdata$over50k,SplitRatio=0.8)
ctrain = subset(cdata,spl==TRUE)
ctest= subset(cdata,spl==FALSE)
spl = sample.split(ctest$over50k,SplitRatio=0.5)
cvalidation=subset(ctest,spl==TRUE)
ctest= subset(ctest,spl==FALSE)

2. #buildfirstmodel logit
#buildmodel
set.seed(2017)
clogit= glm( over50k ~ . , family="binomial",data= ctrain)
summary(clogit)
#prediction andaccuracy on test
predictTest=predict(clogit,newdata=ctest,type = "response")
t(table(ctest$over50k,predictTest>= 0.5))
(2262+484)/sum(table(ctest$over50k,predictTest>=0.5))
#most frequentprediction
table(cdata$over50k)
24283/(24283+7695)
#AUC
ROCRpred= prediction(predictTest,ctest$over50k)
as.numeric(performance(ROCRpred,"auc")@y.values)
###################################################
# summarylogitaccuracy fort=0.5 0.859 auc=0.91 #
###################################################
#decisiontree
#buildmodel

3. set.seed(2017)
ctree = rpart( over50k ~ . , method="class",data= ctrain,minbucket=100)
prp(ctree)
#buildprediction
predictTest=predict(ctree,newdata=ctest,type = "class")
t(table(ctest$over50k,predictTest))
(2300+405)/sum(table(ctest$over50k,predictTest))
#accuracy=0.846
#rocr auc
predictTest=predict(ctree,newdata=ctest)
predictTest=predictTest[,2]
ROCRpred= prediction(predictTest,ctest$over50k)
as.numeric(performance(ROCRpred,"auc")@y.values)
##################################################################
# summarytree minbucket=100 accuracy fort=0.5 0.846 auc=0.855 #
##################################################################
#parameterstuningdecisiontree
set.seed(2017)
cartGrid = expand.grid( .cp= seq(0.001,0.1,0.001))
fitControl =trainControl( method="cv",number= 10 )
rezCV=train( over50k~ . , data = ctrain, method= "rpart",trControl = fitControl,tuneGrid=cartGrid )
rezCV
cvmod= rpart(over50k~.,data=ctrain,method="class",cp=0.001)

4. prp(cvmod)
#accuracy
predictTest=predict(cvmod,newdata=ctest,type="class")
table(ctest$over50k,predictTest)
(2329+448)/sum(table(ctest$over50k,predictTest))
#accuracy 0.868
#rocr auc
predictTest=predict(cvmod,newdata=ctest)
predictTest=predictTest[,2]
ROCRpred= prediction(predictTest,ctest$over50k)
as.numeric(performance(ROCRpred,"auc")@y.values)
#auc 0.883
##################################################################
# summarydecisiontree cvaccuracy for t=0.5 0.868 auc=0.883 #
##################################################################
#random forest
set.seed(2017)
#buildmodel
crf = randomForest(over50k~ . , data = ctrain)
#predicton test

5. predictTest=predict(crf,newdata=ctest)
#accuracy calculation
t(table(ctest$over50k,predictTest))
(2426+214)/(sum(table(ctest$over50k,predictTest)))
#AUC calculation
predictTest=predict(crf,newdata=ctest,type="prob")
ROCRpred= prediction(predictTest[,2],ctest$over50k)
as.numeric(performance(ROCRpred,"auc")@y.values)
##################################################################
# auc=0.889 acc=0.826 #
##################################################################
#parameterstuningrandomforest
# make take a lot of time
metric<- "Accuracy"
control <- trainControl(method="cv",number=10,search="grid")
tunegrid<- expand.grid(.mtry=c(sqrt(ncol(ctrain))))
modellist<- list()
for (ntree inc(700,1000, 1200, 1400 )) {
for(mtry in c(2,3,4,5,6,7,9)) {
tunegrid= expand.grid(.mtry=mtry)
set.seed(2017)
fit<- train(over50k~.,data=ctrain[1:5000,],method="rf",metric=metric,tuneGrid=tunegrid,
trControl=control,ntree=ntree)

6. key<- toString(ntree*10000+mtry)
modellist[[key]]<- fit
print(c(ntree,mtry))
print(fit)
}
}
#bestrf
set.seed(2017)
crf = randomForest(over50k~ . , data = ctrain,ntree=1400,mtry=2)
predictTest=predict(crf,newdata=ctest)
t(table(ctest$over50k,predictTest))
(2427+217)/(sum(table(ctest$over50k,predictTest)))
predictTest=predict(crf,newdata=ctest,type="prob")
ROCRpred= prediction(predictTest[,2],ctest$over50k)
as.numeric(performance(ROCRpred,"auc")@y.values)
##################################################################
# auc=0.901 accuracy=0.827 #
##################################################################
#################################################################
#winnerlogit.validationset

7. set.seed(2017)
clogit= glm( over50k ~ . , family="binomial",data= ctrain)
summary(clogit)
#predictionandaccuracy on test
predictTest=predict(clogit,newdata=cvalidation,type ="response")
t(table(cvalidation$over50k,predictTest>= 0.5))
(2256+460)/sum(table(cvalidation$over50k,predictTest>=0.5))
#AUC
ROCRpred= prediction(predictTest,cvalidation$over50k)
as.numeric(performance(ROCRpred,"auc")@y.values)
################################
#baggingmtry=numberof features
set.seed(2017)
#buildmodel
crf = randomForest(over50k~ . , data = ctrain,mtry=12)
#predicton test
predictTest=predict(crf,newdata=ctest)
#accuracy calculation
t(table(ctest$over50k,predictTest))
(2398+214)/(sum(table(ctest$over50k,predictTest)))

8. #AUC calculation
predictTest=predict(crf,newdata=ctest,type="prob")
ROCRpred= prediction(predictTest[,2],ctest$over50k)
as.numeric(performance(ROCRpred,"auc")@y.values)
importance(crf)
##################################################################
# auc=0.848 acc=0.817 #
##################################################################
#####################
#boosting
install.packages("gbm")
library(gbm)
#transformdependentvariable to01 variable
ctrain$over50k=as.integer(ctrain$over50k)-1
ctest$over50k=as.integer(ctest$over50k)-1
#buildmodel
set.seed(2017)
cboost= gbm(over50k~ . , data = ctrain,distribution="bernoulli",n.trees=5000,interaction.depth=4)
#prediction
predictTest=predict(cboost,newdata=ctest,n.trees=5000,type='response')

9. #qualityof prediction
t(table(ctest$over50k,predictTest>0.5))
(2308+475)/(sum(table(ctest$over50k,predictTest)))
ROCRpred= prediction(predictTest,ctest$over50k)
as.numeric(performance(ROCRpred,"auc")@y.values)
##################################################################
# auc=0.922 acc=0.8700 #
##################################################################
# auc
#boosting 0.922
#logit 0.91
#tuningrand forest0.901
#random forest 0.889
#tuningtree 0.883
#decisiontree 0.855
#baging 0.848