%% Advanced Human Activity Classification
% This script extends the example from ClassificationAnalysis.m to evaluate
% other clustering methods. In particular, it addresses:
%
% * Neural Networks
% * K Nearest Neighbors
% * Naive Bayes
% * Linear Discriminant Analysis
% * Decision Trees
% * Bagged Decision Trees
% * Random Forests
% * Support Vector Machines
% * Ensembles of Learners
% * Evaluating and Comparing Classification Solutions
% 
%  The dataset is courtesy:
%  Davide Anguita, Alessandro Ghio, Luca Oneto, Xavier Parra and Jorge L.
%  Reyes-Ortiz. Human Activity Recognition on Smartphones using a
%  Multiclass Hardware-Friendly Support Vector Machine. International
%  Workshop of Ambient Assisted Living (IWAAL 2012). Vitoria-Gasteiz,
%  Spain. Dec 2012

%% Load Data
% The data is saved in a MAT file. Each row of the variable Xtrain
% represents one person/instance. The corresponding row of ytrain contains
% the label of the activity being performed
load Data\HAData

% Set random number generator so results are repeatable
rng(1)

%% Feature Importance
% The complexity of the model and its likelihood to overfit can be reduced
% by reducing the number of features included. A metric like a paired t
% test can be used to identify the top features that help separate classes.
% We can then investigate whether a model trained using only these features
% can perform comparably with a model trained on the entire dataset

[featIdx, pair] = featureScore(Xtrain, ytrain, 5);

redF = unique(featIdx(:));
features = features(redF);
Xtrain = Xtrain(:,redF);
Xtest = Xtest(:,redF);

%% Open Pool of Parallel Workers
% If Parallel Computing Toolbox is available, the computation will be
% distributed to 2 workers for speeding up the evaluation.

if isempty(gcp('nocreate'))
    parpool
end

%% Neural Networks
% Neural Network Toolbox supports supervised learning with feedforward,
% radial basis, and dynamic networks. It supports both classification and
% regression algorithms. It also supports unsupervised learning with
% self-organizing maps and competitive layers.
% 
% One can make use of the interactive tools to setup, train and validate a
% neural network. It is then possible to auto-generate the code for the
% purpose of automation. In this example, the auto-generated code has been
% updated to utilize a parallel pool of workers, if available. This is
% achieved by simply setting the _useParallel_ flag while making a call to
% |train|.
% 
%   [net,~] = train(net,inputs,targets,'useParallel','yes');
% 
% If a GPU is available, it may be utilized by setting the _useGPU_ flag.
% 
% The trained network is used to make a prediction on the test data and
% confusion matrix is generated for comparison with other techniques.

net = patternnet(20); % Initialize network

% Setup Division of Data for Training, Validation, Testing
net.divideParam.trainRatio = 70/100;
net.divideParam.valRatio = 15/100;
net.divideParam.testRatio = 15/100;

% Train the Network
net = train(net, Xtrain', dummyvar(ytrain)', 'UseParallel','yes');

scoretest = net(Xtest')';
[~, Y_nn] = max(scoretest,[],2);
C_nn = confusionmat(ytest, Y_nn);
dispConfusion(C_nn, 'Neural Net', labels);
MC_nn = 100 - sum(sum(diag(C_nn)))/sum(sum(C_nn))*100;
fprintf('Overall misclassification rate on test set: %0.2f%%\n',MC_nn)

%% Classification Using Nearest Neighbors
% Categorizing query points based on their distance to points in a training
% dataset can be a simple yet effective way of classifying new points.
% Various distance metrics such as euclidean, correlation, hamming,
% mahalonobis or your own distance metric may be used.

% Train the classifier
knn = fitcknn(Xtrain,ytrain,'Distance','seuclidean','NumNeighbors',10,...
    'PredictorNames',features);

% Make a prediction for the test set
[Y_knn, Yscore_knn] = predict(knn,Xtest);

% Compute the confusion matrix
C_knn = confusionmat(ytest,Y_knn);
dispConfusion(C_knn, 'K Nearest Neighbors', labels);
MC_knn = 100 - sum(sum(diag(C_knn)))/sum(sum(C_knn))*100;
fprintf('Overall misclassification rate on test set: %0.2f%%\n',MC_knn)

%% Naive Bayes Classification
% Naive Bayes classification is based on estimating P(X|Y), the probability
% or probability density of features X given class Y. The Naive Bayes
% classification object provides support for normal (Gaussian), kernel,
% multinomial, and multivariate multinomial distributions

% Train the classifier
Nb = fitcnb(Xtrain,ytrain);

% Make a prediction for the test set
Y_nb = predict(Nb,Xtest);

% Compute the confusion matrix
C_nb = confusionmat(ytest,Y_nb);
% Examine the confusion matrix for each class as a percentage of the true class
dispConfusion(C_nb, 'Naive Bayes', labels);
MC_nb = 100 - sum(sum(diag(C_nb)))/sum(sum(C_nb))*100;
fprintf('Overall misclassification rate on test set: %0.2f%%\n',MC_nb)

%% Linear Discriminant Analysis
% Discriminant analysis is a classification method. It assumes that
% different classes generate data based on different Gaussian
% distributions. Linear discriminant analysis is also known as the Fisher
% discriminant. 
da = fitcdiscr(Xtrain,ytrain,'PredictorNames',features);

% Make a prediction for the test set
[Y_lda, Yscore_lda] = predict(da,Xtest); 

% Compute the confusion matrix
C_lda = confusionmat(ytest,Y_lda);
dispConfusion(C_lda, 'LDA', labels);
MC_lda = 100 - sum(sum(diag(C_lda)))/sum(sum(C_lda))*100;
fprintf('Overall misclassification rate on test set: %0.2f%%\n',MC_lda)

%% Decision Trees
% Classification trees and regression trees are two kinds of decision
% trees. A decision tree is a flow-chart like structure in which internal
% node represents test on an attribute, each branch represents outcome of
% test and each leaf node represents a response (decision taken after
% computing all attributes). Classification trees give responses that are
% categorical, such as 'true' or 'false'. Regression trees give numeric
% responses.

% Train the classifier
t = fitctree(Xtrain,ytrain,'PredictorNames',features);

% Make a prediction for the test set
Y_t = predict(t,Xtest);

% Compute the confusion matrix
C_t = confusionmat(ytest,Y_t);
% Examine the confusion matrix for each class as a percentage of the true class
C_t = dispConfusion(C_t,'Classification Tree',labels);
MC_t = 100 - sum(sum(diag(C_t)))/sum(sum(C_t))*100;
fprintf('Overall misclassification rate on test set: %0.2f%%\n',MC_t)

% view the tree
view(t,'mode','graph')

%% Bagged Decision Trees
% A single decision tree can be a very easy model to interpret, especially
% if pruned to a small size. However single trees tend to be poor
% predictors. Several techniques exist for combining many trees together to
% improve predictions. 
%
% The first is called Bagging, which stands for Bootstrap Aggregation. A
% series of trees are grown from independent, bootstrapped samples of the
% input data. To compute prediction for the ensemble of trees, TreeBagger
% takes an average of predictions from individual trees (for regression) or
% takes votes from individual trees (for classification). Ensemble
% techniques such as bagging combine many weak learners to produce a strong
% learner. 

% Train the classifier
ops = statset('UseParallel',true);
tb = TreeBagger(50,Xtrain,ytrain,'method','classification','Options',ops,...
    'NVarToSample','all','PredictorNames',features);

% Make a prediction for the test set
[Y_tb, Yscore_tb] = predict(tb,Xtest);
Y_tb=str2num(cell2mat(Y_tb)); %#ok
% Compute the confusion matrix
C_tb = confusionmat(ytest,Y_tb);
% Examine the confusion matrix for each class as a percentage of the true class
dispConfusion(C_tb, 'Bagged Trees',labels);
MC_tb = 100 - sum(sum(diag(C_tb)))/sum(sum(C_tb))*100;
fprintf('Overall misclassification rate on test set: %0.2f%%\n',MC_tb)

%% Random Forest
% Random forests are an extension of bagged decision trees. Random forests
% only make available a random subset of the predictors for each node in
% the decision tree to pick from. If a single predictor is much stronger
% than the others, then it would normally be used very early and often in
% the tree branching even on indepedently bootstrapped input samples. This
% correlates the trees and makes them vulnerable to overfitting. By
% randomizing the predictors available at each node, random forests help to
% decorrelate the trees. However if the trees are already not correlated
% then random forests could reduce accuracy by not allowing the tree to use
% the best possible predictors.

% Train the classifier
rf = TreeBagger(50,Xtrain,ytrain,'method','classification','Options',ops,...
    'PredictorNames',features);

% Make a prediction for the test set
[Y_rf, Yscore_rf] = predict(rf,Xtest);
Y_rf=str2num(cell2mat(Y_rf)); %#ok
% Compute the confusion matrix
C_rf = confusionmat(ytest,Y_rf);
% Examine the confusion matrix for each class as a percentage of the true class
dispConfusion(C_rf, 'Random Forest',labels);
MC_rf = 100 - sum(sum(diag(C_rf)))/sum(sum(C_rf))*100;
fprintf('Overall misclassification rate on test set: %0.2f%%\n',MC_rf)

%% Support Vector Machines
% Support vector machines (SVMs) try to find the hyperplane that best
% separates the classes. However a single SVM can only work for binary
% outputs (i.e. yes/no, true/false, etc.) not for multiclass problems.
% Multiple SVMs can be combined to handle multiclass problems. New in
% R2014b is the fitcecoc function, which can automatically extend any
% binary classifier, such as SVMs, to a multiclass problem.

% Train the classifier
svmmdl = templateSVM('KernelFunction','rbf');
mcsvm = fitcecoc(Xtrain,ytrain,'Coding','onevsall','Learners',svmmdl);

% Make a prediction for the test set
[Y_mcsvm, Yscore_mcsvm] = predict(mcsvm,Xtest);
% Compute the confusion matrix
C_mcsvm = confusionmat(ytest,Y_mcsvm);
% Examine the confusion matrix for each class as a percentage of the true class
dispConfusion(C_mcsvm, 'Support Vector Machine',labels);
MC_mcsvm = 100 - sum(sum(diag(C_mcsvm)))/sum(sum(C_mcsvm))*100;
fprintf('Overall misclassification rate on test set: %0.2f%%\n',MC_mcsvm)

%% Combine Classifiers
% There are different heuristic methods that can be used to combine
% classifiers. Formal ensemble learning methods train a group of
% classifiers in a systematic way to improve their combined performance.
% Bagging is one method as is Boosting. The _fitensemble_ method provides a
% number of these techniques. A simpler method that is often used in
% practice is to combine the outputs of independently trained classifiers
% through either a voting scheme or by combining predictor scores.

% Combine four best techniques through simple voting scheme
comb = [Y_nn,Y_knn,Y_lda,Y_mcsvm];
Y_mode = mode(comb,2);
% Compute the confusion matrix
C_mode = confusionmat(ytest,Y_mode);
% Examine the confusion matrix for each class as a percentage of the true class
dispConfusion(C_mode, 'Voting Scheme',labels);
MC_mode = 100 - sum(sum(diag(C_mode)))/sum(sum(C_mode))*100;
fprintf('Overall misclassification rate on test set: %0.2f%%\n',MC_mode)

%% Compare Models with Misclassification Rates
% There are many different ways to compare the accuracy and effectiveness
% of different classifiers. Here we look simply at the overall
% misclassification rate, and see that voting scheme perfomed best followed
% by the Support Vector Machines.

methods = {'Neural Net','KNN','NBayes','LDA','Tree','Bagged','RF','SVM','Voting'};
MC_comb = [MC_nn,MC_knn,MC_nb,MC_lda,MC_t,MC_tb,MC_rf,MC_mcsvm,MC_mode];
bar(MC_comb);set(gca,'YGrid', 'on','XTickLabel',methods,'XTickLabelRotation',-45);
ylabel('Overall Misclassification Rate (percent)')