% Version 1.000
%
% Code provided by Ruslan Salakhutdinov and Geoff Hinton
%
% Permission is granted for anyone to copy, use, modify, or distribute this
% program and accompanying programs and documents for any purpose, provided
% this copyright notice is retained and prominently displayed, along with
% a note saying that the original programs are available from our
% web page.
% The programs and documents are distributed without any warranty, express or
% implied.  As the programs were written for research purposes only, they have
% not been tested to the degree that would be advisable in any important
% application.  All use of these programs is entirely at the user's own risk.

% Version 1.100
%
% Updated by Computational Cognitive Neuroscience Lab
% University of Padova
% ccnl.psy.unipd.it
%
% Implementation on Cluster using MPITB for Octave

function deeptrain(maxepoch)
	
	%MPI Init functions
	MPI_Init;
	[info rnk]=MPI_Comm_rank(MPI_COMM_WORLD);
	[info siz]=MPI_Comm_size(MPI_COMM_WORLD);

	%deep network parameters
	sizelayer=[30 60 120];
	numlayer=length(sizelayer);
	batchsize = 10;
	
    directory=''; % set the directory of data
    dataset='Artificial_Words';
    addpath(directory);


	%RBM parameters
	epsilonw      = 0.1;   % Learning rate for weights 
	epsilonvb     = 0.1;   % Learning rate for biases of visible units 
	epsilonhb     = 0.1;   % Learning rate for biases of hidden units 
	weightcost  = 0.0002;   
	initialmomentum  = 0.5;
	finalmomentum    = 0.9;

	% load training data
	if(rnk==0)
		[inputdata, targets] = load_data(directory,dataset);				% Load datafiles and constructs a big input/output matrix
        numdims=size(inputdata,2);                              % Input size
        tardims=size(targets,2);                                % Size of target encoding
        totnum =size(inputdata,1);								% totnum: total number of training patterns
        %rand('state',0);
        randomorder=randperm(totnum);
        numbatches=ceil(totnum/batchsize);						% Make batches of this size
        batchdata    = zeros(batchsize, numdims, numbatches);   % Prepare matrixes with batches. All data will be here.
        batchtargets = zeros(batchsize, tardims, numbatches);
        A = [batchsize, numdims, numbatches];					% Batch-matrix. NUMBATCHES will be parceled into more processors		    
        % Distribute data
        for b=1:numbatches										% Prepare input and output batch data
            batchdata(:,:,b)    = inputdata(randomorder(1+(b-1)*batchsize:b*batchsize), :);
            batchtargets(:,:,b) = targets(randomorder(1+(b-1)*batchsize:b*batchsize), :);
        end
    
        % Clear original data
        clear inputdata targets;
    
		tic();
    		% STORE PROJECT INFO  
    		  % STORE PROJECT INFO               
            Proj.Layers=[numdims sizelayer]; 
    		Proj.nLayers=numlayer;
            Proj.Err=zeros(maxepoch,Proj.nLayers);
    		Proj.nPatterns=totnum;
    		Proj.Batchsize=batchsize;
    		Proj.nBatches=numbatches;
    		Proj.nCPU=siz;
    		Proj.nEpoch=maxepoch;
	end	
	
	% Pretrains a deep networks.

	for layer=1:numlayer
		% run new RBM
		numhid=sizelayer(layer);
	    	rbm_par;

	   	if(rnk==0)
		        % save the RBM weights
			rbmWeights.layer=layer;
			rbmWeights.weight=vishid;
			rbmWeights.hidbiases=hidbiases;
			rbmWeights.visbiases=visbiases;
			WeightsLayer{1,layer}=rbmWeights;
		
			% update input data and parameters for next layer
			batchdata=batchposhidprobs;
			 A=size(batchdata);
             totnum=A(1)*A(3);
             numbatches=totnum/batchsize;		
	    	end
	end

	if(rnk==0)
    		% SAVE Proj and WeightsLayer
		Proj.Time=toc();
		fname = ['DeepNet_WORDS_' num2str(batchsize) '.mat'];
    		save('-mat-binary', fname, 'Proj', 'WeightsLayer');
	end 

	MPI_Finalize;
	MPITB_Clean;
end

