001package deepnetts.net;
002
003import deepnetts.net.layers.AbstractLayer;
004import deepnetts.net.layers.activation.ActivationType;
005import deepnetts.net.layers.ConvolutionalLayer;
006import deepnetts.net.layers.FullyConnectedLayer;
007import deepnetts.net.layers.InputLayer;
008import deepnetts.net.layers.MaxPoolingLayer;
009import deepnetts.net.layers.OutputLayer;
010import deepnetts.net.layers.SoftmaxOutputLayer;
011import deepnetts.net.loss.BinaryCrossEntropyLoss;
012import deepnetts.net.loss.CrossEntropyLoss;
013import deepnetts.net.loss.LossFunction;
014import deepnetts.net.loss.LossType;
015import deepnetts.net.loss.MeanSquaredErrorLoss;
016import deepnetts.net.train.BackpropagationTrainer;
017import deepnetts.util.DeepNettsException;
018import deepnetts.util.ImagePreprocessing;
019import deepnetts.util.RandomGenerator;
020import deepnetts.util.Tensor;
021import java.io.Serializable;
022import java.lang.reflect.InvocationTargetException;
023import java.util.ArrayList;
024import java.util.List;
025import java.util.logging.Level;
026import java.util.logging.Logger;
027
028/**
029 * Convolutional neural network is an extension of feed forward network, which can
030 * include  2D and 3D adaptive preprocessing layers (Convolutional and MaxPooling layer),
031 * which specialized to learn to recognize features in images. Images are fed as 3-dimensional tensors.
032 * Although primary used for images, they can also be applied to other types of
033 * multidimensional problems.
034 *
035 * @see ConvolutionalLayer
036 * @see MaxPoolingLayer
037 * @see BackpropagationTrainer
038 *
039 * @author Zoran Sevarac
040 */
041public class ConvolutionalNetwork extends NeuralNetwork<BackpropagationTrainer> implements Serializable {
042    
043    private ConvolutionalNetwork() {
044        super();
045        setTrainer(new BackpropagationTrainer(this));
046    }
047
048    // override set input to apply image preprocessing
049    @Override
050    public void setInput(Tensor input) {
051        if (getPreprocessing() !=null) getPreprocessing().apply(input); // do this only in inference mode! i to ne iz ide nego kad ucitava sliku spolja za buffered imagr! Image set vec sve uradi sto treba
052        super.setInput(input);
053    }
054    
055//    public void setInput(BufferedImage input) { // ne bi trebalo jer secem android
056//        
057//    }
058    
059    // set input za bufferedImage??? a sta kad ga nema na androidu???
060    
061    public static ConvolutionalNetwork.Builder builder() {
062        return new Builder();
063    }
064
065    /**
066     * Builder for convolutional neural network
067     */
068    public static class Builder {
069
070        private final ConvolutionalNetwork neuralNet = new ConvolutionalNetwork();
071
072        private ActivationType defaultActivationType = ActivationType.RELU;
073        private Class<CrossEntropyLoss> defaultLossFunction = CrossEntropyLoss.class;
074        private boolean setDefaultActivation = false;
075
076        /**
077         * Input layer with specified width and height, and 3 channels by
078         * default.
079         *
080         * @param width
081         * @param height
082         * @return
083         */
084        public Builder addInputLayer(int width, int height) {
085            InputLayer inLayer = new InputLayer(width, height, 3);
086            neuralNet.setInputLayer(inLayer);
087            neuralNet.addLayer(inLayer);
088
089            return this;
090        }
091
092        /**
093         * Input layer with specified width, height and number of channels (depth).
094         *
095         * @param width
096         * @param height
097         * @param channels
098         * @return
099         */
100        public Builder addInputLayer(int width, int height, int channels) {
101            InputLayer inLayer = new InputLayer(width, height, channels);
102            neuralNet.setInputLayer(inLayer);
103            neuralNet.addLayer(inLayer);
104
105            return this;
106        }
107
108        public Builder addFullyConnectedLayer(int width) {
109            FullyConnectedLayer layer = new FullyConnectedLayer(width);
110            neuralNet.addLayer(layer);
111            return this;
112        }
113
114        /**
115         * Add dense layer with specified width and activation function.
116         * In dense layer each neuron is connected to all outputs from previous layer.
117         * @param width layer width
118         * @param activationType type of activation function
119         * @return current builder instance
120         * @see ActivationType
121         */
122        public Builder addFullyConnectedLayer(int width, ActivationType activationType) {
123            FullyConnectedLayer layer = new FullyConnectedLayer(width, activationType);
124            neuralNet.addLayer(layer);
125            return this;
126        }
127
128        public Builder addOutputLayer(int width, Class<? extends OutputLayer> clazz) { // ActivationType.SOFTMAX
129            try {
130                OutputLayer outputLayer = clazz.getDeclaredConstructor(Integer.TYPE).newInstance(width);
131                neuralNet.addLayer(outputLayer);
132                neuralNet.setOutputLayer(outputLayer);
133            } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | SecurityException | IllegalArgumentException | InvocationTargetException ex) {
134                Logger.getLogger(ConvolutionalNetwork.class.getName()).log(Level.SEVERE, null, ex);
135            }
136
137            return this;
138        }
139
140        public Builder addOutputLayer(int width, ActivationType activationType) {
141            OutputLayer outputLayer = null;
142            if (activationType.equals(ActivationType.SOFTMAX)) {
143                outputLayer = new SoftmaxOutputLayer(width);
144            } else {
145                outputLayer = new OutputLayer(width);
146                outputLayer.setActivationType(activationType);
147            }
148
149            neuralNet.setOutputLayer(outputLayer);
150            neuralNet.addLayer(outputLayer);
151
152            return this;
153        }
154
155        // TODO:
156        // Channels - first everywhere
157        // Filter.size(3).stride(2), Filters.size(3,3) Filters.size(5) Filters.size(5,5)
158        //  size, width, height, stride, padding?
159        // channels
160        // channels, activation
161        // channels, filter
162        // channels, filter, activation
163        
164        
165        public Builder addConvolutionalLayer(int filterSize, int channels) {
166            ConvolutionalLayer convolutionalLayer = new ConvolutionalLayer(filterSize, filterSize, channels, defaultActivationType);
167            neuralNet.addLayer(convolutionalLayer);
168            return this;
169        }
170
171        public Builder addConvolutionalLayer(int filterSize, int channels, ActivationType activationType) {
172            ConvolutionalLayer convolutionalLayer = new ConvolutionalLayer(filterSize, filterSize, channels, activationType);
173            neuralNet.addLayer(convolutionalLayer);
174            return this;
175        }
176
177        public Builder addConvolutionalLayer(int filterWidth, int filterHeight, int channels) {
178            ConvolutionalLayer convolutionalLayer = new ConvolutionalLayer(filterWidth, filterHeight, channels, defaultActivationType);
179            neuralNet.addLayer(convolutionalLayer);
180            return this;
181        }
182
183        // size je static i ona instancira filter size, a stride je obicna metoda i ona se ne moze pozvati u statickom kontekstu. Neka klasa bude filter
184        // TODO: numChannels, Filter.size(3, 3).stride(1)       promeni i ovde i u CE, i svim primerima
185        public Builder addConvolutionalLayer(int filterWidth, int filterHeight, int channels, int stride) {
186            ConvolutionalLayer convolutionalLayer = new ConvolutionalLayer(filterWidth, filterHeight, stride, channels, defaultActivationType);
187            neuralNet.addLayer(convolutionalLayer);
188            return this;
189        }
190
191        public Builder addConvolutionalLayer(int filterWidth, int filterHeight, int channels, ActivationType activationType) {
192            ConvolutionalLayer convolutionalLayer = new ConvolutionalLayer(filterWidth, filterHeight, channels, activationType);
193            neuralNet.addLayer(convolutionalLayer);
194            return this;
195        }
196
197        public Builder addConvolutionalLayer(int filterWidth, int filterHeight, int channels, int stride, ActivationType activationType) {
198            ConvolutionalLayer convolutionalLayer = new ConvolutionalLayer(filterWidth, filterHeight, channels, stride, activationType);
199            neuralNet.addLayer(convolutionalLayer);
200            return this;
201        }
202
203        public Builder addMaxPoolingLayer(int filterSize, int stride) {
204            MaxPoolingLayer poolingLayer = new MaxPoolingLayer(filterSize, filterSize, stride);
205            neuralNet.addLayer(poolingLayer);
206            return this;
207        }
208
209        public Builder addMaxPoolingLayer(int filterWidth, int filterHeight, int stride) {
210            MaxPoolingLayer poolingLayer = new MaxPoolingLayer(filterWidth, filterHeight, stride);
211            neuralNet.addLayer(poolingLayer);
212            return this;
213        }
214
215        public Builder hiddenActivationFunction(ActivationType activationType) {
216            this.defaultActivationType = activationType;
217            setDefaultActivation = true;
218            return this;
219        }
220
221        public Builder lossFunction(Class<? extends LossFunction> clazz) {
222            try {
223                LossFunction loss = clazz.getDeclaredConstructor(NeuralNetwork.class).newInstance(neuralNet);
224                neuralNet.setLossFunction(loss);
225            } catch (NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) {
226                Logger.getLogger(ConvolutionalNetwork.class.getName()).log(Level.SEVERE, null, ex);
227            }
228
229            return this;
230        }
231
232        public Builder lossFunction(LossType lossType) {
233            LossFunction loss = null;
234            switch (lossType) {
235                case MEAN_SQUARED_ERROR:
236                    loss = new MeanSquaredErrorLoss(neuralNet);
237                    break;
238                case CROSS_ENTROPY:
239                    if (neuralNet.getOutputLayer().getWidth() == 1) {
240                        if (neuralNet.getOutputLayer().getActivationType() != ActivationType.SIGMOID )
241                            throw new DeepNettsException("Illegal combination of activation and loss functions (Sigmoid activation must be used with Cross Entropy Loss)");
242                        loss = new BinaryCrossEntropyLoss(neuralNet);
243                    } else {
244                        if (neuralNet.getOutputLayer().getActivationType() != ActivationType.SOFTMAX )
245                         //   throw new DeepNettsException("Illegal combination of activation and loss functions (Softmax activation must be used with Cross Entropy Loss)");
246                        loss = new CrossEntropyLoss(neuralNet);
247                    }
248                    break;
249            }
250            neuralNet.setLossFunction(loss);
251
252            return this;
253        }
254
255        public Builder randomSeed(long seed) {
256            RandomGenerator.getDefault().initSeed(seed);
257            return this;
258        }
259
260        public ConvolutionalNetwork build() {
261            // connect and init layers, weights matrices etc.
262            AbstractLayer prevLayer = null;
263
264            for (int i = 0; i < neuralNet.getLayers().size(); i++) {
265                AbstractLayer layer = neuralNet.getLayers().get(i);
266                 if (setDefaultActivation && !(layer instanceof InputLayer) && !(layer instanceof OutputLayer)) { // ne za izlazni layer
267                    layer.setActivationType(defaultActivationType); // ali ovo ne treba ovako!!! ako je vec nesto setovano onda nemoj to d agazis
268                }
269                layer.setPrevLayer(prevLayer);
270                if (prevLayer != null) {
271                    prevLayer.setNextlayer(layer);
272                }
273                prevLayer = layer; // current layer becomes prev layer in next iteration
274            }
275
276            // init all layers
277            neuralNet.getLayers().forEach((layer) -> {
278                layer.init();
279            });
280
281            // if loss is not set use default loss function
282            if (neuralNet.getLossFunction() == null) {
283                Builder.this.lossFunction(defaultLossFunction);
284            }
285
286            return neuralNet;
287        }
288    }
289
290    /**
291     * Returns all weights from this network as list of strings.
292     * TODO; for convolutional layer get filter weights
293     *
294     * @return
295     */
296    public List<String> getWeights() {
297        List weightsList = new ArrayList();
298        for (AbstractLayer layer : getLayers()) {
299            if (layer instanceof ConvolutionalLayer) {
300                Tensor filters = ((ConvolutionalLayer)layer).getFilters();
301                //String filterStr = Tensor.valuesAsString(filters);
302                String filterStr = filters.toString();
303                weightsList.add(filterStr);
304            } else {
305                weightsList.add(layer.getDeltaWeights().toString());
306            }
307        }
308        return weightsList;
309    }
310
311    public void setWeights(List<String> weights) {
312        int weightsIdx=0;
313
314        for (int layerIdx = 1; layerIdx < getLayers().size(); layerIdx++) {
315            AbstractLayer layer = getLayers().get(layerIdx);
316            if (layer instanceof ConvolutionalLayer) {
317                ((ConvolutionalLayer)layer).setFilters(weights.get(weightsIdx));
318                weightsIdx++;
319            } else if (layer instanceof FullyConnectedLayer || layer instanceof OutputLayer) {
320                layer.setWeights(weights.get(weightsIdx));
321                weightsIdx++;
322            }
323        }
324    }
325
326
327
328
329    public List<String> getDeltaWeights() {
330        List weightsList = new ArrayList();
331        for (AbstractLayer layer : getLayers()) {
332                weightsList.add(layer.getDeltaWeights().toString());
333        }
334        return weightsList;
335    }
336
337    // returns outputs for all layers
338    public List<String> getAllOutputs() {
339        List outputsList = new ArrayList();
340        for (AbstractLayer layer : getLayers()) {
341            outputsList.add(layer.getOutputs());
342        }
343        return outputsList;
344    }
345
346}