Train on 23000 samples, validate on 2000 samples
Epoch 1/12
23000/23000 [==============================] - 3s - loss: 0.5294 - acc: 0.7890 - val_loss: 0.1734 - val_acc: 0.9355
Epoch 2/12
23000/23000 [==============================] - 3s - loss: 0.2078 - acc: 0.9198 - val_loss: 0.1083 - val_acc: 0.9635
Epoch 3/12
23000/23000 [==============================] - 3s - loss: 0.1551 - acc: 0.9420 - val_loss: 0.0833 - val_acc: 0.9690
Epoch 4/12
23000/23000 [==============================] - 3s - loss: 0.1266 - acc: 0.9532 - val_loss: 0.0742 - val_acc: 0.9725
Epoch 5/12
23000/23000 [==============================] - 3s - loss: 0.1144 - acc: 0.9583 - val_loss: 0.0678 - val_acc: 0.9740
Epoch 6/12
23000/23000 [==============================] - 3s - loss: 0.1094 - acc: 0.9610 - val_loss: 0.0630 - val_acc: 0.9735
Epoch 7/12
23000/23000 [==============================] - 3s - loss: 0.1001 - acc: 0.9641 - val_loss: 0.0609 - val_acc: 0.9755
Epoch 8/12
23000/23000 [==============================] - 3s - loss: 0.0984 - acc: 0.9658 - val_loss: 0.0595 - val_acc: 0.9755
Epoch 9/12
23000/23000 [==============================] - 3s - loss: 0.0935 - acc: 0.9688 - val_loss: 0.0584 - val_acc: 0.9770
Epoch 10/12
23000/23000 [==============================] - 3s - loss: 0.0948 - acc: 0.9662 - val_loss: 0.0577 - val_acc: 0.9780
Epoch 11/12
23000/23000 [==============================] - 3s - loss: 0.0977 - acc: 0.9656 - val_loss: 0.0570 - val_acc: 0.9790
Epoch 12/12
23000/23000 [==============================] - 3s - loss: 0.0857 - acc: 0.9693 - val_loss: 0.0564 - val_acc: 0.9795
Train on 23000 samples, validate on 2000 samples
Epoch 1/1
23000/23000 [==============================] - 3s - loss: 0.0845 - acc: 0.9697 - val_loss: 0.0559 - val_acc: 0.9800
/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/layers/core.py:622: UserWarning: output_shape argument not specified for layer lambda_2 and cannot be automatically inferred with the Theano backend. Defaulting to output shape (None, 3, 224, 224) (same as input shape). If the expected output shape is different, specify it via the output_shape argument.
.format(self.name, input_shape))
Train on 23000 samples, validate on 2000 samples
Epoch 1/2
23000/23000 [==============================] - 40s - loss: 0.0816 - acc: 0.9727 - val_loss: 0.0451 - val_acc: 0.9850
Epoch 2/2
23000/23000 [==============================] - 40s - loss: 0.0369 - acc: 0.9872 - val_loss: 0.0503 - val_acc: 0.9860
Problem occurred during compilation with the command line below:
/usr/bin/g++ -shared -g -O3 -fno-math-errno -Wno-unused-label -Wno-unused-variable -Wno-write-strings -march=broadwell -mmmx -mno-3dnow -msse -msse2 -msse3 -mssse3 -mno-sse4a -mcx16 -msahf -mmovbe -maes -mno-sha -mpclmul -mpopcnt -mabm -mno-lwp -mfma -mno-fma4 -mno-xop -mbmi -mbmi2 -mno-tbm -mavx -mavx2 -msse4.2 -msse4.1 -mlzcnt -mrtm -mhle -mrdrnd -mf16c -mfsgsbase -mno-rdseed -mno-prfchw -mno-adx -mfxsr -mxsave -mxsaveopt -mno-avx512f -mno-avx512er -mno-avx512cd -mno-avx512pf -mno-prefetchwt1 -mno-clflushopt -mno-xsavec -mno-xsaves -mno-avx512dq -mno-avx512bw -mno-avx512vl -mno-avx512ifma -mno-avx512vbmi -mno-clwb -mno-pcommit -mno-mwaitx --param l1-cache-size=32 --param l1-cache-line-size=64 --param l2-cache-size=46080 -mtune=broadwell -DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION -m64 -fPIC -I/home/ubuntu/anaconda2/lib/python2.7/site-packages/numpy/core/include -I/home/ubuntu/anaconda2/include/python2.7 -I/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof -fvisibility=hidden -o /home/ubuntu/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-2.7.12-64/tmpiCKFY_/6736e2eaf88ec8b10a4d131b6445fd3c.so /home/ubuntu/.theano/compiledir_Linux-4.4–generic-x86_64-with-debian-stretch-sid-x86_64-2.7.12-64/tmpiCKFY_/mod.cpp -L/home/ubuntu/anaconda2/lib -lpython2.7
ERROR (theano.gof.cmodule): [Errno 12] Cannot allocate memory
OSError Traceback (most recent call last)
in ()
2 i = str(i)
3 model = train_last_layer(i)
----> 4 train_dense_layers(i, model)
in train_dense_layers(i, model)
28
29 conv_model.fit_generator(batches, samples_per_epoch=batches.n, nb_epoch=1,
—> 30 validation_data=val_batches, nb_val_samples=val_batches.n)
31 for layer in conv_model.layers[16:]: layer.trainable = True
32 conv_model.fit_generator(batches, samples_per_epoch=batches.N, nb_epoch=8,
/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/models.pyc in fit_generator(self, generator, samples_per_epoch, nb_epoch, verbose, callbacks, validation_data, nb_val_samples, class_weight, max_q_size, nb_worker, pickle_safe, initial_epoch, **kwargs)
933 nb_worker=nb_worker,
934 pickle_safe=pickle_safe,
–> 935 initial_epoch=initial_epoch)
936
937 def evaluate_generator(self, generator, val_samples,
/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc in fit_generator(self, generator, samples_per_epoch, nb_epoch, verbose, callbacks, validation_data, nb_val_samples, class_weight, max_q_size, nb_worker, pickle_safe, initial_epoch)
1453
1454 do_validation = bool(validation_data)
-> 1455 self._make_train_function()
1456 if do_validation:
1457 self._make_test_function()
/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/training.pyc in _make_train_function(self)
766 [self.total_loss] + self.metrics_tensors,
767 updates=updates,
–> 768 **self._function_kwargs)
769
770 def _make_test_function(self):
/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/backend/theano_backend.pyc in function(inputs, outputs, updates, **kwargs)
967 msg = ‘Invalid argument “%s” passed to K.function’ % key
968 raise ValueError(msg)
–> 969 return Function(inputs, outputs, updates=updates, **kwargs)
970
971
/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/backend/theano_backend.pyc in init(self, inputs, outputs, updates, **kwargs)
953 allow_input_downcast=True,
954 on_unused_input=‘ignore’,
–> 955 **kwargs)
956
957 def call(self, inputs):
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/compile/function.pyc in function(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)
318 on_unused_input=on_unused_input,
319 profile=profile,
–> 320 output_keys=output_keys)
321 # We need to add the flag check_aliased inputs if we have any mutable or
322 # borrowed used defined inputs
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/compile/pfunc.pyc in pfunc(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input, output_keys)
477 accept_inplace=accept_inplace, name=name,
478 profile=profile, on_unused_input=on_unused_input,
–> 479 output_keys=output_keys)
480
481
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/compile/function_module.pyc in orig_function(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input, output_keys)
1775 on_unused_input=on_unused_input,
1776 output_keys=output_keys).create(
-> 1777 defaults)
1778
1779 t2 = time.time()
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/compile/function_module.pyc in create(self, input_storage, trustme, storage_map)
1639 theano.config.traceback.limit = 0
1640 _fn, _i, _o = self.linker.make_thunk(
-> 1641 input_storage=input_storage_lists, storage_map=storage_map)
1642 finally:
1643 theano.config.traceback.limit = limit_orig
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/link.pyc in make_thunk(self, input_storage, output_storage, storage_map)
688 return self.make_all(input_storage=input_storage,
689 output_storage=output_storage,
–> 690 storage_map=storage_map)[:3]
691
692 def make_all(self, input_storage, output_storage):
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/vm.pyc in make_all(self, profiler, input_storage, output_storage, storage_map)
1001 storage_map,
1002 compute_map,
-> 1003 no_recycling))
1004 if not hasattr(thunks[-1], ‘lazy’):
1005 # We don’t want all ops maker to think about lazy Ops.
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/op.pyc in make_thunk(self, node, storage_map, compute_map, no_recycling)
968 try:
969 return self.make_c_thunk(node, storage_map, compute_map,
–> 970 no_recycling)
971 except (NotImplementedError, utils.MethodNotDefined):
972 logger.debug(‘Falling back on perform’)
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/op.pyc in make_c_thunk(self, node, storage_map, compute_map, no_recycling)
877 logger.debug(‘Trying CLinker.make_thunk’)
878 outputs = cl.make_thunk(input_storage=node_input_storage,
–> 879 output_storage=node_output_storage)
880 fill_storage, node_input_filters, node_output_filters = outputs
881
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/cc.pyc in make_thunk(self, input_storage, output_storage, storage_map, keep_lock)
1198 cthunk, in_storage, out_storage, error_storage = self.compile(
1199 input_storage, output_storage, storage_map,
-> 1200 keep_lock=keep_lock)
1201
1202 res = _CThunk(cthunk, init_tasks, tasks, error_storage)
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/cc.pyc in compile(self, input_storage, output_storage, storage_map, keep_lock)
1141 output_storage,
1142 storage_map,
-> 1143 keep_lock=keep_lock)
1144 return (thunk,
1145 [link.Container(input, storage) for input, storage in
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/cc.pyc in cthunk_factory(self, error_storage, in_storage, out_storage, storage_map, keep_lock)
1593 else:
1594 module = get_module_cache().module_from_key(
-> 1595 key=key, lnk=self, keep_lock=keep_lock)
1596
1597 vars = self.inputs + self.outputs + self.orphans
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/cmodule.pyc in module_from_key(self, key, lnk, keep_lock)
1140 try:
1141 location = dlimport_workdir(self.dirname)
-> 1142 module = lnk.compile_cmodule(location)
1143 name = module.file
1144 assert name.startswith(location)
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/cc.pyc in compile_cmodule(self, location)
1504 lib_dirs=self.lib_dirs(),
1505 libs=libs,
-> 1506 preargs=preargs)
1507 except Exception as e:
1508 e.args += (str(self.fgraph),)
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/cmodule.pyc in compile_str(module_name, src_code, location, include_dirs, lib_dirs, libs, preargs, py_module, hide_symbols)
2180
2181 try:
-> 2182 p_out = output_subprocess_Popen(cmd)
2183 compile_stderr = decode(p_out[1])
2184 except Exception:
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/misc/windows.pyc in output_subprocess_Popen(command, **params)
73 params[‘stdout’] = subprocess.PIPE
74 params[‘stderr’] = subprocess.PIPE
—> 75 p = subprocess_Popen(command, **params)
76 # we need to use communicate to make sure we don’t deadlock around
77 # the stdour/stderr pipe.
/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/misc/windows.pyc in subprocess_Popen(command, **params)
34
35 try:
—> 36 proc = subprocess.Popen(command, startupinfo=startupinfo, **params)
37 finally:
38 if stdin is not None:
/home/ubuntu/anaconda2/lib/python2.7/subprocess.pyc in init(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags)
709 p2cread, p2cwrite,
710 c2pread, c2pwrite,
–> 711 errread, errwrite)
712 except Exception:
713 # Preserve original exception in case os.close raises.
/home/ubuntu/anaconda2/lib/python2.7/subprocess.pyc in _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite)
1233 gc.disable()
1234 try:
-> 1235 self.pid = os.fork()
1236 except:
1237 if gc_was_enabled:
OSError: [Errno 12] Cannot allocate memory