Not sure if this will help you along your way, but if it does please let me know, as I adapted this to work with Python 3.6, it converts the MNIST files into JPG. I was thinking this was a way forward to work with our conditions, but haven’t yet followed through on getting it completely configured for our use. One thing that has puzzled me is that the valid directory / technique uses images, whereas I was expecting labels like 1 or 0. So that eludes me a bit…
# -*- coding: utf-8 -*-
"""
Created on Sun Nov 5 17:51:01 2017
Adapted for Python 3.6 from :
https://gist.github.com/ischlag/41d15424e7989b936c1609b53edd1390
Simple python script which takes the mnist data from tensorflow and builds a data
set based on jpg files and text files containing the image paths and labels.
Parts of it are from the mnist tensorflow example.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import gzip
import os
import sys
import time
from six.moves import urllib
from six.moves import xrange # pylint: disable=redefined-builtin
from scipy.misc import imsave
import tensorflow as tf
import numpy as np
import csv
SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/'
WORK_DIRECTORY = 'data'
IMAGE_SIZE = 28
NUM_CHANNELS = 1
PIXEL_DEPTH = 255
NUM_LABELS = 10
def maybe_download(filename):
"""Download the data from Yann's website, unless it's already here."""
if not tf.gfile.Exists(WORK_DIRECTORY):
tf.gfile.MakeDirs(WORK_DIRECTORY)
filepath = os.path.join(WORK_DIRECTORY, filename)
if not tf.gfile.Exists(filepath):
filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath)
with tf.gfile.GFile(filepath) as f:
size = f.Size()
print('Successfully downloaded', filename, size, 'bytes.')
return filepath
def extract_data(filename, num_images):
"""Extract the images into a 4D tensor [image index, y, x, channels].
Values are rescaled from [0, 255] down to [-0.5, 0.5].
"""
print('Extracting', filename)
with gzip.open(filename) as bytestream:
bytestream.read(16)
buf = bytestream.read(IMAGE_SIZE * IMAGE_SIZE * num_images)
data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)
#data = (data - (PIXEL_DEPTH / 2.0)) / PIXEL_DEPTH
data = data.reshape(num_images, IMAGE_SIZE, IMAGE_SIZE, 1)
return data
def extract_labels(filename, num_images):
"""Extract the labels into a vector of int64 label IDs."""
print('Extracting', filename)
with gzip.open(filename) as bytestream:
bytestream.read(8)
buf = bytestream.read(1 * num_images)
labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int64)
return labels
train_data_filename = maybe_download('train-images-idx3-ubyte.gz')
train_labels_filename = maybe_download('train-labels-idx1-ubyte.gz')
test_data_filename = maybe_download('t10k-images-idx3-ubyte.gz')
test_labels_filename = maybe_download('t10k-labels-idx1-ubyte.gz')
# Extract it into np arrays.
train_data = extract_data(train_data_filename, 60000)
train_labels = extract_labels(train_labels_filename, 60000)
test_data = extract_data(test_data_filename, 10000)
test_labels = extract_labels(test_labels_filename, 10000)
if not os.path.isdir("mnist/train-images"):
os.makedirs("mnist/train-images")
if not os.path.isdir("mnist/test-images"):
os.makedirs("mnist/test-images")
# process train data
with open("mnist/train-labels.csv", 'w', newline='') as csvFile:
writer = csv.writer(csvFile, delimiter=',', quotechar='"')
for i in range(len(train_data)):
imsave("mnist/train-images/" + str(i) + ".jpg", train_data[i][:,:,0])
writer.writerow(["train-images/" + str(i) + ".jpg", train_labels[i]])
# repeat for test data
with open("mnist/test-labels.csv", 'w', newline='') as csvFile:
writer = csv.writer(csvFile, delimiter=',', quotechar='"')
for i in range(len(test_data)):
imsave("mnist/test-images/" + str(i) + ".jpg", test_data[i][:,:,0])
writer.writerow(["test-images/" + str(i) + ".jpg", test_labels[i]])