CNTK Study (1): Converting MNIST Files

In the CNTK MNIST example, there is a .py file that first converts the image files into a text representation so CNTK can read them.

File location: \cntk\Examples\Image\DataSets\MNIST\mnist_utils.py

import sys
import urllib
import gzip
import shutil
import os
import struct
import numpy as np

// Define a function to load data

def loadData(src, cimg):
 print ('Downloading ' + src)
 gzfname, h = urllib.urlretrieve(src, './delete.me')  // Download the data file from the URL
 print ('Done.')
 try:
 with gzip.open(gzfname) as gz: // Open it with gzip
 n = struct.unpack('I', gz.read(4)) // Read 4 bytes and unpack as unsigned int
 # Read magic number.
 if n[0] != 0x3080000: // If the file header is wrong, treat the file as invalid
 raise Exception('Invalid file: unexpected magic number.')
 # Read number of entries.
 n = struct.unpack('>I', gz.read(4))[0]  // Read another 4 bytes as unsigned int
 if n != cimg: // If it does not match the expected image count, raise an exception
 raise Exception('Invalid file: expected {0} entries.'.format(cimg)) 
 crow = struct.unpack('>I', gz.read(4))[0] // Read 4 bytes as the number of rows
 ccol = struct.unpack('>I', gz.read(4))[0]  // Read 4 bytes as the number of columns
 if crow != 28 or ccol != 28: // If rows/cols are not 28, the image file is wrong
 raise Exception('Invalid file: expected 28 rows/cols per image.')
 # Read data.  // Read data; the total size is rows × columns times how many bytes each pixel needs
 res = np.fromstring(gz.read(cimg * crow * ccol), dtype = np.uint8) //
 finally:
 os.remove(gzfname)
 return res.reshape((cimg, crow * ccol))  // When returning the data, arrange it as a 2D array

// Read the label file

def loadLabels(src, cimg):
 print 'Downloading ' + src
 gzfname, h = urllib.urlretrieve(src, './delete.me')
 print 'Done.'
 try:
 with gzip.open(gzfname) as gz:
 n = struct.unpack('I', gz.read(4))
 # Read magic number.
 if n[0] != 0x1080000:
 raise Exception('Invalid file: unexpected magic number.')
 # Read number of entries.
 n = struct.unpack('>I', gz.read(4))
 if n[0] != cimg:
 raise Exception('Invalid file: expected {0} rows.'.format(cimg))
 # Read labels.
 res = np.fromstring(gz.read(cimg), dtype = np.uint8)
 finally:
 os.remove(gzfname)
 return res.reshape((cimg, 1))// Also return a 2D array

if __name__ == "__main__":
 trnData = loadData('http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz', 60000) // Download the image file
 trnLbl = loadLabels('http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz', 60000) // Download the label file
 trn = np.hstack((trnLbl, trnData)) // Merge the arrays
 print 'Writing train text file...'
 np.savetxt(r'./../Data/Train-28x28.txt', trn, fmt = '%u', delimiter=' ') // Save the array as text
 print 'Done.'
 testData = loadData('http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz', 10000) // Download test data
 testLbl = loadLabels('http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz', 10000) // Download test labels
 test = np.hstack((testLbl, testData)) // Merge the arrays
 print 'Writing test text file...'
 np.savetxt(r'./../Data/Test-28x28.txt', test, fmt = '%u', delimiter=' ') // Write out as a txt file
 print 'Done.'

 

However there is a problem: download speeds inside China are too slow, so I modified the file. Change:

def loadData(src, cimg):
 print ('Downloading ' + src)
 gzfname, h = urllib.urlretrieve(src, './delete.me')  // Download the data file from the URL

All changed to:

def loadLabels(gzfname, cimg):
 #print ('Downloading ' + src)
 #gzfname, h = urlretrieve(src, './delete.me')

Then modify the install_mnist.py file as follows:

from __future__ import print_function
import mnist_utils as ut

if __name__ == "__main__":
 train = ut.load('./train-images-idx3-ubyte.gz',
 './train-labels-idx1-ubyte.gz', 60000)
 print ('Writing train text file...')
 ut.savetxt(r'./Train-28x28_cntk_text.txt', train)
 print ('Done.')
 test = ut.load('./t10k-images-idx3-ubyte.gz',
 './t10k-labels-idx1-ubyte.gz', 10000)
 print ('Writing test text file...')
 ut.savetxt(r'./Test-28x28_cntk_text.txt', test)
 print ('Done.')

Find a way to download the MNIST data into that directory, which avoids the download-speed problem.