CNTK研究（一）：MNIST的文件轉換

CNTK的MNIST例子中，它是一個py文件，首先是將圖像文件轉換為文本來表達，以方便CNTK中讀取。

文件位置：\cntk\Examples\Image\DataSets\MNIST\mnist_utils.py

import sys
import urllib
import gzip
import shutil
import os
import struct
import numpy as np

//定義一個讀數據的函數

def loadData(src, cimg):
print ('Downloading ' + src)
gzfname, h = urllib.urlretrieve(src, './delete.me') //從URL下載該數據文件
print ('Done.')
try:
with gzip.open(gzfname) as gz: //使用gzip打開它
n = struct.unpack('I', gz.read(4)) //讀4個字節解包成無符號整型
# Read magic number.
if n[0] != 0x3080000: //如果文件開頭不對，則認為文件不對
raise Exception('Invalid file: unexpected magic number.')
# Read number of entries.
n = struct.unpack('>I', gz.read(4))[0] //再讀4個字節解包成無符號整型
if n != cimg: //如果不屬於圖像文件，則拋出異常
raise Exception('Invalid file: expected {0} entries.'.format(cimg))
crow = struct.unpack('>I', gz.read(4))[0] //讀取4個數據為行
ccol = struct.unpack('>I', gz.read(4))[0] //讀取4個字節為列
if crow != 28 or ccol != 28: //如果行與列不等於28，說明圖像文件有誤
raise Exception('Invalid file: expected 28 rows/cols per image.')
# Read data. //讀取數據，後面需要讀取的大小為行乘以列再乘以表達每個象素需要多少字節便是總長度
res = np.fromstring(gz.read(cimg * crow * ccol), dtype = np.uint8) //
finally:
os.remove(gzfname)
return res.reshape((cimg, crow * ccol)) //返回數據時，排成2維數組表達

//讀取標籤文件

def loadLabels(src, cimg):
print 'Downloading ' + src
gzfname, h = urllib.urlretrieve(src, './delete.me')
print 'Done.'
try:
with gzip.open(gzfname) as gz:
n = struct.unpack('I', gz.read(4))
# Read magic number.
if n[0] != 0x1080000:
raise Exception('Invalid file: unexpected magic number.')
# Read number of entries.
n = struct.unpack('>I', gz.read(4))
if n[0] != cimg:
raise Exception('Invalid file: expected {0} rows.'.format(cimg))
# Read labels.
res = np.fromstring(gz.read(cimg), dtype = np.uint8)
finally:
os.remove(gzfname)
return res.reshape((cimg, 1))//同樣返回2維數組表達

if __name__ == "__main__":
trnData = loadData('http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz', 60000) //下載圖像文件
trnLbl = loadLabels('http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz', 60000) //下載標籤文件
trn = np.hstack((trnLbl, trnData)) //將數組進行合併
print 'Writing train text file...'
np.savetxt(r'./../Data/Train-28x28.txt', trn, fmt = '%u', delimiter='\t') //將數組用文本形式保存
print 'Done.'
testData = loadData('http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz', 10000) //下載測試數據
testLbl = loadLabels('http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz', 10000) //下載測試標籤
test = np.hstack((testLbl, testData)) //將數組進行合併
print 'Writing test text file...'
np.savetxt(r'./../Data/Test-28x28.txt', test, fmt = '%u', delimiter='\t') //轉寫為 txt 文件
print 'Done.'

然而這有個問題，在國內下得太慢，所以修改了下文件，將：

def loadData(src, cimg):
print ('Downloading ' + src)
gzfname, h = urllib.urlretrieve(src, './delete.me') //從URL下載該數據文件

均修改為：

def loadLabels(gzfname, cimg):
#print ('Downloading ' + src)
#gzfname, h = urlretrieve(src, './delete.me')

然後將install_mnist.py文件修改為：

from __future__ import print_function
import mnist_utils as ut

if __name__ == "__main__":
train = ut.load('./train-images-idx3-ubyte.gz',
'./train-labels-idx1-ubyte.gz', 60000)
print ('Writing train text file...')
ut.savetxt(r'./Train-28x28_cntk_text.txt', train)
print ('Done.')
test = ut.load('./t10k-images-idx3-ubyte.gz',
'./t10k-labels-idx1-ubyte.gz', 10000)
print ('Writing test text file...')
ut.savetxt(r'./Test-28x28_cntk_text.txt', test)
print ('Done.')

找個方法把MNIST的數據下載放在該目錄下，這樣就避免下了載速度上的問題。