#使用pandas包的最简示例
 chunker = pd.read_csv(PATH_LOAD, chunksize = CHUNK_SIZE)

#按列按需读取
  columns = ("date_time",  "user_id")
  chunks_train = pd.read_csv(filename, usecols = columns, chunksize = 100000)

#分块分行读取
for rawPiece in chunker_rawData:
  current_chunk_size = len(rawPiece.index)   #rawPiece 是dataframe
  for i in range(current_chunk_size ):
    timeFlag = timeShape(rawPiece.ix[i])   #获取第i行的数据

#使用pandas包的方法，转换操作多，效率低下。
    userList = []
    content = pd.read_csv(filename)
    for i in range(len(content)):
        line = content.ix[i]['id']
        userList.append(line)

#直接使用原生操作读取外部数据，效率较高，但数据操作不及pandas方便
    userList = []
    f = open(filename)
    content = f.readlines()
    for line in content:
        line = line.replace('\n', '').split(',')
        userList.append(line)

inx1 = DataFrame(np.random.randn(nSample_neg), columns = ['randVal'])
inx2 = DataFrame(range(nSample_neg), columns = ['inxVal'])
inx = pd.concat([inx1, inx2], axis = 1)

# encoding=utf-8

import redis
from hashlib import md5

class SimpleHash(object):
    def __init__(self, cap, seed):
        self.cap = cap
        self.seed = seed

    def hash(self, value):
        ret = 0
        for i in range(len(value)):
            ret += self.seed * ret + ord(value[i])
        return (self.cap - 1) & ret


class BloomFilter(object):
    def __init__(self, host='localhost', port=6379, db=0, blockNum=1, key='bloomfilter'):
        """
        :param host: the host of Redis
        :param port: the port of Redis
        :param db: witch db in Redis
        :param blockNum: one blockNum for about 90,000,000; if you have more strings for filtering, increase it.
        :param key: the key's name in Redis
        """
        self.server = redis.Redis(host=host, port=port, db=db)
        self.bit_size = 1 << 31  # Redis的String类型最大容量为512M，现使用256M
        self.seeds = [5, 7, 11, 13, 31, 37, 61]
        self.key = key
        self.blockNum = blockNum
        self.hashfunc = []
        for seed in self.seeds:
            self.hashfunc.append(SimpleHash(self.bit_size, seed))

    def isContains(self, str_input):
        if not str_input:
            return False
        m5 = md5()
        m5.update(str_input)
        str_input = m5.hexdigest()
        ret = True
        name = self.key + str(int(str_input[0:2], 16) % self.blockNum)
        for f in self.hashfunc:
            loc = f.hash(str_input)
            ret = ret & self.server.getbit(name, loc)
        return ret

    def insert(self, str_input):
        m5 = md5()
        m5.update(str_input)
        str_input = m5.hexdigest()
        name = self.key + str(int(str_input[0:2], 16) % self.blockNum)
        for f in self.hashfunc:
            loc = f.hash(str_input)
            self.server.setbit(name, loc, 1)


if __name__ == '__main__':
""" 第一次运行时会显示 not exists!，之后再运行会显示 exists! """
    bf = BloomFilter()
    if bf.isContains('http://www.baidu.com'):   # 判断字符串是否存在
        print 'exists!'
    else:
        print 'not exists!'
        bf.insert('http://www.baidu.com')

import random

def loaddata(filename):
    dataMat = []
    with open(filename) as fp:
        for line in fp.readlines():
            dataMat.append(line.strip().split('\t'))
    return dataMat

def simple_sampling(dataMat, num):
    try:
        samples = random.sample(dataMat, num)
        return samples
    except:
        print('sample larger than population')

def loaddata(filename):
    dataMat = []
    with open(filename) as fp:
        for line in fp.readlines():
            dataMat.append(line.strip().split('\t'))
    return dataMat

def systematic_sampling(dataMat, num):
    k = int(len(dataMat)/num)
    samples = [random.sample(dataMat[i*k:(i+1)*k], 1) for i in range(num)]
    return samples

import numpy as np
import matplotlib.pyplot as plt
def pool(L,k):
    arr = L[:k]
    for i,e in enumerate(L[k:]):
        r = np.random.randint(0,k+i+1)
        if r<=k-1:
            arr[r] = e
    return arr
def count(q,n):
    L = array("d")
    l1 = array("d")
    l2 = array("d")
    for e in q:
        L.append(e)
    for e in range(n):
        l1.append(L.count(e))
    for e in l1:
        l2.append(e/sum(l1))
    return l1,l2
if __name__ == '__main__':
    a = np.array([[i]*10000000 for i in range(3)])#生成等量的0，1，2
    a.shape = 1,-1
    L = a[0]
    k = 300000#设置要抽取的样本的数量，一般远小于总体数量
    p = pool(L, k)
    l1 = ['value=%d'% x for x in range(3)]
    plt.pie(count(p,3)[0],labels=l1,labeldistance=0.1,autopct='%1.2f%%')
    plt.title("Reservoir sampling")
    plt.show()

def pca(dataMat, topNfeat=9999999):
    meanVals = np.mean(dataMat, axis=0)
    #print(meanVals)
    meanRemoved = dataMat - meanVals #remove mean
    #print(meanRemoved)
    covMat = np.cov(meanRemoved, rowvar=0)
    eigvals,eigvects = np.linalg.eig(np.mat(covMat)) #计算协方差矩阵的特征值和特征向量
    eig_valind = np.argsort(eigvals)                    #sort, sort goes smallest to largest
    eig_valind = eig_valind[:-(topNfeat+1):-1]  #cut off unwanted dimensions
    red_eigvects = eigvects[:,eig_valind]          #reorganize eig vects largest to smallest
    low_datamat = meanRemoved * red_eigvects#transform data into new dimensions
    reconmat = (low_datamat * red_eigvects.T) + meanVals
    return low_datamat, reconmat

学习时间  分数
9         39
15        56
25        93
14        61
10        50
18        75
0         32
16        85
5         42
19        70
16        66
20        80

import numpy as np
import matplotlib.pyplot as plt
def loadDataSet(fileName, delim='\t'):
    fr = open(fileName)
    stringArr = [line.strip().split(delim) for line in fr.readlines()]
    datArr = [list(map(float,line)) for line in stringArr]
    #print(mat(datArr))
    fr.close()
    return np.mat(datArr)

def pca(dataMat, topNfeat=9999999):
    meanVals = np.mean(dataMat, axis=0)
    #print(meanVals)
    meanRemoved = dataMat - meanVals #remove mean
    #print(meanRemoved)
    covMat = np.cov(meanRemoved, rowvar=0)
    eigvals,eigvects = np.linalg.eig(np.mat(covMat)) #计算协方差矩阵的特征值和特征向量
    eig_valind = np.argsort(eigvals)                    #sort, sort goes smallest to largest
    eig_valind = eig_valind[:-(topNfeat+1):-1]  #cut off unwanted dimensions
    red_eigvects = eigvects[:,eig_valind]          #reorganize eig vects largest to smallest
    low_datamat = meanRemoved * red_eigvects#transform data into new dimensions
    reconmat = (low_datamat * red_eigvects.T) + meanVals
    return low_datamat, reconmat

def plotBestFit(dataSet1,dataSet2):
    dataArr1 = np.array(dataSet1)
    dataArr2 = np.array(dataSet2)
    n = np.shape(dataArr1)[0]
    n1=shape(dataArr2)[0]
    xcord1 = []; ycord1 = []
    xcord2 = []; ycord2 = []
    xcord3=[];ycord3=[]
    j=0
    for i in range(n):
        xcord1.append(dataArr1[i,0]); ycord1.append(dataArr1[i,1])
        xcord2.append(dataArr2[i,0]); ycord2.append(dataArr2[i,1])
    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.scatter(xcord1, ycord1, s=30, c='red', marker='s')
    ax.scatter(xcord2, ycord2, s=30, c='green')

    plt.xlabel('X1'); plt.ylabel('X2');
    plt.show()

if __name__=='__main__':
    mata=loadDataSet('score')
    a,b= pca(mata, 4)
    plotBestFit(a,b)