[07.24a] changes in _dataCleanning.py and survey.py

u.go
2013-07-24T01:29:27Z

_dataCleanning.py

| #!/usr/bin/env python
| #encoding: utf-8
| #__author__ = 'actor2019'
|
| import csv
| import cStringIO
| import itertools
|
| # with open('test.csv', 'rb') as csvfile:
| # spamreader = csv.reader(csvfile, delimiter=',', quotechar='|')
| # for row in spamreader:
| # print '-'.join(row)
|
|
| # with open('test.csv', 'rb') as csvfile:
| # EXAMPLE_SIZE=256
| # sniffer = csv.Sniffer()
| # _dialect='csv.Dialect()'
| # _hasHeader=True
| # OB=[]
| # _file=csvfile.read()
| # _example=_file[0:EXAMPLE_SIZE]
| # _dialect,_hasHeader=sniffer.sniff(_example,delimiters=' ,\t'),sniffer.has_header(_example)
| #
| # #todo:note - with open('test.csv', 'rb') as csvfile, csvfile could only be use once
| # with open('test.csv', 'rb') as csvfile:
| # print type(csvfile)
| # iocsv=cStringIO.StringIO(csvfile.read())
| # print type(iocsv)
| # csvRows=csv.reader(iocsv,dialect=_dialect)
| # # csvRows= _csvLines if _hasHeader else _csvLines
| # for row in csvRows:
| # OB.append(row)
| # result=dict(
| # OB=OB,
| # # _csvLines='\n'.join([', '.join(row) for row in _csvLines]),
| # # _example=_example,
| # # _csvLines=str(_csvLines),
| # support=0.3,
| # accuracy=0.5,
| # )
| # print result
|
|
|
| def plCleaning(plfile,DEC=[]):
| """
| def csvCleaning(csvfile): return data
| """
| plRows=plfile.readlines()
| cleanRows=[x for x in map(lambda s: s.strip(),plRows) if x is not '']
|
| #init
| numOB,numAT,support,accuracy,decision,infoMsg,attNameList,AT,OB=([] for i in range(9))
|
| for row in cleanRows:
| argList=row[row.find('(')+1:row.rfind(')')].split(',')
| if row.startswith('object'):
| numOB=int(argList[0])#numOB
| numAT=int(argList[1])#numAT
| elif row.startswith('support'):
| support=float(argList[0])#support
| elif row.startswith('accuracy'):
| accuracy=float(argList[0])#accuracy
| elif row.startswith('decision'):
| decision=int(argList[0]) #decision
| elif row.startswith('attrib_values'):
| import re
| indexes=[i.start() for i in re.finditer(r',',row)]
| attNameList.append(row[indexes[0]+1:indexes[1]])#att_name_list
| argListAtt=row[row.find('[')+1:row.rfind(']')].split(',')
| #todo: transform every arg into a list for the purpose of OR descriptor (including DEC)
| AT.append([[arg] for arg in argListAtt])#AT
| elif row.startswith('data'):
| argListData=row[row.find('[')+1:row.rfind(']')].split(',')
| t_argList=[[] for _ in itertools.repeat(None,numAT)]
| count=0
| x=0
| while count<numAT:# How about using eval
| if argListData[x].find('[') != -1: #found [
| t_argList[count].append(argListData[x].replace('[',''))
| x=x+1
| while argListData[x].find(']') == -1: # while didn't find the next half ]
| t_argList[count].append(argListData[x])
| x=x+1
| t_argList[count].append(argListData[x].replace(']',''))
| x=x+1
| count=count+1
| continue
| if argListData[x].find('[') == -1: # did not find [
| if argListData[x] in ['nil','none','*','-']:
| t_argList[count]=[arg[0] for arg in AT[count]]
| else:
| t_argList[count].append(argListData[x])
| x=x+1
| count=count+1
| OB.append(t_argList)
| elif any(row.startswith(x) for x in ['resolution','total_cases']):
| infoMsg='Lines begin with '+','.join(['total_cases','resolution']) +' not used.'
| else:
| pass
| #todo:qa
|
|
| countOB=len(OB)
| if DEC:
| _OB=[]
| _numOB=0
| for ob in OB:
| if ob[decision-1][0] in DEC:
| _OB.append(ob)
| _numOB+=1
| OB = _OB
| numOB=_numOB
| AT[decision-1].remove(DEC)
|
|
| from collections import defaultdict
| dict2return,localDict = defaultdict(),locals()
| for var in 'numOB,numAT,support,accuracy,decision,infoMsg,attNameList,AT,OB'.split(','):
| dict2return[var]=localDict[var]
| # print OB
| # print AT
| # print u'Number of objects: {new} ( <- from {old})'.format(old=countOB,new=len(OB))
| print u'Number of objects: {new}'.format(old=countOB,new=len(OB))
| return dict(dict2return)
|

survey.py

| #!/usr/bin/env python
| #encoding: utf-8
|
| from _dataCleanning import plCleaning
| from _utilities import calCriteriaFromDC
|
|
| def ruleGeneration(argWrapper, orDescriptors=None):
| if not orDescriptors: orDescriptors = []
| dataWrapper=argWrapper
| localsList=['numOB', 'numAT', 'support', 'accuracy', 'decision',
| 'infoMsg', 'attNameList','AT', 'OB']
| numOB,numAT,support,accuracy,decision,infoMsg,attNameList,AT,OB=\
| tuple([dataWrapper[k] for k in localsList])
|
|
| # Todo: modify AT based on orDescriptors
| # print str(orDescriptors) >>>[[u'temperature', [u'normal', u'high']]]
| for od in orDescriptors:
| idx_att=attNameList.index(od[0])
| at_index_att=AT[idx_att]
| for cb in od[1:]:
| for av in cb:
| at_index_att.remove(at_index_att[at_index_att.index([av])])
| AT[idx_att].append(cb)
|
|
| D = decision - 1
| conIndexes=filter(lambda x:x !=D,xrange(0,numAT))
| import itertools
| CF=[]
| DCStep1=[]
| gStep=2
| NumOfParticles=6
|
| # todo:s1 rules with ONE CON attribute
| for d in xrange(0,len(AT[D])):
| CF.append([ [] for _ in itertools.repeat(None,len(conIndexes))])
| lDCAll=[[] for _ in itertools.repeat(None,numAT)]
| for C in conIndexes:
| # OR descriptor(AT[C]):
| # ['normal', 'high', 'very_high'] -> [['normal', 'high'], ['very_high']]
| for k in xrange(0,len(AT[C])):
| thisDC=[[] for _ in itertools.repeat(None,NumOfParticles)]
| #todo: sATV=set([AT[C][k]])? AT[C][k] is already a list if use OR descriptor
| sATV=set(AT[C][k])
| sDECV=set(AT[D][d])
| for ob in xrange(0,numOB):
| sCON=set(OB[ob][C])
| sDEC=set(OB[ob][D])
| if sCON==sATV:#inf(p)
| if sDEC==sDECV:
| thisDC[0].append(ob+1)
| elif sDECV.issubset(sDEC):
| thisDC[1].append(ob+1)
| else:
| thisDC[2].append(ob+1)
| elif sATV.issubset(sCON): #sup(p)-inf(p)
| if sDEC==sDECV:
| thisDC[3].append(ob+1)
| elif sDECV.issubset(sDEC):
| thisDC[4].append(ob+1)
| else:
| thisDC[5].append(ob+1)
| else: #p'
| #thisDC[6].append(ob+1)
| pass
| # compute criteria values
| minsupp,minacc,maxsupp,maxacc = calCriteriaFromDC(thisDC,numOB)
| if minsupp >=support and minacc>=accuracy:
| CF[d][0].append([[C,k,d,minsupp,minacc,maxsupp,maxacc,'Lower']])
| elif maxsupp >=support and maxacc>=accuracy:
| CF[d][0].append([[C,k,d,minsupp,minacc,maxsupp,maxacc,'Upper']])
| elif maxsupp >=support:
| CF[d][0].append([[C,k,d,minsupp,minacc,maxsupp,maxacc,'flag4merge']])
| else:
| #only append to CF the promising DC better than 'flag4merge'
| pass
| #append all thisDC to lDCAll
| lDCAll[C].append(thisDC)
| #eof: for k
| #eof: for C
| DCStep1.append(lDCAll)
| # eof: for d
|
| if gStep==2:
| #current step
| cf=1
| DCCurrent=[ [] for _ in itertools.repeat(None,len(AT[D]))]
| for d in xrange(0,len(AT[D])):
| lenCFm1=len(CF[d][cf-1])
| for k in xrange(0,lenCFm1):
| # flag4merge rules CAN be used in multiple complex rules
| if CF[d][cf-1][k][0][7] != 'flag4merge':
| continue
| iC, iK, iD = CF[d][cf-1][k][0][0:3]
| # for: k to merge
| for km in xrange(k+1,lenCFm1):
| if CF[d][cf-1][km][0][7] != 'flag4merge':
| continue
| iCm,iKm,iDm=CF[d][cf-1][km][0][0:3]
| # not appropriate to merge
| if iC==iCm or iD!=iDm:
| continue
| DCK=DCStep1[iD][iC][iK]
| DCKm=DCStep1[iDm][iCm][iKm]
| mDC=[ [] for _ in itertools.repeat(None,NumOfParticles)]
| mDC[0]=list(set(DCK[0])& set(DCKm[0]))
| mDC[1]=list(set(DCK[1])& set(DCKm[1]))
| mDC[2]=list(set(DCK[2])& set(DCKm[2]))
| mDC[3]=list((set(DCK[0])& set(DCKm[3]))|(set(DCK[3])& set(DCKm[0]))|(set(DCK[3])& set(DCKm[3])))
| mDC[4]=list((set(DCK[1])& set(DCKm[4]))|(set(DCK[4])& set(DCKm[1]))|(set(DCK[4])& set(DCKm[4])))
| mDC[5]=list((set(DCK[2])& set(DCKm[5]))|(set(DCK[5])& set(DCKm[2]))|(set(DCK[5])& set(DCKm[5])))
| minsupp,minacc,maxsupp,maxacc = calCriteriaFromDC(mDC,numOB)
| if minsupp >=support and minacc>=accuracy:
| # CF[d][cf-1][km][0][7] = 'flag4mergeDone'
| CF[d][cf].append([[iC,iK,iD,minsupp,minacc,maxsupp,maxacc,'Lower'],
| [iCm,iKm,iDm,minsupp,minacc,maxsupp,maxacc,'Lower']])
| #todo:mark DCStep2 unused
| # DCCurrent[d].append([[iC,iK,iD],[iCm,iKm,iDm],mDC])
| elif maxsupp >=support and maxacc>=accuracy:
| # CF[d][cf-1][km][0][7] = 'flag4mergeDone'
| CF[d][cf].append([[iC,iK,iD,minsupp,minacc,maxsupp,maxacc,'Upper'],
| [iCm,iKm,iDm,minsupp,minacc,maxsupp,maxacc,'Upper']])
| # DCCurrent[d].append([[iC,iK,iD],[iCm,iKm,iDm],mDC])
| else:
| #the merged DC still don't satisfy
| pass
| # eof: for km
| #eof: for k
| #eof: for d
| # #DCStep2
| DCStep2=DCCurrent
| #eof: if gStep==2
| for k in ['gStep','D','CF','DCStep1']:
| dataWrapper[k]=locals()[k]
| return dataWrapper
| #eof: def ruleGeneration
|
|
| def _dTCF2list(argWrapper):
| localsList=['numOB', 'numAT', 'support', 'accuracy', 'decision',
| 'infoMsg', 'attNameList','AT', 'OB','gStep','D','CF','DCStep1']
| numOB,numAT,support,accuracy,decision,infoMsg,attNameList,AT,OB,gStep,D,CF,DCStep1=\
| tuple([argWrapper[k] for k in localsList])
|
| data=[]
| from collections import OrderedDict
| # thisOb: {'CON,p -> DEC,q':,m4:, 'Lower/Upper':}
| for d in xrange(0,len(AT[D])):
| for icf in xrange(0,gStep):
| for k in xrange(0,len(CF[d][icf])):
| if CF[d][icf][k][0][7]=='flag4merge':
| continue
| thisOb=OrderedDict({'Rules':len(data)+1})
| cpdq=''
| indd=CF[d][icf][k][0][2]
| for j in xrange(0,icf+1):
| indC,indk,indd=CF[d][icf][k][j][0:3]
| # '&' ->','
| avCk=AT[indC][indk][0] if len(AT[indC][indk])==1 else str(AT[indC][indk])
| cpdq=cpdq+attNameList[indC]+':'+avCk+'& '
| avDd=AT[D][indd][0] if len(AT[D][indd])==1 else str(AT[D][indd])
| cpdq=cpdq[0:-2]+'->'+attNameList[D]+':'+avDd
| thisOb['CON,p -> DEC,q']=cpdq
| m4p1=CF[d][icf][k][0][3:8]
| for i,a in enumerate(['minsupp','minacc','maxsupp','maxacc','Lower/Upper']):
| thisOb[a]=m4p1[i]
| data.append(thisOb)
| return data
|
|
| # (ちょっと古いバージョンから)
| with open('flu.pl', 'rb') as plFile:
|
| # # 例1：flu の決定属性値は yes、no
| DEC=['yes']
| argWrapper = plCleaning(plFile,DEC)
|
| # # 例２：mammo の決定属性値は0、1
| # DEC=['1']
| # argWrapper = plCleaning(plFile,DEC)
|
|
| # orDescriptors example:
| #['temperature',['normal','high']]
| # ['AttName2',[0,1,2],[8,9,10]]
| # orDescriptors=[[u'temperature', [u'normal', u'high']]]
| # dataWrapper=ruleGeneration(argWrapper,orDescriptors=orDescriptors)
|
| dataWrapper=ruleGeneration(argWrapper)
|
| rules=_dTCF2list(dataWrapper)
| for rule in rules:
| print u' '.join([str(v) for v in rule.values()])