Skip to content

Instantly share code, notes, and snippets.

@jasstionzyf

jasstionzyf/MultiLabel.java

Created July 7, 2017 05:52
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save jasstionzyf/536c4448380acafe58b05fd4d8469cee to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save jasstionzyf/536c4448380acafe58b05fd4d8469cee to your computer and use it in GitHub Desktop.
Download ZIP
Raw
application.properties
masterUrl=spark://172.16.241.100:7077
jarsPath=hdfs://172.16.241.111:9000/data/datacenter-hadoop.jar
appName=clusterImageFeatures
spark.serializer=org.apache.spark.serializer.KryoSerializer
spark.driver.maxResultSize=0
spark.rdd.compress=true
spark.kryoserializer.buffer.max=1024m
spark.eventLog.enabled=false
spark.files.overwrite=true
spark.ui.enabled=true
spark.executor.memory=10g
cluster_storageLevel=MEMORY_AND_DISK_SER
spark.rpc.message.maxSize=512
spark.executor.cores=3
spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps
isLocal=false
spark.network.timeout=60000s
spark.local.dir=/data_hadoop_2/
cluster_storageLevel=DISK_ONLY
multiLabeledTrainFile=/data/mlib_data/multilabelfile_train.csv
multiLabeledTestFile=/data/mlib_data/multilabelfile_test.csv
batchSize=3
labelIndexTo=6319
scoreThreadHold=0.6
learningRate=0.01
numEpochs=50
numOutputs=6320
numHiddenNodes=1000
spark.hadoop.fs.defaultFS =hdfs://172.16.241.111:9000
spark.serializer=org.apache.spark.serializer.KryoSerializer
spark.kryo.registrator=org.nd4j.Nd4jRegistrator
spark.driver.allowMultipleContexts = true
spark.memory.storageFraction=0.7
spark.memory.fraction=0.8
batchSizePerWorke=3
Raw
MultiLabel.java
package com.comm.component.dl4j;
import com.google.common.collect.Lists;
import com.yufei.utils.CommonUtil;
import javafx.collections.transformation.SortedList;
import javafx.util.Pair;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.storage.StorageLevel;
import org.datavec.api.records.reader.RecordReader;
import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
import org.datavec.api.split.FileSplit;
import org.deeplearning4j.datasets.datavec.RecordReaderDataSetIterator;
import org.deeplearning4j.eval.Evaluation;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.WorkspaceMode;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
import org.deeplearning4j.spark.api.TrainingMaster;
import org.deeplearning4j.spark.impl.multilayer.SparkDl4jMultiLayer;
import org.deeplearning4j.spark.impl.paramavg.ParameterAveragingTrainingMaster;
import org.deeplearning4j.util.ModelSerializer;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.api.ops.impl.accum.MatchCondition;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.indexing.BooleanIndexing;
import org.nd4j.linalg.indexing.conditions.Conditions;
import org.nd4j.linalg.io.ClassPathResource;
import org.nd4j.linalg.lossfunctions.LossFunctions;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.*;
/**
* Created by jasstion on 01/06/2017.
*/
public class MultiLabel {
protected final static org.slf4j.Logger mLog = LoggerFactory.getLogger(MultiLabel.class);
public static void MultiLabeledTrainAndTest(Properties settings, JavaSparkContext sc) throws Exception{
int batchSizePerWorker=Integer.parseInt(settings.getProperty("batchSizePerWorker","10"));
String multiLabeledTrainFile=settings.getProperty("multiLabeledTrainFile");
String multiLabeledTestFile=settings.getProperty("multiLabeledTestFile");
int batchSize=Integer.parseInt(settings.getProperty("batchSize","10"));
int labelIndexTo=Integer.parseInt(settings.getProperty("labelIndexTo","6319"));
double scoreThreadHold=Double.parseDouble(settings.getProperty("scoreThreadHold","0.6"));
int seed = Integer.parseInt(settings.getProperty("seed","123"));
double learningRate =Double.parseDouble(settings.getProperty("learningRate","0.01"));
int numEpochs =Integer.parseInt(settings.getProperty("numEpochs","1")); ;
int numInputs = 2048;
int numOutputs =Integer.parseInt(settings.getProperty("numOutputs","6320"));;
int numHiddenNodes =Integer.parseInt(settings.getProperty("numHiddenNodes","1000"));
RecordReader rr = new CSVRecordReader();
rr.initialize(new FileSplit(new File(multiLabeledTrainFile)));
DataSetIterator trainIter = new RecordReaderDataSetIterator(rr,batchSize,0,labelIndexTo,true);
RecordReader rrTest = new CSVRecordReader();
rrTest.initialize(new FileSplit(new File(multiLabeledTestFile)));
DataSetIterator testIter = new RecordReaderDataSetIterator(rrTest,batchSize,0,labelIndexTo,true);
List<org.nd4j.linalg.dataset.DataSet> trainDataList = new ArrayList<>();
List<org.nd4j.linalg.dataset.DataSet> testDataList = new ArrayList<>();
while (trainIter.hasNext()) {
try {
trainDataList.add(trainIter.next());
}catch (Exception e){
mLog.info("error tp process train item");
}
}
while (testIter.hasNext()) {
try{
testDataList.add(testIter.next());
}catch (Exception e){
mLog.info("error tp process test item");
}
}
JavaRDD<org.nd4j.linalg.dataset.DataSet> trainData = sc.parallelize(trainDataList).persist(StorageLevel.MEMORY_AND_DISK_SER());
JavaRDD<org.nd4j.linalg.dataset.DataSet> testData = sc.parallelize(testDataList).persist(StorageLevel.MEMORY_AND_DISK_SER());
MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
.seed(seed)
.iterations(1)
.optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT)
.learningRate(learningRate)
.updater(Updater.NESTEROVS).momentum(0.9)
.list()
.layer(0, new DenseLayer.Builder().nIn(numInputs).nOut(numHiddenNodes)
.weightInit(WeightInit.XAVIER)
.activation(Activation.RELU)
.build())
.layer(1, new OutputLayer.Builder(LossFunctions.LossFunction.COSINE_PROXIMITY)
.activation(Activation.SIGMOID).weightInit(WeightInit.XAVIER)
.nIn(numHiddenNodes).nOut(numOutputs).build())
.pretrain(false).backprop(true).build();
conf.setInferenceWorkspaceMode(WorkspaceMode.SINGLE);
conf.setTrainingWorkspaceMode(WorkspaceMode.SINGLE);
//Configuration for Spark training: see http://deeplearning4j.org/spark for explanation of these configuration options
TrainingMaster tm = new ParameterAveragingTrainingMaster.Builder(batchSizePerWorker) //Each DataSet object: contains (by default) 32 examples
.averagingFrequency(5)
.workerPrefetchNumBatches(0) //Async prefetching: 2 examples per worker
.batchSizePerWorker(batchSizePerWorker)
.storageLevel(StorageLevel.MEMORY_AND_DISK_SER())
.build();
//Create the Spark network
SparkDl4jMultiLayer sparkNet = new SparkDl4jMultiLayer(sc, conf, tm);
MultiLayerNetwork mode=null;
//Execute training:
for (int i = 0; i < numEpochs; i++) {
mode=sparkNet.fit(trainData);
Nd4j.getWorkspaceManager().destroyAllWorkspacesForCurrentThread();
mLog.info("Completed Epoch {}", i);
}
//对测试结果进行验证
//对于预测vector获取topNum分数最高的，然后和lables vector no zeros elements 进行交集操作，得出交集的个数作为标签结果质量的衡量
testIter.reset();
while(testIter.hasNext()){
DataSet t = testIter.next();
INDArray features = t.getFeatureMatrix();
INDArray lables = t.getLabels();
INDArray predicted = mode.output(features,false);
BooleanIndexing.replaceWhere(predicted, 0.0, Conditions.lessThan(scoreThreadHold));
BooleanIndexing.replaceWhere(predicted, 1.0, Conditions.greaterThanOrEqual(scoreThreadHold));
predicted.muli(lables);
MatchCondition op = new MatchCondition(predicted, Conditions.equals(1.0));
int matchLabeledNum = Nd4j.getExecutioner().exec(op,Integer.MAX_VALUE).getInt(0);
mLog.info("matchLabeledNum: "+matchLabeledNum+"");
}
String modelSavedpath="/data/mlib_data/multiLabeledModel.zip";
ModelSerializer.writeModel(mode,modelSavedpath,true);
}
public static void main(String[] args) throws Exception{
String multiLabeledModelPath="/data/mlib_data/multiLabeledModel.zip";
MultiLayerNetwork model= ModelSerializer.restoreMultiLayerNetwork(new File(multiLabeledModelPath));
String t1=
"0.78045719862,0.0299201924354,0.420334964991,0.332571715117,0.31880363822,0.697145402431,0.713323056698,0.922250807285,1.18828821182,0.316269278526,0.394945442677,0.28993549943,0.640204191208,0.256769657135,0.185593828559,0.186182081699,0.188495963812,0.356428533792,0.0408461838961,0.230980783701,0.142582118511,0.291987717152,0.0925933495164,0.207951843739,0.0616482533514,0.297145843506,1.40524995327,0.288228034973,0.469323128462,0.340316921473,0.143274366856,0.262888997793,0.443491876125,0.108093835413,0.988363802433,0.474824488163,0.805051326752,1.18487846851,0.303155034781,0.309827119112,0.527673780918,0.448305040598,1.10348463058,0.350689530373,0.712052941322,0.422892391682,0.870871722698,0.326191455126,0.612790942192,0.131685048342,0.28846475482,0.894923150539,0.493356287479,0.263356298208,0.130682945251,0.0990545302629,0.874135375023,0.156553074718,0.266656339169,0.443401455879,0.306883215904,0.198194012046,0.122733145952,0.415086984634,0.321140110493,0.606312692165,0.138291969895,1.23109281063,0.6183770895,0.109809502959,0.937213480473,0.0274696461856,0.702575266361,0.286649882793,1.13459289074,0.529453098774,0.266480535269,0.0757255107164,0.515020549297,1.1170591116,0.149234980345,0.490118145943,0.799854278564,0.436883032322,0.216320618987,0.280596077442,0.104254148901,0.385599464178,0.418349772692,0.364298641682,0.455970317125,0.153568789363,0.398602306843,0.912377357483,0.267240613699,0.0503337234259,0.527883410454,0.0260379500687,0.642993330956,0.360006451607,0.497466742992,0.219401478767,0.182346180081,0.484224140644,0.0469531752169,1.09967267513,0.829146981239,0.279294610023,0.196875557303,0.169167160988,0.174874067307,0.257383525372,1.1850720644,0.617714107037,0.242995291948,0.230284303427,0.332745760679,0.0304184313864,0.509839832783,1.20727825165,0.15220977366,1.10271847248,0.274921894073,0.360559880733,0.383916497231,0.157332420349,0.217326268554,0.282393693924,0.276792705059,1.51695430279,0.125488042831,0.111907377839,0.379829585552,0.395299613476,0.121192388237,0.799848079681,0.10123129189,0.0955021753907,0.131367191672,0.894854366779,0.0257305912673,0.0322850383818,0.300218015909,0.326450645924,0.0829000771046,0.620928823948,0.928899645805,0.206189870834,0.425546199083,0.225672125816,0.238575905561,0.340766251087,0.172874629498,0.459969878197,0.279356479645,0.372089982033,0.364162504673,0.142073124647,0.319530755281,0.487176477909,0.10102314502,0.448662281036,0.297013759613,0.145868837833,0.593244016171,0.126365587115,0.354956328869,0.530263662338,0.44037270546,0.912460386753,0.338060706854,0.363397955894,0.353675961494,0.118255227804,0.460938721895,0.170444130898,0.159251630306,0.955493211746,0.0281460881233,1.13864409924,0.736714363098,0.409209400415,0.12443100661,1.01665282249,0.56444144249,0.0184952765703,0.454392194748,0.0547554120421,0.0967007279396,0.201399996877,0.351547211409,0.0939633473754,0.00291643035598,1.41500556469,0.3500610888,0.133123084903,0.12548610568,0.876086711884,0.668595671654,0.366329103708,0.360192060471,0.123450934887,0.206827446818,0.761998534203,0.30903545022,0.340677499771,0.249153316021,0.0871850326657,0.446080565453,0.129708692431,0.797186911106,0.297337085009,0.523686826229,1.12527608871,0.137469559908,0.163046479225,0.172252982855,0.218455925584,0.172812521458,0.0639309436083,0.194983109832,0.378890037537,0.278692185879,0.533556163311,0.325716227293,0.22065217793,0.143413841724,0.156686738133,0.257423400879,0.342053860426,0.451365083456,0.505581498146,0.607901632786,0.0916098579764,0.588377296925,0.350192338228,0.174953430891,0.389387696981,0.287273555994,0.145200371742,0.3799020648,0.0848981291056,0.117222428322,0.120925694704,0.435173124075,0.723478674889,0.151427358389,0.585122704506,0.288936972618,0.440900921822,0.50367808342,1.04750394821,0.194678068161,0.496176868677,0.254495292902,0.882530391216,0.0409997366369,0.353998243809,0.303144961596,1.00639045238,1.60638332367,0.3960955441,0.553176760674,0.205300822854,0.296804785728,0.310998260975,0.285757571459,1.2045468092,0.4268425107,0.186188831925,0.167313888669,0.224124252796,0.476130515337,1.05612921715,0.0628934428096,0.752065122128,0.799817740917,0.239826917648,0.427867293358,0.265598535538,0.348779648542,0.823329031467,0.419896692038,0.0706271082163,0.711086630821,0.0682197883725,0.25664639473,0.275534331799,0.0329573974013,0.404878854752,0.486214697361,0.414933562279,1.67178690434,0.837390363216,0.342126458883,0.0973304659128,0.275180995464,0.0881491154432,1.42983865738,0.0802584215999,0.326498121023,0.04291190207,1.2464376688,0.264485180378,0.347639709711,1.30132079124,0.466975003481,0.446977138519,0.684506595135,0.134998261929,0.215279251337,0.563947737217,0.0409852042794,0.222486674786,0.16882930696,0.808244049549,0.224571689963,0.0405456572771,0.711852192879,0.117655649781,0.311693042517,0.129873543978,0.642852365971,0.519812524319,0.166728228331,0.123541474342,0.178966611624,0.0503949150443,0.136931225657,0.139844357967,0.299242049456,0.0836600288749,0.112200126052,0.310636907816,0.00119942659512,0.515911877155,0.00603260425851,0.233006969094,0.392270803452,0.102130934596,0.130426883698,0.0781128108501,0.685767352581,0.0,0.0624775029719,1.07788014412,0.0403289981186,0.012762167491,0.151328191161,0.212698832154,0.223433345556,0.0808732435107,0.0233415383846,0.0260097645223,0.0904315263033,0.730594277382,0.22208365798,0.219757378101,0.344001948833,0.0968955308199,0.727620184422,0.0962341427803,0.506369113922,0.149518579245,0.296898007393,0.856347382069,0.107002407312,0.237689957023,0.0608373768628,0.0949419736862,0.0280025079846,0.0475765988231,0.00745559390634,0.0465741157532,0.111344844103,0.299966752529,0.755277872086,0.624016940594,0.133637979627,0.231824561954,0.0680224150419,0.0581201501191,0.113221764565,0.848203718662,0.252291142941,0.610662639141,0.235734313726,1.03904485703,0.117471687496,0.081574395299,0.0473164655268,0.000258735381067,0.0792598947883,0.19526655972,0.116156160831,0.480258792639,0.207172110677,0.199486449361,0.016125626862,0.0449394248426,0.112045966089,0.0582880675793,0.0257109217346,0.0343439280987,0.136737063527,0.515285670757,0.0210714507848,0.172955602407,0.288236558437,0.386309623718,0.265674978495,0.705624818802,0.178886085749,0.259748369455,0.150732889771,0.362576812506,0.0228782258928,0.10534401238,0.0817439183593,0.0664382949471,0.228248864412,0.0504238121212,0.0721146166325,0.573955655098,0.0699692070484,0.542943894863,0.413863122463,0.0834305137396,0.272823780775,0.104136891663,0.603894770145,0.271911531687,0.215436473489,0.29043674469,0.0974288508296,0.254132062197,0.307765245438,0.101477101445,0.169228792191,0.353511184454,0.498420357704,0.884420692921,0.196818292141,0.0759235396981,0.0514766126871,0.109744600952,0.0218543689698,0.105409830809,0.188349843025,0.45596280694,0.0889509171247,0.0524764917791,0.14036719501,0.812693476677,0.166160851717,0.0235681664199,0.497383713722,0.424679011106,0.299845546484,0.49012196064,0.0837255716324,0.0660506933928,0.301665514708,0.0629854202271,0.0834603384137,0.148252919316,0.113775365055,0.341927230358,0.105818293989,0.367444008589,0.120509512722,0.50799292326,0.645297825336,0.585892915726,0.365892857313,0.0433569140732,1.29883670807,0.155817463994,0.109008990228,0.0535082742572,0.0993666425347,0.172887861729,0.0898534655571,0.424733191729,0.879217207432,0.151029139757,0.0898728966713,0.334753364325,0.0351119786501,0.0246910173446,0.321908414364,0.129035994411,1.72306704521,0.20708990097,0.10694552213,0.0290539637208,0.358162969351,0.56077092886,0.0380373336375,0.334549874067,0.754700422287,0.301891237497,0.536951839924,0.447073101997,0.238656938076,0.0474444031715,0.154088556767,0.068417340517,1.24425649643,0.348230779171,0.120594352484,0.153643772006,0.328118413687,0.0613052807748,0.0286911986768,0.0832844823599,0.0246495939791,0.346696287394,0.334197819233,0.284341424704,0.258418142796,0.145188137889,0.0497153624892,0.205184936523,0.171066626906,0.00587373552844,0.529326677322,0.120983555913,0.186026796699,0.126844912767,0.0316416919231,0.0646310448647,0.0468294769526,0.0795638859272,0.109634928405,0.152058541775,0.312677979469,0.217675358057,0.514586389065,0.302985280752,0.0807054489851,0.0381999015808,0.039359614253,0.543830871582,0.301310956478,0.20838406682,0.0886624157429,0.17822894454,0.135992795229,0.0398668125272,0.0,0.0807981565595,0.00759145338088,0.5444034338,0.0,0.360001534224,0.0973558798432,0.0991958677769,0.00484571279958,0.104287654161,0.21819499135,0.00433064438403,0.0980242341757,0.15560887754,0.157727211714,0.336426883936,0.0417726486921,0.317041188478,0.0692051053047,0.691726207733,0.0302800275385,0.284557372332,0.612085223198,0.249919250607,0.398207813501,0.0485797636211,0.115905843675,0.109697468579,0.140782132745,0.232576563954,0.0447880923748,0.988627195358,0.0819891989231,0.0456864349544,0.375585138798,0.060499586165,0.0614393502474,0.432949990034,0.0540361851454,0.69608181715,0.088680550456,0.53665292263,0.26789277792,0.124783962965,0.320423543453,0.330974549055,0.382241904736,0.330626577139,0.520333230495,0.303645759821,0.499115049839,0.0,0.0315231233835,0.357880294323,0.0108219794929,0.222714826465,0.283780485392,0.149202227592,0.348202049732,0.0522895082831,0.91867852211,0.0058659282513,0.0269492976367,0.315242499113,0.170504122972,0.0116247842088,0.0,0.905167877674,0.0273323114961,0.543454408646,0.221148833632,0.0914552584291,0.0825597941875,0.577453911304,0.097031198442,0.0943603590131,0.0319372005761,0.200702041388,0.0183852557093,0.00775174563751,0.150672361255,0.231942787766,0.0877205729485,0.234979867935,0.435006886721,0.123412199318,0.460381865501,0.126404970884,0.0470745153725,0.940707087517,0.256038576365,0.116454951465,0.27558991313,0.205518841743,0.205815285444,0.306548625231,0.475210994482,0.0262763313949,0.549805283546,0.247323349118,0.110819227993,0.0815779790282,0.463855206966,0.132848262787,0.163348719478,0.0654058307409,0.0968836694956,0.111699387431,0.227122873068,0.295507013798,0.0589850619435,0.133528336883,0.239226192236,0.081115655601,0.0984198525548,0.195482328534,0.100241973996,0.519070506096,0.0898541882634,0.05110315606,0.193586558104,0.069239705801,0.233188182116,0.229975298047,0.0501622892916,0.214280083776,0.556434988976,0.0617706775665,0.147985398769,0.26548832655,0.193730801344,0.0756323114038,0.115584857762,0.0,0.820625245571,0.816745936871,0.234646856785,0.238110214472,0.0290135573596,0.17818044126,0.384023398161,0.166575536132,0.0286512505263,0.236124619842,0.465945631266,0.0465712100267,0.0,0.461912870407,0.0408978126943,0.123621791601,0.532084345818,0.260158061981,0.0153628401458,0.48518794775,0.405613511801,0.0974799916148,0.002028263174,0.189232319593,0.504923462868,0.246768072248,0.0729176402092,0.133252218366,0.0218441896141,0.447903633118,0.0793664753437,0.0495242327452,0.351225763559,0.109924413264,0.00966833159328,0.0856710001826,0.0927796810865,0.698047220707,0.502553343773,0.25794890523,0.206694707274,0.0880682691932,0.0170097667724,1.73579013348,0.0637416169047,0.0817649289966,0.0185244064778,0.321384638548,0.163928464055,0.062129445374,0.256677061319,0.579614818096,0.173389941454,0.9894515872,0.6310364604,0.208888083696,0.0308347996324,0.119926378131,0.243046328425,0.184711173177,0.0658067911863,0.362985730171,0.0211843606085,0.0842475816607,0.200719848275,0.0838585495949,0.11920376122,0.326634794474,0.156621620059,0.540325760841,0.170342847705,0.220006614923,0.0416090786457,0.667329072952,0.0373006202281,0.698617398739,0.234514787793,0.0171660780907,0.0563105121255,0.197930648923,0.112733706832,0.0979594066739,0.692545115948,0.00808825716376,0.853014647961,0.25189948082,0.115653760731,0.223612859845,0.023414131254,0.289404928684,0.125571340322,0.19720159471,0.0372763350606,0.420067369938,0.0786448419094,0.147509276867,0.0923555344343,0.173000633717,0.291506201029,0.373142927885,0.249525487423,0.152018472552,0.231566652656,0.125194177032,0.174589917064,0.634802341461,0.225696355104,0.00340418401174,0.00469005247578,0.154069796205,0.287550926208,0.486680388451,0.269406229258,0.45289799571,0.457286983728,0.0989334583282,0.538056313992,0.733662128448,0.684168815613,0.210190027952,0.0966897830367,0.2868309021,0.286401808262,0.449031233788,0.362015873194,0.0956060141325,0.442764699459,0.227455183864,0.098563067615,0.0288855936378,0.056629691273,0.15892624855,0.639261901379,0.146750047803,0.00974144972861,0.424619674683,0.159807235003,0.271481305361,0.598587810993,0.0426071584225,0.155890747905,0.633493959904,0.365672409534,0.683591961861,0.420900136232,0.0343602858484,0.636603832245,0.165346175432,0.0427745431662,0.587379395962,0.264892429113,0.484593957663,0.0160824917257,0.10023188591,0.114807315171,0.189667135477,0.239232182503,0.187239289284,0.617372810841,0.236187443137,0.260342329741,0.33669924736,0.510977864265,0.881796240807,0.538290679455,0.318985104561,0.046101488173,0.259321480989,0.00403044559062,0.173104524612,1.3634134531,0.325212448835,0.0918364673853,0.399604678154,0.0382711812854,0.0981498286128,0.616583168507,0.0569974519312,0.112085923553,0.02026399225,1.17203462124,0.0525064393878,0.240658760071,0.250843375921,0.292535811663,0.200012952089,0.277422368526,0.234376654029,0.430871963501,0.20595254004,0.424166768789,0.357867270708,0.34291690588,0.216806277633,0.11245110631,0.0990190431476,0.0643108785152,0.214184775949,0.416813164949,0.0668666288257,0.616961717606,0.00414608838037,0.100115574896,0.095753878355,0.243013516068,0.277141869068,0.0490010716021,0.63415646553,0.188323676586,0.162424489856,0.313941329718,0.224429160357,0.332141429186,0.437361180782,0.152219071984,0.319601297379,0.155224367976,0.0594333559275,0.422896981239,0.122184768319,0.288677632809,0.234547570348,0.310994416475,0.168996661901,0.0680059045553,0.0426286421716,0.0667366161942,0.560410678387,0.0497597455978,0.462919056416,0.02571217902,0.0836968049407,0.303579717875,0.059099689126,0.0898979380727,0.360378712416,0.619358837605,0.735063314438,0.180358931422,0.453228294849,0.112389825284,0.734800338745,0.33439669013,0.577627956867,0.348726153374,0.278669774532,0.161295577884,0.0656482130289,0.0247750356793,0.138580188155,0.26059743762,0.852203071117,0.037310898304,0.225803926587,0.101273424923,0.220575004816,1.7656083107,0.265425622463,0.230806201696,0.210186034441,0.461532145739,1.20325469971,0.242867380381,0.199308410287,0.106213927269,0.1240689978,0.236381679773,0.685944378376,0.261161774397,0.169013649225,0.0811783075333,0.309051841497,0.0858168005943,0.0596194416285,0.412534713745,0.717387795448,0.290355622768,0.0285697802901,0.247420608997,0.471375375986,0.0616730898619,0.167034626007,0.160229444504,0.0936371311545,0.262804120779,0.440886169672,0.203226432204,0.417889505625,0.0895887315273,0.674274861813,0.249485060573,0.0796529427171,0.202420040965,0.0,0.385306924582,0.221840068698,0.183984830976,0.0609185881913,0.180287137628,0.0539876520634,0.249147951603,0.35754314065,0.343293607235,0.0768077969551,0.292384654284,0.724269747734,0.125615864992,0.394745230675,0.191120550036,0.247578755021,0.0395897477865,0.158707797527,0.215793952346,0.15858964622,0.378185033798,0.387967824936,0.141412422061,0.0656268969178,0.538658320904,0.196453273296,0.115438863635,0.380016028881,0.277905136347,0.76518368721,0.00450088828802,0.171447321773,0.594636142254,0.0766415372491,0.4630895257,0.167693436146,0.153810292482,0.278192579746,0.218754068017,0.0999440327287,0.383337080479,0.219326049089,0.230356529355,0.34562727809,0.664133071899,0.0573113001883,0.344821602106,0.300983220339,0.0199478920549,0.03403628245,0.138504967093,0.301464378834,0.0389890559018,0.174383923411,0.0216700807214,0.164094895124,0.370119214058,0.0887486860156,0.04371066764,0.977654993534,0.0765948593616,0.0151954144239,0.127007573843,0.339424222708,0.521603524685,0.0296190921217,0.182283788919,0.320682317019,0.0739738568664,0.142701044679,0.198986366391,0.104207761586,0.0477009005845,0.12718680501,0.493101477623,0.339687913656,0.0351283662021,0.30796456337,0.882607698441,0.00673687737435,0.412808179855,0.15635304153,0.0985356122255,0.22170342505,0.0634418949485,0.0247932672501,0.162322878838,0.0924688577652,0.439838171005,0.124194800854,0.401001989841,0.572617650032,0.279551535845,0.315096914768,0.506493449211,0.213337391615,0.0370602980256,0.487327486277,0.356599628925,0.259278267622,1.17932784557,0.405414044857,0.0954484716058,0.174125090241,0.752398967743,0.10812394321,0.907628178596,0.273178964853,0.029475396499,0.0180439781398,0.0404286831617,0.737952589989,0.165483355522,0.435557544231,0.778393983841,0.100809082389,0.179776266217,0.101578004658,0.163955628872,1.06988441944,0.822328984737,0.329649180174,0.723385334015,0.0,0.0767976045609,1.81443417072,0.185639619827,0.30491527915,0.0411467924714,0.588662445545,0.213699012995,0.309020668268,0.689097046852,0.216810122132,1.0139477253,1.01719796658,0.420674085617,0.440511524677,0.0921934321523,0.223051443696,0.938626348972,0.0962019339204,0.167036786675,0.486917555332,0.095629543066,0.649649143219,0.0392030254006,0.00263355672359,0.0085542537272,0.039966635406,0.143329799175,0.169206231833,0.451641768217,0.0439451374114,0.0109827453271,0.103517569602,0.00332419900224,1.10922932625,0.0305266231298,0.25640758872,0.0784168764949,0.22039949894,0.437524139881,0.0524113737047,0.0285085365176,0.756192684174,2.10565114021,1.07021296024,0.0300823263824,0.211363404989,0.038214687258,0.38951715827,1.16110181808,1.22023463249,0.442546367645,0.0,0.153647601604,0.249762877822,1.27462232113,0.00398811604828,0.605591237545,0.36657166481,0.0162050817162,0.220355719328,1.37358427048,0.871842503548,0.506398737431,0.00126433139667,0.104754462838,0.0893413200974,0.29436185956,0.0368239060044,0.187787890434,0.352899760008,0.164333000779,0.9029712677,0.282760113478,1.69920134544,0.575873315334,0.165213569999,2.03004407883,0.0331678688526,0.202400743961,0.0663023069501,0.0954115837812,0.322103202343,0.471058487892,0.0373438932002,0.364478468895,0.495486885309,0.178397864103,2.64804673195,1.13652729988,0.581656694412,0.759989917278,0.0926590561867,0.521550655365,0.279314279556,1.84242784977,0.194370508194,0.72606498003,0.190426811576,0.145905539393,0.0827555954456,0.0700085386634,1.07512950897,0.349079251289,0.00572335347533,0.159039050341,0.0,0.418454200029,0.137747615576,0.888288855553,0.568627536297,0.0444913245738,0.398213326931,0.547487437725,0.0,0.0481125339866,0.480906367302,0.557335853577,0.0335348471999,0.366164088249,0.490068554878,0.360268712044,0.239668399096,0.614749252796,1.54078471661,0.00584710761905,0.0960219278932,0.392137914896,0.718632459641,0.0462491512299,0.202420592308,0.0100462874398,0.182617276907,0.373130142689,0.461084336042,0.591275393963,0.305173814297,0.181974053383,1.55530929565,0.00396627467126,0.00638807518408,0.0292851552367,0.058528535068,0.0637547671795,0.0176266469061,0.438214659691,0.927628278732,0.00818292889744,0.185436531901,0.0104441475123,0.426349401474,0.413896173239,0.0102653419599,0.193759471178,0.420456647873,0.408870458603,0.262871533632,0.981372952461,0.0784137621522,0.00561604928225,0.0341993160546,0.136701315641,0.260909408331,0.0244785603136,0.960208237171,0.730493068695,0.0257785152644,0.00482282135636,1.62211096287,1.40928697586,0.270593076944,0.0,0.0929093211889,0.0928202569485,1.01095306873,0.295922011137,0.919104635715,0.0565133914351,0.0392495319247,1.32005608082,0.0828777328134,0.185492947698,0.284389317036,0.0144618954509,0.286901146173,0.266155779362,0.0433819256723,0.267766088247,0.00550555856898,0.171093001962,0.0660206973553,0.774269282818,0.0560988746583,0.537157058716,0.0358852818608,0.170274496078,0.291037559509,0.145905092359,0.0999322682619,0.239481344819,0.261325240135,0.990607619286,0.338349312544,0.334529459476,1.48857355118,0.246932730079,0.62024474144,0.0179472230375,0.0,0.729630172253,0.227358102798,0.0196478441358,0.0382107794285,0.0268973205239,0.064184345305,0.121808335185,0.735052764416,0.188209980726,0.694021224976,0.669881761074,0.262824714184,1.02484536171,0.344434767962,0.346989959478,0.351865053177,1.10877037048,0.985978603363,1.21281301975,0.61542069912,0.576838612556,0.484014511108,0.256332069635,0.000465658027679,0.0110111823305,0.00664069643244,0.15625551343,0.32505479455,0.186694145203,0.220469668508,0.00964854098856,0.598525822163,0.119858153164,1.01539409161,0.413034975529,0.239266619086,0.266185820103,0.150865763426,0.0346029549837,0.0835099294782,0.173062592745,0.173202931881,0.104171872139,0.254671037197,0.12847571075,0.316680431366,0.141194820404,0.663191437721,0.116143092513,0.295903295279,0.109218686819,0.250530868769,0.0285062268376,0.307014703751,0.536296069622,0.36050093174,0.139382183552,0.106678418815,0.152927517891,0.134488672018,0.757481098175,0.0131200309843,0.34274995327,0.0614972710609,0.493394702673,1.5908356905,0.377164453268,0.39751303196,0.333727031946,0.0569485202432,0.052483305335,0.118138372898,0.89188927412,0.476105362177,0.935485780239,1.49390017986,0.640363812447,0.0622704327106,0.463797390461,0.313012242317,0.013657017611,0.143890291452,0.71065056324,0.475090801716,0.521677315235,0.0544055327773,0.107769072056,0.577439069748,0.843071818352,0.00846375338733,0.0417283810675,0.0979923605919,1.48996698856,0.342655152082,0.681839466095,0.00200832867995,0.459403812885,0.28796428442,0.226467609406,0.157367303967,0.0380024239421,0.106850512326,0.258231133223,0.050451464951,0.140507921576,0.730896830559,0.38563683629,0.825888395309,0.460978388786,1.44784998894,0.449597299099,0.643733143806,0.084270812571,0.261488378048,0.205286160111,0.685399413109,0.0447766445577,0.0133968750015,0.134545788169,0.234395682812,0.036536693573,0.603774011135,2.01720142365,0.873439192772,0.333510130644,0.0276930686086,0.353652745485,0.769282281399,0.210805907845,1.03356337547,0.186281591654,0.066035836935,0.0208834316581,1.47536015511,0.184342607856,0.196599751711,0.000789360376075,0.929623007774,0.000820487504825,0.532330513,1.58563160896,0.193584144115,0.00550667801872,0.570930123329,0.128518819809,0.899965882301,0.328396499157,0.392145186663,1.61903786659,0.130467563868,0.682239830494,0.334823578596,0.18794503808,0.10324972868,0.747837245464,0.659962534904,1.03664970398,0.123248517513,0.303369402885,0.341566383839,0.897938847542,0.183549672365,0.639069020748,0.264121323824,0.0242574382573,0.202028036118,0.574307799339,0.284860700369,0.54991620779,0.444007813931,0.33443453908,0.235812440515,0.0195163376629,0.184842213988,0.0131156612188,0.173488944769,0.0162880606949,0.259789705276,0.331799387932,0.876691043377,0.214750647545,0.360672205687,0.260344356298,0.404089003801,0.0641502737999,1.19931221008,0.193486213684,0.0139155359939,1.405854702,1.47940707207,0.26093313098,0.784079968929,0.00270651513711,0.743290305138,1.9095968008,0.152789831161,0.137080967426,0.815683186054,0.0825709179044,0.272101104259,0.391794353724,0.26602050662,0.0,0.896493673325,0.549890697002,0.947268843651,0.856482028961,0.241089373827,1.23888742924,0.019984619692,0.208681479096,0.123666517437,0.414181441069,0.424944758415,1.65332758427,0.79891782999,0.0390235893428,0.223461061716,0.00954080186784,0.0802864208817,0.394157916307,0.485299676657,0.00281521002762,0.00694415671751,0.271683424711,0.334625393152,0.520215988159,0.0353891476989,0.259749591351,0.50136089325,0.103133000433,0.035559989512,0.0274477880448,0.0112718846649,0.103152036667,0.894903063774,0.136145099998,1.21196424961,1.48360991478,0.378003627062,0.26723626256,0.909831881523,0.240759924054,0.5005017519,0.395499557257,0.870692431927,1.48782968521,0.111812382936,0.702907502651,1.01922082901,0.0347519554198,0.293311208487,0.0827097222209,0.20158521831,0.623993992805,1.10028135777,1.03144419193,0.0102110570297,0.00197234936059,0.591554760933,1.58203923702,0.132609426975,0.0838264375925,0.758621692657,0.00788631290197,0.0925155431032,0.352643758059,0.186696693301,0.0631145909429,0.293080449104,0.528681159019,0.0269745700061,0.0714835003018,0.00499825552106,0.0787766426802,0.613181233406,0.0373739041388,0.0899370312691,0.869100391865,0.130638107657,0.0756408572197,0.220979258418,0.498122572899,0.139098703861,0.469101369381,0.0804531723261,0.627973437309,0.416075944901,0.309111475945,1.61314070225,0.420875191689,0.00460355915129,0.122216321528,0.0288113001734,0.717196106911,0.142468973994,0.0639190897346,0.120164230466,0.0232041124254,0.29077321291,0.846429884434,0.062934152782,0.632802069187,0.382881253958,0.46478292346,0.0671190321445,0.108648605645,0.173281684518,0.11258725822,1.36437439919,0.297060191631,1.11144995689,0.554524898529,0.234779804945,0.00931127369404,0.456830918789,0.275482535362,0.379945039749,0.271560460329,0.427795052528,0.414078742266,0.112027354538,0.359713286161,2.31256079674,0.00227372604422,0.0509854182601,0.757978379726,1.07191348076,0.226904183626,0.0619727000594,0.680145204067,0.096976146102,0.156818717718,0.606055378914,0.151952475309,0.655544161797,0.027482798323,0.109623380005,0.539927601814,0.0488088689744,0.803938150406,0.0493581742048,0.240348041058,0.161526173353,0.13880109787,0.698099434376,0.190353482962,0.183193773031,0.0861342325807,0.799754977226,0.0162109360099,0.134410008788,0.559146404266,0.150414034724,0.561150550842,0.281530708075,0.168401628733,0.112196303904,0.149782016873,0.446459084749,0.404351681471,0.161130324006,0.12395144999,0.205673024058,0.364714562893,0.00461224466562,1.47051227093,0.287521183491,0.168303310871,0.120325177908,0.200696080923,0.0214677527547,0.312058210373,0.612558245659,1.82462513447,1.04592609406,1.07583069801,0.104550674558,0.385119855404,0.771660268307,0.0589025244117,0.133039757609,0.0120826754719,0.187496542931,0.00390499504283,1.14210319519,0.167288824916,0.562765955925,0.706343412399,0.139840096235,0.192461773753,0.492763221264,0.00899639073759,0.00247892620973,0.221047237515,0.547046482563,0.0278882198036,1.52163517475,0.333651721478,0.638749837875,0.0699326917529,0.333980977535,1.19062042236,0.187966138124,0.10307431221,1.33323395252,0.198327928782,0.331980526447,0.534711360931,0.599549293518,0.141367271543,0.154799520969,0.967928767204,0.921099245548,0.952286958694,0.179580494761,0.122886098921,0.468090802431,0.083534784615,0.134668081999,0.431305736303,0.91877156496,1.6200594902,0.679100215435,0.0584733523428,0.192718699574,0.256771862507,0.596422970295,0.355031102896,0.0465433485806,0.0,0.781020820141,0.817221701145,0.127271831036,0.084343649447,0.0361483469605,0.0482152141631,0.00176528980955,0.163496792316,0.457131803036,0.00310657825321,0.439253956079,0.0146500943229,0.200843736529,0.413456976414,0.278072208166,0.397045552731,0.337003737688,1.03985738754,0.504556715488,1.0906264782,0.442872107029,0.765421271324,0.0511680953205,0.200121074915,0.0401756875217,0.236966580153,0.00584204448387,0.165664821863,0.0720405653119,0.0319103188813,1.12694132328,1.3076518774,0.0623326934874,0.991663515568,0.696230232716,0.127672925591,0.503301024437,0.162004113197,0.56482899189,1.04435956478,0.206437543035,0.0722280591726,0.345145821571,1.43374979496,0.100906543434,0.413912326097,0.249668821692,0.535221219063,0.161098763347,0.827925562859,0.0178932193667,0.31698846817,0.160239368677,0.310660898685,0.125606939197,0.525636792183,0.53752887249,0.43828818202,0.0277030076832,0.357996463776,0.22023896873,0.501267671585,0.405443429947,0.312545597553,0.188267722726,1.35010778904,0.275808274746,0.698728740215,0.456077039242,0.64832419157,0.361968010664,0.322597980499,0.0268850103021,0.516086876392,0.259655535221,0.105399057269,0.283887982368,0.772011220455,0.973999798298,0.314538419247,0.124733544886,0.239467099309,1.09844303131,0.591026425362,0.62568372488,0.179925858974,0.591371715069,0.0214129239321,2.25432658195,0.678184986115,0.516432464123,0.0703075379133,0.162686482072,0.212926283479,0.3521296978,0.618643045425,0.417493611574,0.000508492114022,0.405338615179,1.10413575172,0.665278732777,0.36066660285,0.147962912917,0.0526118837297,0.515992939472,0.0553496293724,1.67852175236,0.328661084175,0.448877960443,0.181507751346,0.26990917325,0.187748268247,0.101679608226,0.239505663514,0.00243898760527,1.04030323029,0.42715498805,0.103445857763,0.214418083429,0.108330614865,0.228436365724,0.0893272683024,0.630406677723,0.267909020185,0.534792423248,0.362615168095,0.0434423573315,0.278288394213,0.0930247157812,0.754614591599,1.43500602245,0.216528400779,0.0307234358042,0.5832208395,0.295989215374,0.785259544849,0.442814826965,0.410700351,0.280782938004,0.152820780873,0.365579098463,0.256241202354,0.633183777332,0.0655091628432,0.887889802456,0.472531884909,0.620692908764,0.693180322647,0.942319452763,0.062408734113,1.03169000149,0.48921328783,0.497305095196,0.760065197945,0.496768474579,0.860466480255,0.0206931382418,0.516183912754,0.181644842029,0.359989464283,0.385128170252,0.4665171206,0.20311152935,0.0702991113067,0.287494897842,1.08045482635,0.0810824930668,0.655074894428,0.164481535554,0.403649061918,0.573355317116,0.315146565437,0.626646161079,0.287204235792,0.819795668125,0.583982110023,0.662785410881,0.726345300674,0.0753648132086,0.651945114136,0.243012011051,0.705553114414,0.265413969755,0.207642003894,0.902829349041,0.214318409562,0.119643695652,0.775385499001,0.161241665483,0.227637186646,1.40765976906,0.301206737757,0.0539208799601,0.113254666328,0.00886082276702,0.320334732533,0.0327097699046,0.24562856555,0.267331540585,0.343531608582,0.0173868220299,0.0260851271451,0.2791031003,0.642625510693,0.257842481136,0.681263327599,0.264339029789,1.02168238163,0.371981889009,1.04131245613,0.65196877718,0.057388573885,0.179854825139,0.724390506744,0.839556097984,0.641131877899,0.118094101548,0.315822422504,0.0,0.39092874527,0.0305593255907,1.31775760651,1.20077860355,0.132992208004,0.429686248302,0.352282196283,0.783320188522,0.70085477829,0.418055862188,0.243431061506,0.18268802762,0.998040437698,0.0445269607008,0.796256303787,0.0811899900436,0.520536541939,0.274946898222,0.0430716834962,0.0504497177899,0.00437662471086,0.371785998344,0.341195732355,0.668997168541,0.0315348058939,0.454613775015,0.118505351245,0.172765180469,0.738605320454,0.999241411686,0.733806490898,0.0843045935035,0.437201589346,0.128907084465,0.195774778724,0.124479614198,0.571222007275,0.711775183678,0.214035972953,0.533574044704,0.165908634663,0.264396965504,0.137902572751,0.316470742226,0.0,0.840401172638,0.173300623894,0.554762840271,0.538427650928,0.222352042794,0.344940215349,1.83160126209,0.792651176453,0.286293268204,0.215133175254,0.330308258533,0.424402832985,0.762334704399,0.231905266643,0.663652122021,0.160559535027,0.151636675,0.939603984356,0.26179215312,0.0018231025897,0.632824718952,0.12683750689,0.817208349705,0.278607934713,0.171115338802,0.206534653902,0.00386900268495,0.453920125961,0.730683207512,0.324975550175,1.03940165043,0.648264408112,0.0254544168711,0.180150046945,0.324006974697,0.48850864172,0.320257902145,0.534283399582,0.171733081341,0.451618522406,0.260657727718,0.175121039152,0.922362744808";
double[] featuresArray= Lists.newArrayList(t1.split(",")).parallelStream()
.mapToDouble(va -> Double.parseDouble(va)).toArray();;
INDArray features =Nd4j.create(featuresArray);
INDArray predicted = model.output(features,false);
double scoreThreadHold=0.6;
BooleanIndexing.replaceWhere(predicted, 0.0, Conditions.lessThan(scoreThreadHold));
BooleanIndexing.replaceWhere(predicted, 1.0, Conditions.greaterThanOrEqual(scoreThreadHold));
int[] shape= predicted.shape();
int rowNum=shape[0];
int columnNum=shape[1];
int fixedTopNum=40;
List<Pair<Integer,Double>> list=Lists.newArrayListWithExpectedSize(fixedTopNum);
mLog.info(String.valueOf(list.isEmpty()));
Pair<Integer,Double> pairWithMinValue=Collections.min(list, new Comparator<Pair<Integer,Double>>() {
@Override
public int compare(Pair<Integer,Double> p1, Pair<Integer,Double> p2) {
double compareValue=p1.getValue()-p2.getValue();
int compareIntValue=0;
if(compareValue>0){
compareIntValue= 1;
}
if(compareValue<0){
compareIntValue =-1;
}
return compareIntValue;
}
});
//
// for(int i=0;i<rowNum;i++){
// List<Pair<Integer,Double>> list=Lists.newArrayListWithExpectedSize(fixedTopNum);
// for(int j=0;j<columnNum;j++){
// double value=predicted.getDouble(i,j);
// if(value>0.6){
// //consider to pick up to final labels
//
//
// }
//
//
// }
//
//
//
//
// }
MatchCondition op = new MatchCondition(predicted, Conditions.equals(1.0));
int matchLabeledNum = Nd4j.getExecutioner().exec(op,Integer.MAX_VALUE).getInt(0);
//MatchCondition op1 = new MatchCondition(predicted, Conditions.equals(1.0));
int realLabeledNum = Nd4j.getExecutioner().exec(op,Integer.MAX_VALUE).getInt(0);
INDArray argMaxAlongDim0 = Nd4j.argMax(predicted,1);
mLog.info(predicted.toString());
mLog.info("matchLabeledNum: "+matchLabeledNum+"");
//41566
Map<String,String> ksMap= (Map<String, String>) CommonUtil.deSerializeObj("/data/projects/vcg_work/searchRecommend/src/main/resources/kwWordMap.obj");
mLog.info(ksMap.get("36295"));
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment