Win7上Spark WordCount运行过程及异常

package com.husor.Spark/**
* Created by huxiu on 2014/11/26.
*/import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.SparkContext._object SparkWordCount { def main(args: Array[String]) { println("Test is starting......") System.setProperty("hadoop.home.dir", "d:\\winutil\\") //val conf = new SparkConf().setAppName("WordCount")
// .setMaster("spark://Master:7077")
// .setSparkHome("SPARK_HOME")
// .set("spark.cores.max","2") //val spark = new SparkContext(conf)

//以本地模式运行WordCount程序
val spark = new SparkContext("local","WordCount",System.getenv("SPARK_HOME"))
val file = spark.textFile("hdfs://Master:9000/data/test1")

//将输出结果直接输出到控制台上
//file.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_+_).collect().foreach(println) val wordCounts = file.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_+_)

//将输出结果直接输出到hdfs上
wordCounts.saveAsTextFile("hdfs://Master:9000/user/huxiu/WordCountOutput")

spark.stop() println("Test is Succeed!!!") }
}

<property>
<name>mapreduce.jobtracker.staging.root.dir</name>
<value>/user</value>
</property>

[Spark@Master hadoop]$ hadoop fs -mkdir /user
// :: WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[Spark@Master hadoop]$ hadoop fs -mkdir /user/huxiu
// :: WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[Spark@Master hadoop]$ hadoop fs -chown huxiu:huxiu /user/huxiu
// :: WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable

"C:\Program Files\Java\jdk1.7.0_67\bin\java" -Didea.launcher.port= "-Didea.launcher.bin.path=D:\ScalaIDE\IntelliJ IDEA Community Edition 14.0.1\bin" -Dfile.encoding=UTF- -classpath "C:\Program Files\Java\jdk1.7.0_67\jre\lib\charsets.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\deploy.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\javaws.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\jce.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\jfr.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\jfxrt.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\management-agent.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\plugin.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\resources.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\rt.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\access-bridge-64.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\dnsns.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\jaccess.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\localedata.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\sunec.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\sunjce_provider.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\sunmscapi.jar;C:\Program Files\Java\jdk1.7.0_67\jre\lib\ext\zipfs.jar;D:\IntelliJ_IDE\WorkSpace\out\production\Test;D:\vagrant\data\Scala2.10.4\lib\scala-actors-migration.jar;D:\vagrant\data\Scala2.10.4\lib\scala-actors.jar;D:\vagrant\data\Scala2.10.4\lib\scala-library.jar;D:\vagrant\data\Scala2.10.4\lib\scala-reflect.jar;D:\vagrant\data\Scala2.10.4\lib\scala-swing.jar;D:\SparkSrc\spark-assembly-1.1.0-hadoop2.4.0.jar;D:\ScalaIDE\IntelliJ IDEA Community Edition 14.0.1\lib\idea_rt.jar" com.intellij.rt.execution.application.AppMain com.husor.Spark.SparkWordCount
Test is starting......
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
// :: INFO SecurityManager: Changing view acls to: huxiu,
// :: INFO SecurityManager: Changing modify acls to: huxiu,
// :: INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(huxiu, ); users with modify permissions: Set(huxiu, )
// :: INFO Slf4jLogger: Slf4jLogger started
// :: INFO Remoting: Starting remoting
// :: INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriver@huxiu-PC:54972]
// :: INFO Remoting: Remoting now listens on addresses: [akka.tcp://sparkDriver@huxiu-PC:54972]
// :: INFO Utils: Successfully started service 'sparkDriver' on port .
// :: INFO SparkEnv: Registering MapOutputTracker
// :: INFO SparkEnv: Registering BlockManagerMaster
// :: INFO DiskBlockManager: Created local directory at C:\Users\huxiu\AppData\Local\Temp\spark-local--9dad
// :: INFO Utils: Successfully started service 'Connection manager for block manager' on port .
// :: INFO ConnectionManager: Bound socket to port with id = ConnectionManagerId(huxiu-PC,)
// :: INFO MemoryStore: MemoryStore started with capacity 969.6 MB
// :: INFO BlockManagerMaster: Trying to register BlockManager
// :: INFO BlockManagerMasterActor: Registering block manager huxiu-PC: with 969.6 MB RAM
// :: INFO BlockManagerMaster: Registered BlockManager
// :: INFO HttpFileServer: HTTP File server directory is C:\Users\huxiu\AppData\Local\Temp\spark-423dcd83-624e-404a-bbf6-a1190f77290f
// :: INFO HttpServer: Starting HTTP Server
// :: INFO Utils: Successfully started service 'HTTP file server' on port .
// :: INFO Utils: Successfully started service 'SparkUI' on port .
// :: INFO SparkUI: Started SparkUI at http://huxiu-PC:4040
// :: WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
// :: INFO AkkaUtils: Connecting to HeartbeatReceiver: akka.tcp://sparkDriver@huxiu-PC:54972/user/HeartbeatReceiver
// :: INFO MemoryStore: ensureFreeSpace() called with curMem=, maxMem=
// :: INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 159.9 KB, free 969.4 MB)
// :: INFO FileInputFormat: Total input paths to process :
// :: INFO deprecation: mapred.tip.id is deprecated. Instead, use mapreduce.task.id
// :: INFO deprecation: mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id
// :: INFO deprecation: mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap
// :: INFO deprecation: mapred.task.partition is deprecated. Instead, use mapreduce.task.partition
// :: INFO deprecation: mapred.job.id is deprecated. Instead, use mapreduce.job.id
// :: INFO SparkContext: Starting job: saveAsTextFile at SparkWordCount.scala:
// :: INFO DAGScheduler: Registering RDD (map at SparkWordCount.scala:)
// :: INFO DAGScheduler: Got job (saveAsTextFile at SparkWordCount.scala:) with output partitions (allowLocal=false)
// :: INFO DAGScheduler: Final stage: Stage (saveAsTextFile at SparkWordCount.scala:)
// :: INFO DAGScheduler: Parents of final stage: List(Stage )
// :: INFO DAGScheduler: Missing parents: List(Stage )
// :: INFO DAGScheduler: Submitting Stage (MappedRDD[] at map at SparkWordCount.scala:), which has no missing parents
// :: INFO MemoryStore: ensureFreeSpace() called with curMem=, maxMem=
// :: INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 3.3 KB, free 969.4 MB)
// :: INFO DAGScheduler: Submitting missing tasks from Stage (MappedRDD[] at map at SparkWordCount.scala:)
// :: INFO TaskSchedulerImpl: Adding task set 1.0 with tasks
// :: INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID , localhost, ANY, bytes)
// :: INFO Executor: Running task 0.0 in stage 1.0 (TID )
// :: INFO HadoopRDD: Input split: hdfs://Master:9000/data/test1:0+27
// :: INFO Executor: Finished task 0.0 in stage 1.0 (TID ). bytes result sent to driver
// :: INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID ) in ms on localhost (/)
// :: INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool
// :: INFO DAGScheduler: Stage (map at SparkWordCount.scala:) finished in 0.500 s
// :: INFO DAGScheduler: looking for newly runnable stages
// :: INFO DAGScheduler: running: Set()
// :: INFO DAGScheduler: waiting: Set(Stage )
// :: INFO DAGScheduler: failed: Set()
// :: INFO DAGScheduler: Missing parents for Stage : List()
// :: INFO DAGScheduler: Submitting Stage (MappedRDD[] at saveAsTextFile at SparkWordCount.scala:), which is now runnable
// :: INFO MemoryStore: ensureFreeSpace() called with curMem=, maxMem=
// :: INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 56.2 KB, free 969.4 MB)
// :: INFO DAGScheduler: Submitting missing tasks from Stage (MappedRDD[] at saveAsTextFile at SparkWordCount.scala:)
// :: INFO TaskSchedulerImpl: Adding task set 0.0 with tasks
// :: INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID , localhost, PROCESS_LOCAL, bytes)
// :: INFO Executor: Running task 0.0 in stage 0.0 (TID )
// :: INFO BlockFetcherIterator$BasicBlockFetcherIterator: maxBytesInFlight: , targetRequestSize:
// :: INFO BlockFetcherIterator$BasicBlockFetcherIterator: Getting non-empty blocks out of blocks
// :: INFO BlockFetcherIterator$BasicBlockFetcherIterator: Started remote fetches in ms
// :: INFO FileOutputCommitter: Saved output of task 'attempt_201411261415_0000_m_000000_1' to hdfs://Master:9000/user/huxiu/WordCountOutput/_temporary/0/task_201411261415_0000_m_000000
// :: INFO SparkHadoopWriter: attempt_201411261415_0000_m_000000_1: Committed
// :: INFO Executor: Finished task 0.0 in stage 0.0 (TID ). bytes result sent to driver
// :: INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID ) in ms on localhost (/)
// :: INFO DAGScheduler: Stage (saveAsTextFile at SparkWordCount.scala:) finished in 0.847 s
// :: INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool
// :: INFO SparkContext: Job finished: saveAsTextFile at SparkWordCount.scala:, took 1.469630513 s
// :: INFO SparkUI: Stopped Spark web UI at http://huxiu-PC:4040
// :: INFO DAGScheduler: Stopping DAGScheduler
// :: INFO MapOutputTrackerMasterActor: MapOutputTrackerActor stopped!
// :: INFO ConnectionManager: Selector thread was interrupted!
// :: INFO ConnectionManager: ConnectionManager stopped
// :: INFO MemoryStore: MemoryStore cleared
// :: INFO BlockManager: BlockManager stopped
// :: INFO BlockManagerMaster: BlockManagerMaster stopped
Test is Succeed!!!
// :: INFO SparkContext: Successfully stopped SparkContext
// :: INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.
// :: INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.
// :: INFO Remoting: Remoting shut down
// :: INFO RemoteActorRefProvider$RemotingTerminator: Remoting shut down.Process finished with exit code